ЛР 4 - БПИ2303 - Любченко - на 4 #125
+5
-1
@@ -7,7 +7,7 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -18,3 +18,7 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
# Published Nginx port for lab4.
|
||||
PUBLIC_PORT=18081
|
||||
API_IMAGE_TAG=local
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# MTUSI Messenger, lab4 — PostgreSQL + Redis/Valkey HTTP API.
|
||||
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
|
||||
# on another machine with the same platform yields the same layers.
|
||||
#
|
||||
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
|
||||
# Development-only tooling lives in requirements-dev.txt and never enters this image.
|
||||
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app/src
|
||||
|
||||
# Pinned runtime dependencies (versions fixed for reproducible builds).
|
||||
RUN pip install --no-cache-dir \
|
||||
psycopg2-binary==2.9.11 \
|
||||
redis==7.0.1 \
|
||||
bcrypt==4.0.1 \
|
||||
boto3==1.42.97 \
|
||||
Pillow==12.3.0
|
||||
|
||||
# Unprivileged runtime account; the image's default root user is never used at runtime.
|
||||
RUN groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
ARG APP_REVISION=local
|
||||
ENV APP_REVISION=$APP_REVISION
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=10001:10001 src/ /app/src/
|
||||
COPY --chown=10001:10001 migrations/ /app/migrations/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
# Configuration comes from the environment; secrets are read from files at runtime.
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_GRACE_SECONDS=8 \
|
||||
SESSION_TTL_SECONDS=3600 \
|
||||
SESSION_STORE=redis \
|
||||
DB_POOL_MIN=2 \
|
||||
DB_POOL_MAX=10 \
|
||||
DB_CONNECT_TIMEOUT=5 \
|
||||
DB_STATEMENT_TIMEOUT=10
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,6 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
PUBLIC_PORT ?= $(or $(shell sed -n 's/^PUBLIC_PORT=//p' .env 2>/dev/null),18081)
|
||||
BASE_URL ?= http://localhost:$(PUBLIC_PORT)
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
@@ -1,37 +1,75 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа — Любченко Никита, БПИ2303.
|
||||
- Целевая оценка — **4**. Трек оценки 5 не выбран.
|
||||
- Commit/PR добавляет автор после публикации.
|
||||
- Использованы контракт лабораторной, реализация lab3, Python 3.12, psycopg2, Redis, bcrypt, boto3, Pillow и MinIO. ИИ использовался при реализации, подготовке проверок и отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine с Compose, Make и Python 3.9+ для публичных тестов. При первой сборке нужен доступ к реестрам образов, PyPI, GitHub и прокси модулей Go. Базовый Go-образ берётся из публичного реестра AWS ECR. Для модулей указан резервный прокси на случай ошибки основного.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
cp -n .env.example .env
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build
|
||||
python3 scripts/manage.py restore
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Адрес по умолчанию — `http://localhost:18081`. Его можно изменить через `PUBLIC_PORT` в `.env`. Bootstrap сохраняет существующие значения секретов. Compose запускает миграции и инициализацию бакета до API. Версии исходников MinIO и mc указаны в `infra/Dockerfile.s3`. Для остановки с сохранением данных применяется `docker compose down`. Команда `docker compose down -v` предназначена для намеренного удаления учебных данных PostgreSQL, Redis и S3.
|
||||
|
||||
Для OpenAPI-валидации установите `requirements-dev.txt` в отдельный venv и передайте его Python в `make validate PYTHON=...`. Скрипты негативных и нагрузочных проверок также используют Pillow. Для проверки перезапуска и переключения доступны следующие команды.
|
||||
|
||||
```sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 scripts/manage.py restore
|
||||
python3 student-tests/attachment_resilience.py --base-url http://localhost:18081
|
||||
```
|
||||
|
||||
## Архитектура и согласованность
|
||||
|
||||
Nginx направляет запросы на две API-реплики. PostgreSQL хранит пользователей, чаты, сессии, сообщения, метаданные файлов и результаты идемпотентности. Redis кэширует сессии. Приватный MinIO хранит оригиналы в `s3_data`. Любая реплика может выдать оригинал после проверки членства в чате. Настройки адреса, региона и бакета передаются через `S3_ENDPOINT`, `S3_REGION` и `S3_BUCKET`. Учётные данные берутся из файлов Compose secrets.
|
||||
|
||||
Загрузка принимает сырые PNG/JPEG. API читает блоками по 64 KiB, считает SHA-256 и проверяет изображение из временного файла. Предел API — 10 MiB и 40 млн пикселей. Для upload Nginx буферизует запрос с верхним пределом 11 MiB, чтобы пограничное превышение лимита приложения вернуло JSON 413. Каждая API допускает до десяти одновременных загрузок и одно декодирование за раз. Чтение тела ограничено 30 секундами, у S3 connect timeout 2 секунды и read timeout 5 секунд. Скачивание передаётся потоком.
|
||||
|
||||
API резервирует запись `pending`, блокирует её на время PUT и фиксирует `ready` перед ответом 201. Если PUT прошёл, а транзакция оборвалась, запись остаётся `pending`. Cleaner раз в минуту удаляет до 25 просроченных записей после 15-минутного grace period. Он пропускает строки, заблокированные активным PUT. Повторное удаление объекта безопасно, а сбой S3 оставляет запись для повторной попытки.
|
||||
|
||||
Вложения привязаны к чату до отправки сообщения. PostgreSQL проверяет совпадение чата, уникальность ссылок и число позиций. Порядок ссылок сохраняется в истории. Сообщение, ссылки и запись результата идемпотентности создаются одной транзакцией. Пустой и отсутствующий `attachment_ids` используют прежний хеш lab3. При изменении вложений или их порядка под тем же ключом API возвращает 409.
|
||||
|
||||
Бакет создаётся повторяемым bootstrap. MinIO и bootstrap получают административные секреты. API и cleaner получают отдельную учётную запись с правами чтения, записи и удаления под префиксом `attachments/` и чтения региона бакета. S3 слушает внутреннюю сеть Compose, веб-консоль отключена.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Проверка | Результат | Файл в evidence/ |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Текстовые сообщения, общая сессия, идемпотентность lab3 | `Текстовый прогон через Nginx` | Обе API ответили; повтор с пустым списком совпал | `text-baseline.txt` |
|
||||
| 201 ready, оригинал и права участника | `make test` | Публичный smoke прошёл с реальным S3 | `smoke.txt` |
|
||||
| Порядок вложений, повтор, запрет чужого чата | `student-tests/message_attachments.py` | История и повтор совпали; некорректные ссылки отклонены | `message-attachments-http.txt` |
|
||||
| Лимиты, повреждённый файл, MIME и ACL | `student-tests/attachment_errors.py` | Ожидаемые 400, 404, 413, 415; при отказе S3 — 503 | `attachment-errors-live.txt` |
|
||||
| Отказ загрузившей реплики и перезапуск с volumes | `student-tests/attachment_resilience.py` | Оригинал совпал побайтно; подтверждены разные X-Instance-Id | `attachment-resilience.txt` |
|
||||
| Прерывание загрузки и сбой после PUT | `student-tests/attachments_http.py, attachment_cleanup.py и s3_cleanup_live.py` | Прерванный запрос не создаёт запись; реальный объект после сбоя удалён; активный PUT защищён блокировкой | `attachments-http.txt, attachment-cleanup.txt, cleanup-live.txt` |
|
||||
| 10 одновременных загрузок | `student-tests/upload_load.py` | 10/10, все SHA-256 совпали; память ниже лимита | `upload-load.json` |
|
||||
| Повторный bootstrap и приватный бакет | `docker compose run --rm s3-bootstrap и s3-bootstrap verify` | Запись и чтение под API credentials; анонимный GET — 403 | `s3-bootstrap-repeat.txt, s3-private.txt` |
|
||||
| Публичная проверка persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS | `persistence.txt` |
|
||||
| Публичный persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | Сессия, сообщения и оригинал сохранились | `persistence.txt` |
|
||||
| Публичный failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Две реплики до отказа, 131 успешная запись, 0 ошибок, восстановление 0,875 с | `failover.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Эксперименты и ограничения
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Окружение прогона — Docker Engine 29.5.3, Compose 5.1.4, aarch64, 10 CPU и 7,75 GiB RAM для Docker VM. Каждая API ограничена 0,5 CPU и 256 MiB RAM. Перед нагрузкой стенд уже обслуживал smoke и проверки вложений.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Нагрузка стартовала общим барьером. Десять клиентов загрузили по 8,674,572 байт PNG (1700 × 1700). Всего 10 запросов за 1.35 с, 7.41 успешных загрузок/с, ошибок 0. p50/p95/p99 — 1.258/1.349/1.349 с. Выборка cgroup снималась каждые 50 мс.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
| Реплика | Базовая память, MiB | Выборочный пик, MiB | CPU, с |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| api-a | 75.3 | 147.1 | 0.565 |
|
||||
| api-b | 90.7 | 144.2 | 0.564 |
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
После остановки загрузившей реплики другая вернула тот же SHA-256 за 5,66 с от начала сценария. После перезапуска стенда оригинал получен за 6,93 с. При остановке S3 операции с вложениями возвращали 503, текстовая отправка продолжала работать. После запуска MinIO оригинал снова скачался побайтно.
|
||||
|
||||
Стенд рассчитан на отказ одной API-реплики и перезапуск с сохранением volumes. Общие PostgreSQL, Redis, Nginx, MinIO и Docker-хост остаются отдельными компонентами одного стенда. Нагрузочный эксперимент проверяет десять одновременных запросов с заданным размером PNG. Выборочный максимум cgroup включает файловый кэш и процесс измерения, поэтому это метрика памяти контейнера, а не только RSS Python.
|
||||
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
name: mtusi-lab4
|
||||
|
||||
x-api: &api
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
image: mtusi-lab4-api:${API_IMAGE_TAG:-local}
|
||||
environment: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: '8080'
|
||||
APP_ENV: lab4
|
||||
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: redis
|
||||
SHUTDOWN_GRACE_SECONDS: '8'
|
||||
DB_POOL_MIN: '2'
|
||||
DB_POOL_MAX: '16'
|
||||
DB_CONNECT_TIMEOUT: '1'
|
||||
DB_STATEMENT_TIMEOUT: '4'
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
cpus: 0.50
|
||||
mem_limit: 256m
|
||||
stop_grace_period: 9s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1)"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 2
|
||||
start_period: 3s
|
||||
# Deliberately no auto-restart: failover hook must leave its target stopped.
|
||||
restart: 'no'
|
||||
|
||||
services:
|
||||
s3:
|
||||
image: mtusi-lab4-s3:2025-09-07
|
||||
build: &s3-build
|
||||
context: .
|
||||
dockerfile: infra/Dockerfile.s3
|
||||
entrypoint: [/bin/sh, -ec]
|
||||
command:
|
||||
- |
|
||||
MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user)
|
||||
MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password)
|
||||
export MINIO_ROOT_USER MINIO_ROOT_PASSWORD
|
||||
exec minio server /data
|
||||
environment:
|
||||
MINIO_BROWSER: "off"
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
networks: [backend]
|
||||
|
||||
s3-bootstrap:
|
||||
image: mtusi-lab4-s3:2025-09-07
|
||||
build: *s3-build
|
||||
entrypoint: [/bin/sh, /bootstrap.sh]
|
||||
command: []
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes: [./infra/s3-bootstrap.sh:/bootstrap.sh:ro]
|
||||
depends_on: [s3]
|
||||
networks: [backend]
|
||||
|
||||
postgres:
|
||||
image: postgres:15-alpine@sha256:54e0dd8e8cddcdbb9156c2486eed5a43eccb7214d7c6828097d41ce049fb3984
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [pg_data:/var/lib/postgresql/data]
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:8da19c7399dca900c6822a1cf30f9d2285585e2c4b457ff4095b0f4500054d3f
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 96mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [backend]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 10
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
image: mtusi-lab4-api:${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url, cursor_secret]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
cpus: 0.50
|
||||
mem_limit: 256m
|
||||
restart: 'no'
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
attachment-cleaner:
|
||||
image: mtusi-lab4-api:${API_IMAGE_TAG:-local}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${API_IMAGE_TAG:-local}
|
||||
command: [python, -m, messenger.clean_attachments]
|
||||
environment: *api-env
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
networks: [backend]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
api-a:
|
||||
<<: *api
|
||||
api-b:
|
||||
<<: *api
|
||||
|
||||
nginx:
|
||||
image: nginx:1.28.2-alpine@sha256:5b4900b042ccfa8b0a73df622c3a60f2322faeb2be800cbee5aa7b44d241649e
|
||||
entrypoint: [/bin/sh, -ec]
|
||||
command:
|
||||
- 'cp /etc/nginx/lab/upstreams.conf /etc/nginx/upstreams.conf; /bin/sh /etc/nginx/lab/healthwatch.sh & exec nginx -g "daemon off;"'
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra:/etc/nginx/lab:ro
|
||||
ports: ['127.0.0.1:${PUBLIC_PORT:-18081}:8080']
|
||||
networks: [frontend, backend]
|
||||
depends_on:
|
||||
api-a:
|
||||
condition: service_healthy
|
||||
api-b:
|
||||
condition: service_healthy
|
||||
cpus: 0.50
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'kill -0 $$(cat /tmp/healthwatch.pid) && wget -T 1 -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
cursor_secret:
|
||||
file: ./secrets/cursor_secret
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
@@ -3,3 +3,5 @@
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
|
||||
Итоговый прогон от 13 ноября 2026 года описан в [отчёте](../REPORT.md). `upload-load.json` содержит все десять запросов и показания памяти обеих реплик. `attachment-resilience.txt` содержит ID загрузившей и скачавшей реплик, контрольную сумму и время восстановления. `cleanup-live.txt` подтверждает удаление объекта после сбоя между PUT и COMMIT на настоящих PostgreSQL и MinIO. Ранние проверки с подстановками отмечены в соответствующих файлах.
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved
|
||||
PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely
|
||||
@@ -0,0 +1,6 @@
|
||||
Container mtusi-lab4-s3-1 Stopping
|
||||
Container mtusi-lab4-s3-1 Stopped
|
||||
Container mtusi-lab4-s3-1 Starting
|
||||
Container mtusi-lab4-s3-1 Started
|
||||
PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx
|
||||
PASS: S3 outage returned 503, text remained available, original recovered in 0.28s
|
||||
@@ -0,0 +1,19 @@
|
||||
Stopping API replica: api-a b790e8fa9b8f
|
||||
Flag --time has been deprecated, use --timeout instead
|
||||
b790e8fa9b8f2cae0b2d72ab43a49f7edd6942ce8e7c4a61409b980ecf69523f
|
||||
b790e8fa9b8f2cae0b2d72ab43a49f7edd6942ce8e7c4a61409b980ecf69523f
|
||||
005b94ff9f8de9ca8ca22ffebb0f768b8dfe15e26673b2ed000bd84deda89943
|
||||
Container mtusi-lab4-s3-1 Restarting
|
||||
Container mtusi-lab4-nginx-1 Restarting
|
||||
Container mtusi-lab4-redis-1 Restarting
|
||||
Container mtusi-lab4-api-a-1 Restarting
|
||||
Container mtusi-lab4-api-b-1 Restarting
|
||||
Container mtusi-lab4-postgres-1 Restarting
|
||||
Container mtusi-lab4-s3-1 Started
|
||||
Container mtusi-lab4-postgres-1 Started
|
||||
Container mtusi-lab4-api-a-1 Started
|
||||
Container mtusi-lab4-nginx-1 Started
|
||||
Container mtusi-lab4-redis-1 Started
|
||||
Container mtusi-lab4-api-b-1 Started
|
||||
PASS: b790e8fa9b8f -> 005b94ff9f8d, SHA-256 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75, 5.66s
|
||||
PASS: after full restart via b790e8fa9b8f, SHA-256 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75, 6.93s
|
||||
@@ -0,0 +1,3 @@
|
||||
PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant
|
||||
PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping
|
||||
NOTE: metadata and S3 are test substitutes; real Compose integration not exercised
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1 @@
|
||||
PASS: actual MinIO object existed after failed commit and was removed by cleanup
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18081"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.875}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,9 @@
|
||||
api-a running healthy
|
||||
api-b running healthy
|
||||
attachment-cleaner running
|
||||
migrate exited
|
||||
nginx running healthy
|
||||
postgres running healthy
|
||||
redis running healthy
|
||||
s3 running
|
||||
s3-bootstrap exited
|
||||
@@ -0,0 +1 @@
|
||||
PASS: ordered attachments, replay, invalid input, cross-chat rejection and history
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: production Store SQL on PostgreSQL 15, legacy replay, ordered links and message history
|
||||
PASS: conflicting order, foreign/missing/pending attachments and duplicate UUID rejected
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18081"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
b790e8fa9b8f2cae0b2d72ab43a49f7edd6942ce8e7c4a61409b980ecf69523f
|
||||
005b94ff9f8de9ca8ca22ffebb0f768b8dfe15e26673b2ed000bd84deda89943
|
||||
@@ -0,0 +1,9 @@
|
||||
Container mtusi-lab4-s3-1 Running
|
||||
Container mtusi-lab4-s3-bootstrap-run-6a89f8a75fda Creating
|
||||
Container mtusi-lab4-s3-bootstrap-run-6a89f8a75fda Created
|
||||
Bucket created successfully `storage/messenger-attachments`.
|
||||
Access permission for `storage/messenger-attachments` is set to `private`
|
||||
Created policy `lab4-attachments` successfully.
|
||||
Attached Policies: [lab4-attachments]
|
||||
To User: lab4-api-828a0e3a8a5bff5c
|
||||
PASS: private S3 bucket and scoped API policy configured
|
||||
@@ -0,0 +1,4 @@
|
||||
Container mtusi-lab4-s3-1 Running
|
||||
Container mtusi-lab4-s3-bootstrap-run-324b1c08be01 Creating
|
||||
Container mtusi-lab4-s3-bootstrap-run-324b1c08be01 Created
|
||||
PASS: authenticated round trip; anonymous GET returned 403
|
||||
@@ -0,0 +1,4 @@
|
||||
PASS: SQL migrations applied to an existing lab3 schema in PGlite
|
||||
PASS: old message preserved, valid same-chat link accepted
|
||||
PASS: cross-chat links, duplicates, and invalid positions rejected
|
||||
PASS: deleting a chat removes its message links
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:18081"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: text POST, replay with empty attachment_ids, history and shared session
|
||||
API instance IDs: 005b94ff9f8d, b790e8fa9b8f
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"concurrency": 10,
|
||||
"payload_bytes": 8674572,
|
||||
"sha256": "203a03973c23714cb0da1dd23d9c434128e147c584b43fcd4e16f3078aefae9f",
|
||||
"elapsed_seconds": 1.3492113749999999,
|
||||
"successful_rps": 7.411737097161667,
|
||||
"latency_seconds": {
|
||||
"p50": 1.257803875,
|
||||
"p95": 1.348739208,
|
||||
"p99": 1.348739208
|
||||
},
|
||||
"requests": [
|
||||
{
|
||||
"instance": "005b94ff9f8d",
|
||||
"id": "91c3f144-be2a-4a9a-8701-7fdc3d6c9a2c",
|
||||
"seconds": 1.345623708
|
||||
},
|
||||
{
|
||||
"instance": "b790e8fa9b8f",
|
||||
"id": "9b099dcc-30e1-4338-9fb0-3c52e28e7d54",
|
||||
"seconds": 1.348739208
|
||||
},
|
||||
{
|
||||
"instance": "b790e8fa9b8f",
|
||||
"id": "23cf471e-f448-4069-8717-28be9a37631b",
|
||||
"seconds": 1.2876072499999998
|
||||
},
|
||||
{
|
||||
"instance": "005b94ff9f8d",
|
||||
"id": "d94988d8-c641-4c32-8e5c-d375ddcd3d9e",
|
||||
"seconds": 1.294252334
|
||||
},
|
||||
{
|
||||
"instance": "005b94ff9f8d",
|
||||
"id": "6f7c96c1-6016-4448-a8c5-2fb5b70c4fb4",
|
||||
"seconds": 1.093782875
|
||||
},
|
||||
{
|
||||
"instance": "005b94ff9f8d",
|
||||
"id": "cd1aca0e-2b24-4c73-b315-900096dd6a78",
|
||||
"seconds": 1.257803875
|
||||
},
|
||||
{
|
||||
"instance": "b790e8fa9b8f",
|
||||
"id": "50404cba-be99-42e2-ab82-8d24a698865d",
|
||||
"seconds": 1.3204030420000001
|
||||
},
|
||||
{
|
||||
"instance": "005b94ff9f8d",
|
||||
"id": "7730a266-5564-4382-990c-120241f4ce49",
|
||||
"seconds": 1.058885083
|
||||
},
|
||||
{
|
||||
"instance": "b790e8fa9b8f",
|
||||
"id": "e105a930-9530-4b0c-bbc7-988d27c45010",
|
||||
"seconds": 1.132140125
|
||||
},
|
||||
{
|
||||
"instance": "b790e8fa9b8f",
|
||||
"id": "d1171d2a-39fa-49cc-a8c0-5ac9d0f62bbe",
|
||||
"seconds": 1.1198395420000002
|
||||
}
|
||||
],
|
||||
"memory_sample_interval_seconds": 0.05,
|
||||
"resources": {
|
||||
"api-a": {
|
||||
"baseline_bytes": 78909440,
|
||||
"peak_sample_bytes": 154247168,
|
||||
"limit_bytes": 268435456,
|
||||
"samples": 24,
|
||||
"cpu_seconds": 0.564806
|
||||
},
|
||||
"api-b": {
|
||||
"baseline_bytes": 95072256,
|
||||
"peak_sample_bytes": 151191552,
|
||||
"limit_bytes": 268435456,
|
||||
"samples": 25,
|
||||
"cpu_seconds": 0.563697
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/bin/python scripts/validate_openapi.py
|
||||
/Users/atmose/Documents/Codex/2026-11-06/new-chat/work/lab3-venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,13 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM public.ecr.aws/docker/library/golang:1.24.2-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS build
|
||||
ENV CGO_ENABLED=0 GOPROXY="https://proxy.golang.org|https://goproxy.io|direct"
|
||||
WORKDIR /src/minio
|
||||
ADD https://codeload.github.com/minio/minio/tar.gz/refs/tags/RELEASE.2025-09-07T16-13-09Z /tmp/minio.tar.gz
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/minio.tar.gz --strip-components=1 && go build -o /out/minio .
|
||||
WORKDIR /src/mc
|
||||
ADD https://codeload.github.com/minio/mc/tar.gz/refs/tags/RELEASE.2025-08-13T08-35-41Z /tmp/mc.tar.gz
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build tar -xzf /tmp/mc.tar.gz --strip-components=1 && go build -o /out/mc .
|
||||
|
||||
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
|
||||
COPY --from=build /out/minio /out/mc /usr/local/bin/
|
||||
ENTRYPOINT ["minio"]
|
||||
@@ -5,3 +5,12 @@
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
|
||||
Lab4 запускает MinIO и клиент mc из версий, указанных в `Dockerfile.s3`. Bootstrap подключается к MinIO внутри сети Compose. API и cleaner используют отдельные credentials. При повторном запуске bootstrap подтверждает приватность бакета и заново применяет политику приложения.
|
||||
|
||||
```sh
|
||||
docker compose run --rm s3-bootstrap
|
||||
docker compose run --rm s3-bootstrap verify
|
||||
```
|
||||
|
||||
Проверка записывает временный объект под `attachments/`, сверяет его под credentials API, проверяет анонимный HTTP 403 и удаляет тестовый объект.
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# Active readiness bridge for open-source Nginx. No Docker socket or credentials.
|
||||
set -eu
|
||||
probe() {
|
||||
if wget -T 1 -q -O /dev/null "http://$1:8080/health/ready" 2>/dev/null; then
|
||||
echo up > "/tmp/$1.health"
|
||||
else
|
||||
echo down > "/tmp/$1.health"
|
||||
fi
|
||||
}
|
||||
control() { cat "/tmp/control-$1" 2>/dev/null || echo serve:boot; }
|
||||
echo $$ > /tmp/healthwatch.pid
|
||||
while :; do
|
||||
sleep 1
|
||||
probe api-a &
|
||||
first=$!
|
||||
probe api-b &
|
||||
second=$!
|
||||
wait "$first"
|
||||
wait "$second"
|
||||
# Read desired state AFTER probes. A generation is acknowledged only after
|
||||
# the matching upstream configuration has been applied, never from a stale probe.
|
||||
a_control=$(control api-a)
|
||||
b_control=$(control api-b)
|
||||
a_state=$(cat /tmp/api-a.health)
|
||||
b_state=$(cat /tmp/api-b.health)
|
||||
case "$a_control" in drain:*) a_state=down;; esac
|
||||
case "$b_control" in drain:*) b_state=down;; esac
|
||||
: > /tmp/upstreams.next
|
||||
for service in api-a api-b; do
|
||||
if [ "$service" = api-a ]; then state=$a_state; else state=$b_state; fi
|
||||
suffix=''
|
||||
[ "$state" = up ] || suffix=' down'
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=2s%s;\n' "$service" "$suffix" >> /tmp/upstreams.next
|
||||
done
|
||||
if ! cmp -s /tmp/upstreams.next /etc/nginx/upstreams.conf; then
|
||||
cp /etc/nginx/upstreams.conf /tmp/upstreams.previous
|
||||
# Rename on the same filesystem: readers never see missing/partial config.
|
||||
cp /tmp/upstreams.next /etc/nginx/upstreams.next
|
||||
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
|
||||
if nginx -t && nginx -s reload; then
|
||||
echo 'readiness upstream set updated'
|
||||
else
|
||||
cp /tmp/upstreams.previous /etc/nginx/upstreams.next
|
||||
mv -f /etc/nginx/upstreams.next /etc/nginx/upstreams.conf
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
printf '%s %s\n' "$a_control" "$a_state" > /tmp/applied-api-a.next
|
||||
mv -f /tmp/applied-api-a.next /tmp/applied-api-a
|
||||
printf '%s %s\n' "$b_control" "$b_state" > /tmp/applied-api-b.next
|
||||
mv -f /tmp/applied-api-b.next /tmp/applied-api-b
|
||||
done
|
||||
@@ -0,0 +1,49 @@
|
||||
worker_processes 1;
|
||||
worker_shutdown_timeout 6s;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
log_format lab '$time_iso8601 $request_method $uri $status upstream=$upstream_addr '
|
||||
'instance=$upstream_http_x_instance_id time=$request_time';
|
||||
access_log /dev/stdout lab;
|
||||
error_log /dev/stderr warn;
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
include /etc/nginx/upstreams.conf;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 1m;
|
||||
client_body_timeout 5s;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection close;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Request-Id $http_x_request_id;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
# Keep a proxy ceiling above the API limit so the API can return its JSON 413.
|
||||
client_max_body_size 11m;
|
||||
proxy_read_timeout 45s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream off;
|
||||
proxy_pass http://api;
|
||||
}
|
||||
location ~ ^/api/v1/attachments/[^/]+/content$ {
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 10s;
|
||||
# A shared S3 outage returns 503 without marking the API itself down.
|
||||
proxy_next_upstream error timeout http_502 http_504;
|
||||
proxy_pass http://api;
|
||||
}
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
umask 077
|
||||
|
||||
bucket=${S3_BUCKET:-messenger-attachments}
|
||||
case "$bucket" in
|
||||
''|*[!a-z0-9.-]*) echo 'Invalid S3 bucket name' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
config=$(mktemp -d)
|
||||
trap 'rm -rf "$config"' EXIT
|
||||
export MC_CONFIG_DIR="$config"
|
||||
root_user=$(cat /run/secrets/s3_root_user)
|
||||
root_password=$(cat /run/secrets/s3_root_password)
|
||||
api_user=$(cat /run/secrets/s3_access_key_id)
|
||||
api_password=$(cat /run/secrets/s3_secret_access_key)
|
||||
endpoint=${S3_ENDPOINT:-http://s3:9000}
|
||||
|
||||
if [ "${1:-}" = verify ]; then
|
||||
export MC_HOST_api="http://$api_user:$api_password@${endpoint#http://}"
|
||||
key="attachments/probe-${config##*/}"
|
||||
object="api/$bucket/$key"
|
||||
trap 'mc rm "$object" >/dev/null 2>&1 || true; rm -rf "$config"' EXIT
|
||||
printf '%s' 'lab4-private-bucket-probe' | mc pipe "$object" >/dev/null
|
||||
test "$(mc cat "$object")" = 'lab4-private-bucket-probe'
|
||||
python3 - "$endpoint/$bucket/$key" <<'PYTEST'
|
||||
import sys
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
|
||||
try:
|
||||
with urlopen(sys.argv[1], timeout=5):
|
||||
raise SystemExit("FAIL: anonymous GET succeeded")
|
||||
except HTTPError as exc:
|
||||
if exc.code != 403:
|
||||
raise
|
||||
print("PASS: authenticated round trip; anonymous GET returned 403")
|
||||
PYTEST
|
||||
mc rm "$object" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
until mc alias set storage "$endpoint" "$root_user" "$root_password" >/dev/null 2>&1 &&
|
||||
mc admin info storage >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
echo 'S3 bootstrap could not connect or authenticate' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
mc mb --ignore-existing "storage/$bucket"
|
||||
mc anonymous set none "storage/$bucket"
|
||||
cat > "$config/api-policy.json" <<POLICY
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::$bucket/attachments/*"]
|
||||
}, {
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::$bucket"]
|
||||
}]
|
||||
}
|
||||
POLICY
|
||||
mc admin policy create storage lab4-attachments "$config/api-policy.json"
|
||||
mc admin user add storage "$api_user" "$api_password" >/dev/null
|
||||
mc admin policy attach storage lab4-attachments --user "$api_user"
|
||||
echo 'PASS: private S3 bucket and scoped API policy configured'
|
||||
@@ -0,0 +1,2 @@
|
||||
server api-a:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
server api-b:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
@@ -0,0 +1,51 @@
|
||||
-- Migration 0001: initial schema for lab2.
|
||||
-- Idempotent: safe to re-run on an existing database.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
password_hash text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
|
||||
-- full table scan on every page read.
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
|
||||
ON messages (chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
|
||||
ON sessions (user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
|
||||
ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Additive migration: compatible with the lab2 schema and existing messages.
|
||||
CREATE TABLE IF NOT EXISTS message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
key text NOT NULL,
|
||||
request_hash text NOT NULL,
|
||||
response_status integer NOT NULL,
|
||||
response_body jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
-- Results are retained indefinitely in this lab (therefore at least 24 hours).
|
||||
-- There is intentionally no independent cache expiry that could allow duplicates.
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Additive migration. Existing text messages have no attachment links.
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_id_chat_id_key UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
|
||||
object_key text NOT NULL UNIQUE CHECK (object_key LIKE 'attachments/%'),
|
||||
-- pending is internal. The upload API exposes ready after committing metadata.
|
||||
status text NOT NULL CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (id, chat_id),
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_pending_updated_idx
|
||||
ON attachments (updated_at, id) WHERE status = 'pending';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, position),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local credentials once, never print them or overwrite existing files."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = folder / 'postgres_password'
|
||||
|
||||
def save(name, value):
|
||||
path = folder / name
|
||||
if path.exists():
|
||||
return
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as handle:
|
||||
handle.write(value + '\n')
|
||||
|
||||
save('postgres_password', secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
|
||||
save('cache_url', 'redis://redis:6379/0')
|
||||
save('cursor_secret', secrets.token_hex(32))
|
||||
save('s3_root_user', 'lab4-admin-' + secrets.token_hex(8))
|
||||
save('s3_root_password', secrets.token_hex(32))
|
||||
save('s3_access_key_id', 'lab4-api-' + secrets.token_hex(8))
|
||||
save('s3_secret_access_key', secrets.token_hex(24))
|
||||
print('PASS: local secret files ready (existing values preserved)')
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit, bounded operator actions for this lab's Compose project only."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ('api-a', 'api-b')
|
||||
|
||||
|
||||
def command(args, capture=False):
|
||||
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
|
||||
|
||||
|
||||
def inspect(container):
|
||||
return json.loads(command(['docker', 'inspect', container], True))[0]
|
||||
|
||||
|
||||
def containers(service):
|
||||
return compose('ps', '-a', '-q', service, capture=True).splitlines()
|
||||
|
||||
|
||||
def start_services(*services):
|
||||
for service in services:
|
||||
for container in containers(service):
|
||||
command(['docker', 'start', container])
|
||||
|
||||
|
||||
def wait_healthy(service, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
ids = containers(service)
|
||||
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
|
||||
return
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('%s did not become healthy' % service)
|
||||
|
||||
|
||||
def target_service(target):
|
||||
if not target:
|
||||
raise RuntimeError('TARGET_INSTANCE_ID is required')
|
||||
for service in SERVICES:
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
if data['Config']['Hostname'] == target:
|
||||
return service, container
|
||||
raise RuntimeError('Unknown API instance alias')
|
||||
|
||||
|
||||
def set_backend(service, enabled):
|
||||
# Fixed allowlist; shell receives no untrusted instance or credential input.
|
||||
if service not in SERVICES:
|
||||
raise ValueError('Unknown service')
|
||||
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
|
||||
path = '/tmp/control-' + service
|
||||
compose('exec', '-T', 'nginx', 'sh', '-ec',
|
||||
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
|
||||
deadline = time.monotonic() + 10
|
||||
expected = token + (' up' if enabled else ' down')
|
||||
while time.monotonic() < deadline:
|
||||
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
|
||||
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
|
||||
if ack == expected:
|
||||
return
|
||||
time.sleep(.2)
|
||||
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
|
||||
|
||||
|
||||
def stop_one():
|
||||
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
|
||||
running = [c for name in SERVICES for c in containers(name)
|
||||
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
|
||||
and inspect(c)['State']['Running']]
|
||||
if len(running) < 2:
|
||||
raise RuntimeError('Restore the other replica before this experiment')
|
||||
print('Stopping API replica:', service, container[:12], flush=True)
|
||||
command(['docker', 'stop', '--time', '8', container])
|
||||
if inspect(container)['State']['Running']:
|
||||
raise RuntimeError('Target is still running')
|
||||
|
||||
|
||||
def restart():
|
||||
# Never start the one-shot migration service as part of persistence testing.
|
||||
compose('restart', 'postgres', 'redis', 's3', *SERVICES, 'nginx')
|
||||
for service in (*SERVICES, 'nginx'):
|
||||
wait_healthy(service)
|
||||
|
||||
|
||||
def rolling_update():
|
||||
# Run additive migrations once before replacing either API version.
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
for service in SERVICES:
|
||||
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
|
||||
wait_healthy(other)
|
||||
set_backend(other, True)
|
||||
before = containers(service)
|
||||
set_backend(service, False)
|
||||
# The old nginx worker can retain the old upstream table until it exits.
|
||||
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
|
||||
time.sleep(6.5)
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
|
||||
wait_healthy(service)
|
||||
set_backend(service, True)
|
||||
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
|
||||
after = containers(service)
|
||||
if before == after:
|
||||
raise RuntimeError('Replica was not replaced')
|
||||
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
|
||||
|
||||
|
||||
def restore():
|
||||
start_services(*SERVICES)
|
||||
for service in SERVICES:
|
||||
wait_healthy(service)
|
||||
for service in SERVICES:
|
||||
set_backend(service, True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
|
||||
args = parser.parse_args()
|
||||
if args.action == 'stop-one':
|
||||
stop_one()
|
||||
elif args.action == 'restart':
|
||||
restart()
|
||||
elif args.action == 'rolling-update':
|
||||
rolling_update()
|
||||
elif args.action == 'restore':
|
||||
restore()
|
||||
else:
|
||||
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
print(json.dumps({'service': service, 'id': data['Id'][:12],
|
||||
'instance': data['Config']['Hostname'],
|
||||
'running': data['State']['Running'],
|
||||
'started_at': data['State']['StartedAt'],
|
||||
'health': data['State'].get('Health', {}).get('Status'),
|
||||
'image_id': data['Image'],
|
||||
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_limit_bytes': data['HostConfig']['Memory'],
|
||||
'published_ports': data['HostConfig'].get('PortBindings')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py restart
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py rolling-update
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py stop-one
|
||||
@@ -0,0 +1,3 @@
|
||||
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
|
||||
|
||||
__version__ = '1.0.0'
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Entry point: python -m messenger."""
|
||||
import sys
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from . import validation
|
||||
from .attachments import AttachmentStorage, filename_from, upload_file
|
||||
from .db import Database
|
||||
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
|
||||
from .store import Store, utc_now, rfc3339
|
||||
|
||||
SCOPE_CHATS = 'chats'
|
||||
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
|
||||
|
||||
|
||||
class Request:
|
||||
"""One parsed HTTP request; the API layer never touches sockets."""
|
||||
|
||||
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
|
||||
|
||||
def __init__(self, method, path, query, headers, body, request_id, client):
|
||||
self.method = method
|
||||
self.path = path
|
||||
self.query = query
|
||||
self.headers = headers
|
||||
self.body = body
|
||||
self.request_id = request_id
|
||||
self.client = client
|
||||
|
||||
|
||||
class Api:
|
||||
UPLOAD_PATH = re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/attachments$')
|
||||
ROUTES = (
|
||||
('POST', UPLOAD_PATH, 'upload_attachment'),
|
||||
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)$'), 'get_attachment'),
|
||||
('GET', re.compile(r'^/api/v1/attachments/(?P<id>[^/]+)/content$'), 'download_attachment'),
|
||||
('GET', re.compile(r'^/health/live$'), 'health_live'),
|
||||
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
|
||||
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
|
||||
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
|
||||
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
|
||||
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
|
||||
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
|
||||
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
|
||||
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
|
||||
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
|
||||
)
|
||||
|
||||
def __init__(self, config, store: Store, database: Database):
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.database = database
|
||||
self.attachments = AttachmentStorage()
|
||||
|
||||
def dispatch(self, request):
|
||||
"""Return (status, payload). Only ApiError and payloads leave this method."""
|
||||
path_matched = False
|
||||
for method, pattern, handler in self.ROUTES:
|
||||
match = pattern.match(request.path)
|
||||
if match:
|
||||
path_matched = True
|
||||
if method == request.method:
|
||||
return getattr(self, handler)(request, match.groupdict())
|
||||
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
|
||||
|
||||
# ------------------------------------------------------------- health
|
||||
def health_live(self, request, path_params):
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
def health_ready(self, request, path_params):
|
||||
try:
|
||||
self.database.ping()
|
||||
except Exception:
|
||||
raise Unavailable('База данных недоступна')
|
||||
return 200, {'status': 'ok'}
|
||||
|
||||
# -------------------------------------------------------------- users
|
||||
def create_user(self, request, path_params):
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('username', 'display_name', 'password'),
|
||||
('username', 'display_name', 'password'))
|
||||
username = validation.require_username(payload)
|
||||
display_name = validation.require_text_field(
|
||||
payload, 'display_name', 1, self.config.max_display_name_code_points)
|
||||
password = validation.require_text_field(
|
||||
payload, 'password', 12, 128, strip_check=False)
|
||||
password_hash = security.hash_password(password)
|
||||
return 201, self.store.create_user(username, display_name, password_hash)
|
||||
|
||||
def current_user(self, request, path_params):
|
||||
return 200, self._authenticate(request)
|
||||
|
||||
# -------------------------------------------------------------- chats
|
||||
def create_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
|
||||
title = validation.require_text_field(payload, 'title', 1,
|
||||
self.config.max_title_code_points)
|
||||
members = validation.require_member_ids(payload, 100)
|
||||
if actor['id'] not in members:
|
||||
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
|
||||
if not self.store.users_exist(members):
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей')
|
||||
return 201, self.store.create_chat(title, members)
|
||||
|
||||
def list_chats(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
limit, after_key = self._pagination(request, SCOPE_CHATS)
|
||||
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
|
||||
for item in items:
|
||||
item['member_ids'] = self._chat_member_ids(item['id'])
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
|
||||
|
||||
def get_chat(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
chat['member_ids'] = self._chat_member_ids(chat['id'])
|
||||
return 200, chat
|
||||
|
||||
# ----------------------------------------------------------- messages
|
||||
def post_message(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
key = validation.require_idempotency_key(request.headers.get('Idempotency-Key'))
|
||||
payload = self._json_object(request)
|
||||
validation.require_fields(payload, ('text',), ('text', 'attachment_ids'))
|
||||
attachment_ids = validation.require_attachment_ids(payload)
|
||||
text = validation.require_text_field(payload, 'text', 0 if attachment_ids else 1,
|
||||
self.config.max_text_code_points,
|
||||
strip_check=not attachment_ids)
|
||||
return 201, self.store.add_message(chat['id'], actor['id'], text, key, attachment_ids)
|
||||
|
||||
def list_messages(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
|
||||
limit, after_key = self._pagination(request, scope)
|
||||
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
|
||||
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
|
||||
|
||||
def upload_attachment(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
chat = self._member_chat(actor, path_params['chat_id'])
|
||||
filename = filename_from(request.query)
|
||||
mime = request.headers.get('Content-Type', '').split(';', 1)[0].strip().lower()
|
||||
with upload_file(request.body, mime) as (file, size, digest):
|
||||
aid = str(uuid.uuid4())
|
||||
key = 'attachments/' + aid
|
||||
self.store.reserve_attachment(aid, chat['id'], actor['id'], filename,
|
||||
mime, size, digest, key)
|
||||
row = self.store.finish_attachment(
|
||||
aid, lambda object_key: self.attachments.put(object_key, file, size, mime))
|
||||
return 201, self.store.public_attachment(row)
|
||||
|
||||
def _visible_attachment(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
aid = validation.require_uuid_string(path_params['id'], 'attachment_id')
|
||||
row = self.store.visible_attachment(aid, actor['id'])
|
||||
if row is None:
|
||||
raise NotFound('Вложение не найдено')
|
||||
return row
|
||||
|
||||
def get_attachment(self, request, path_params):
|
||||
return 200, self.store.public_attachment(self._visible_attachment(request, path_params))
|
||||
|
||||
def download_attachment(self, request, path_params):
|
||||
row = self._visible_attachment(request, path_params)
|
||||
if request.query.get('variant', ['original']) != ['original']:
|
||||
raise NotFound('Вариант вложения не найден')
|
||||
return 200, self.attachments.download(row)
|
||||
|
||||
# ----------------------------------------------------------- sessions
|
||||
def create_session(self, request, path_params):
|
||||
"""Login: only Authorization: Basic username:password, no JSON body."""
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('basic '):
|
||||
raise Unauthorized('Требуется авторизация Basic')
|
||||
try:
|
||||
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
|
||||
except Exception:
|
||||
raise Unauthorized('Некорректные учётные данные') from None
|
||||
if ':' not in raw:
|
||||
raise Unauthorized('Некорректные учётные данные')
|
||||
username, password = raw.split(':', 1)
|
||||
row = self.store.user_by_username(username)
|
||||
if row is None or not security.verify_password(password, row['password_hash']):
|
||||
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
|
||||
token = security.generate_token()
|
||||
token_hash = security.hash_token(token)
|
||||
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
|
||||
self.store.create_session(token_hash, row['id'], expires_at)
|
||||
user = self.store._public_user({
|
||||
'id': row['id'],
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': row['created_at'],
|
||||
})
|
||||
return 201, {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': rfc3339(expires_at),
|
||||
'user': user,
|
||||
}
|
||||
|
||||
def revoke_session(self, request, path_params):
|
||||
actor = self._authenticate(request)
|
||||
token = self._bearer_token(request)
|
||||
self.store.delete_session(security.hash_token(token))
|
||||
return 204, None
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
def _authenticate(self, request):
|
||||
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
|
||||
|
||||
X-User-Id is ignored: it does not change the session owner and alone
|
||||
grants no access. Basic on business endpoints is 401.
|
||||
"""
|
||||
token = self._bearer_token(request)
|
||||
if token is None:
|
||||
raise Unauthorized('Требуется идентификация пользователя',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user_id = self.store.get_session(security.hash_token(token))
|
||||
if user_id is None:
|
||||
raise Unauthorized('Сессия недействительна или истекла',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
user = self.store.user(user_id)
|
||||
if user is None:
|
||||
raise Unauthorized('Пользователь не найден',
|
||||
www_authenticate='Bearer realm="messenger"')
|
||||
return user
|
||||
|
||||
def _bearer_token(self, request):
|
||||
authorization = request.headers.get('Authorization') or ''
|
||||
if not authorization.lower().startswith('bearer '):
|
||||
return None
|
||||
return authorization[len('bearer '):].strip()
|
||||
|
||||
def _chat_member_ids(self, chat_id):
|
||||
with self.database.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
return [str(row['user_id']) for row in cur.fetchall()]
|
||||
|
||||
def _member_chat(self, actor, raw_chat_id):
|
||||
"""Foreign and missing chats are indistinguishable: both are 404."""
|
||||
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
|
||||
chat = self.store.chat(chat_id)
|
||||
if chat is None or not self.store.is_member(chat, actor['id']):
|
||||
raise NotFound('Чат не найден')
|
||||
return chat
|
||||
|
||||
def _json_object(self, request):
|
||||
content_type = request.headers.get('Content-Type') or ''
|
||||
if content_type.split(';')[0].strip().lower() != 'application/json':
|
||||
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
|
||||
if not request.body:
|
||||
raise InvalidRequest('Требуется JSON-тело запроса')
|
||||
try:
|
||||
payload = json.loads(request.body.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
raise InvalidRequest('Тело запроса не является корректным JSON') from None
|
||||
return validation.require_object(payload)
|
||||
|
||||
def _pagination(self, request, scope):
|
||||
raw_limit = (request.query.get('limit') or [None])[0]
|
||||
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
|
||||
self.config.max_page_limit)
|
||||
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
|
||||
if raw_cursor is None:
|
||||
return limit, None
|
||||
try:
|
||||
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
|
||||
except cursor_codec.InvalidCursor:
|
||||
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
|
||||
return limit, (stamp, identifier)
|
||||
|
||||
def _next_cursor(self, scope, items, has_more):
|
||||
if not has_more or not items:
|
||||
return None
|
||||
last = items[-1]
|
||||
return cursor_codec.encode(scope, last['created_at'], last['id'],
|
||||
self.config.cursor_secret)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Bounded image validation and private S3 storage."""
|
||||
from contextlib import contextmanager
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
from functools import cached_property
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image
|
||||
|
||||
from .errors import InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
||||
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_IMAGE_PIXELS = 40_000_000
|
||||
CHUNK_SIZE = 64 * 1024
|
||||
FORMATS = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||
# Pillow can allocate a full decoded image. Allow only one decode per API process.
|
||||
_decode_slot = threading.Lock()
|
||||
_upload_slots = threading.BoundedSemaphore(10)
|
||||
|
||||
|
||||
def filename_from(query):
|
||||
names = query.get('filename', [])
|
||||
if len(names) != 1:
|
||||
raise InvalidRequest('Укажите одно имя файла')
|
||||
name = names[0]
|
||||
if (not 1 <= len(name) <= 255 or not name.strip()
|
||||
or any(c in '/\\' or unicodedata.category(c) in {'Cc', 'Cs'} for c in name)):
|
||||
raise InvalidRequest('Некорректное имя файла')
|
||||
return name
|
||||
|
||||
|
||||
def validate_image(file, declared_type):
|
||||
try:
|
||||
with _decode_slot:
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=list(FORMATS)) as image:
|
||||
if image.width * image.height > MAX_IMAGE_PIXELS:
|
||||
raise PayloadTooLarge('Изображение превышает 40 млн пикселей')
|
||||
if FORMATS[image.format] != declared_type:
|
||||
raise UnsupportedMediaType('Тип изображения не совпадает с Content-Type')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=list(FORMATS)) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise PayloadTooLarge('Изображение превышает 40 млн пикселей') from None
|
||||
except (OSError, SyntaxError, ValueError, OverflowError, EOFError, struct.error):
|
||||
raise UnsupportedMediaType('Повреждённое или неподдерживаемое изображение') from None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def upload_file(read_chunks, content_type):
|
||||
if content_type not in FORMATS.values():
|
||||
raise UnsupportedMediaType('Ожидается PNG или JPEG')
|
||||
if not _upload_slots.acquire(blocking=False):
|
||||
raise Unavailable('Слишком много одновременных загрузок')
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
for chunk in read_chunks():
|
||||
size += len(chunk)
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLarge()
|
||||
digest.update(chunk)
|
||||
file.write(chunk)
|
||||
if size == 0:
|
||||
raise UnsupportedMediaType('Пустое изображение')
|
||||
validate_image(file, content_type)
|
||||
file.seek(0)
|
||||
yield file, size, digest.hexdigest()
|
||||
finally:
|
||||
_upload_slots.release()
|
||||
|
||||
|
||||
class BinaryResponse:
|
||||
def __init__(self, stream, content_type, size):
|
||||
self.stream = stream
|
||||
self.content_type = content_type
|
||||
self.size = size
|
||||
|
||||
def chunks(self):
|
||||
try:
|
||||
yield from self.stream.iter_chunks(CHUNK_SIZE)
|
||||
except (BotoCoreError, OSError) as exc:
|
||||
raise OSError('S3 response interrupted') from exc
|
||||
|
||||
def close(self):
|
||||
self.stream.close()
|
||||
|
||||
|
||||
class AttachmentStorage:
|
||||
@cached_property
|
||||
def client(self):
|
||||
def secret(name):
|
||||
with open(os.environ[name + '_FILE'], encoding='utf-8') as file:
|
||||
return file.read().strip()
|
||||
return boto3.session.Session().client(
|
||||
's3', endpoint_url=os.environ['S3_ENDPOINT'],
|
||||
region_name=os.environ.get('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
|
||||
request_checksum_calculation="when_required",
|
||||
s3={"addressing_style": "path"}))
|
||||
|
||||
@property
|
||||
def bucket(self):
|
||||
return os.environ['S3_BUCKET']
|
||||
|
||||
def put(self, key, file, size, content_type):
|
||||
try:
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=file,
|
||||
ContentLength=size, ContentType=content_type)
|
||||
except (BotoCoreError, ClientError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
|
||||
def delete(self, key):
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
|
||||
def download(self, row):
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
|
||||
except ClientError as exc:
|
||||
if exc.response.get('Error', {}).get('Code') in {'NoSuchKey', '404'}:
|
||||
raise NotFound('Оригинал вложения не найден') from None
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
except (BotoCoreError, OSError, KeyError):
|
||||
raise Unavailable('Хранилище вложений недоступно') from None
|
||||
if obj['ContentLength'] != row['size_bytes']:
|
||||
obj['Body'].close()
|
||||
raise Unavailable('Размер оригинала не совпадает с метаданными')
|
||||
return BinaryResponse(obj['Body'], row['content_type'], row['size_bytes'])
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Redis session cache with TTL and graceful degradation.
|
||||
|
||||
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
|
||||
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
|
||||
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
|
||||
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
|
||||
anonymous request as trusted
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import redis
|
||||
|
||||
from .config import Config
|
||||
|
||||
logger = logging.getLogger('messenger.cache')
|
||||
|
||||
|
||||
class SessionCache:
|
||||
"""Look-aside cache for session tokens in Redis/Valkey."""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
def open(self):
|
||||
if not self._config.cache_url:
|
||||
self._available = False
|
||||
return
|
||||
try:
|
||||
self._client = redis.Redis.from_url(
|
||||
self._config.cache_url,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
socket_keepalive=True,
|
||||
health_check_interval=30,
|
||||
decode_responses=True,
|
||||
)
|
||||
self._client.ping()
|
||||
self._available = True
|
||||
logger.info('cache_connected', extra={'event': 'cache_connected'})
|
||||
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
|
||||
self._available = False
|
||||
self._client = None
|
||||
logger.warning('cache_unavailable', extra={
|
||||
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
|
||||
|
||||
def close(self):
|
||||
if self._client is not None:
|
||||
try:
|
||||
self._client.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._client = None
|
||||
self._available = False
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
return self._available and self._client is not None
|
||||
|
||||
def get(self, token_hash):
|
||||
"""Return cached user_id or None (miss or unavailable)."""
|
||||
if not self.available:
|
||||
return None
|
||||
try:
|
||||
value = self._client.get('session:%s' % token_hash)
|
||||
return value
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_read_failed', extra={
|
||||
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
|
||||
self._available = False
|
||||
return None
|
||||
|
||||
def set(self, token_hash, user_id, ttl_seconds):
|
||||
"""Cache a session with TTL. Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_write_failed', extra={
|
||||
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
|
||||
self._available = False
|
||||
|
||||
def delete(self, token_hash):
|
||||
"""Remove a session from the cache (logout). Failure is non-fatal."""
|
||||
if not self.available:
|
||||
return
|
||||
try:
|
||||
self._client.delete('session:%s' % token_hash)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning('cache_delete_failed', extra={
|
||||
'event': 'cache_delete_failed'})
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Remove expired upload reservations and their S3 objects."""
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
from .attachments import AttachmentStorage
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .store import Store
|
||||
|
||||
GRACE_SECONDS = 15 * 60
|
||||
INTERVAL_SECONDS = 60
|
||||
BATCH_SIZE = 25
|
||||
|
||||
|
||||
def clean_batch(store, storage):
|
||||
count = 0
|
||||
while count < BATCH_SIZE and store.clean_pending_attachment(GRACE_SECONDS, storage.delete):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--once', action='store_true', help='Process one bounded batch and exit')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
config = Config.from_env()
|
||||
database = Database(config)
|
||||
store = Store(config, database, None)
|
||||
storage = AttachmentStorage()
|
||||
try:
|
||||
database.open()
|
||||
while True:
|
||||
try:
|
||||
deleted = clean_batch(store, storage)
|
||||
if deleted or args.once:
|
||||
logging.info('Expired attachment reservations removed: %s', deleted)
|
||||
except Unavailable:
|
||||
logging.warning('Attachment cleanup deferred because a dependency is unavailable')
|
||||
if args.once:
|
||||
return 1
|
||||
if args.once:
|
||||
return 0
|
||||
time.sleep(INTERVAL_SECONDS)
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Runtime configuration. Every value is read from the environment."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
|
||||
# Contract limits: they cannot be raised by configuration.
|
||||
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
|
||||
CONTRACT_MAX_PAGE_LIMIT = 100
|
||||
CONTRACT_SESSION_TTL_MIN = 1
|
||||
CONTRACT_SESSION_TTL_MAX = 86400
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when an environment value cannot be used."""
|
||||
|
||||
|
||||
def _str(environ, name, default):
|
||||
value = environ.get(name)
|
||||
if value is None or value == '':
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _int(environ, name, default, minimum, maximum):
|
||||
raw = environ.get(name)
|
||||
if raw is None or raw == '':
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
|
||||
if not minimum <= value <= maximum:
|
||||
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
|
||||
return value
|
||||
|
||||
|
||||
def _log_level(environ, name, default):
|
||||
value = _str(environ, name, default).upper()
|
||||
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
|
||||
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
|
||||
return value
|
||||
|
||||
|
||||
def _secret_from_file(environ, name, default=None):
|
||||
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
|
||||
|
||||
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
|
||||
файл (например, Docker secret), а приложение читает его содержимое.
|
||||
Если файл не задан или не существует, используется значение переменной
|
||||
окружения без суффикса _FILE (удобно для локального запуска без секретов).
|
||||
"""
|
||||
path_name = name + '_FILE'
|
||||
path = environ.get(path_name)
|
||||
if path:
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as handle:
|
||||
return handle.read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
return environ.get(name) or default
|
||||
|
||||
|
||||
class Config:
|
||||
"""Effective configuration of one process instance."""
|
||||
|
||||
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
|
||||
max_text_code_points, max_display_name_code_points, max_title_code_points,
|
||||
default_page_limit, max_page_limit, max_body_bytes,
|
||||
database_url, cache_url, session_ttl_seconds, session_store,
|
||||
pool_min, pool_max, connect_timeout, statement_timeout):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.env = env
|
||||
self.log_level = log_level
|
||||
self.cursor_secret = cursor_secret
|
||||
self.shutdown_grace_seconds = shutdown_grace_seconds
|
||||
self.max_text_code_points = max_text_code_points
|
||||
self.max_display_name_code_points = max_display_name_code_points
|
||||
self.max_title_code_points = max_title_code_points
|
||||
self.default_page_limit = default_page_limit
|
||||
self.max_page_limit = max_page_limit
|
||||
self.max_body_bytes = max_body_bytes
|
||||
self.database_url = database_url
|
||||
self.cache_url = cache_url
|
||||
self.session_ttl_seconds = session_ttl_seconds
|
||||
self.session_store = session_store
|
||||
self.pool_min = pool_min
|
||||
self.pool_max = pool_max
|
||||
self.connect_timeout = connect_timeout
|
||||
self.statement_timeout = statement_timeout
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environ=None):
|
||||
environ = os.environ if environ is None else environ
|
||||
secret = _secret_from_file(environ, 'CURSOR_SECRET', '')
|
||||
if not secret:
|
||||
secret = secrets.token_hex(32)
|
||||
session_store = _str(environ, 'SESSION_STORE', 'postgres')
|
||||
if session_store not in ('postgres', 'redis'):
|
||||
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
|
||||
return cls(
|
||||
host=_str(environ, 'APP_HOST', '0.0.0.0'),
|
||||
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
|
||||
env=_str(environ, 'APP_ENV', 'lab'),
|
||||
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
|
||||
cursor_secret=secret.encode('utf-8'),
|
||||
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
|
||||
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
|
||||
1, CONTRACT_MAX_TEXT_CODE_POINTS),
|
||||
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
|
||||
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
|
||||
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
|
||||
CONTRACT_MAX_PAGE_LIMIT),
|
||||
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
|
||||
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
|
||||
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
|
||||
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
|
||||
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
|
||||
session_store=session_store,
|
||||
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
|
||||
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
|
||||
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
|
||||
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
|
||||
)
|
||||
|
||||
def as_log_fields(self):
|
||||
"""Configuration summary safe for logs (no secrets)."""
|
||||
return {
|
||||
'app_env': self.env,
|
||||
'host': self.host,
|
||||
'port': self.port,
|
||||
'log_level': self.log_level,
|
||||
'shutdown_grace_seconds': self.shutdown_grace_seconds,
|
||||
'default_page_limit': self.default_page_limit,
|
||||
'max_page_limit': self.max_page_limit,
|
||||
'max_text_code_points': self.max_text_code_points,
|
||||
'max_body_bytes': self.max_body_bytes,
|
||||
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
|
||||
'database_url_configured': bool(self.database_url),
|
||||
'cache_url_configured': bool(self.cache_url),
|
||||
'session_ttl_seconds': self.session_ttl_seconds,
|
||||
'session_store': self.session_store,
|
||||
'db_pool_min': self.pool_min,
|
||||
'db_pool_max': self.pool_max,
|
||||
'db_connect_timeout': self.connect_timeout,
|
||||
'db_statement_timeout': self.statement_timeout,
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import uuid
|
||||
|
||||
VERSION = 1
|
||||
MAC_BYTES = 16
|
||||
|
||||
|
||||
class InvalidCursor(ValueError):
|
||||
"""Cursor is malformed, forged, expired or belongs to another resource."""
|
||||
|
||||
|
||||
def _key(secret):
|
||||
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
|
||||
|
||||
|
||||
def encode(scope, created_at, object_id, secret):
|
||||
"""Serialize a keyset position bound to one resource scope."""
|
||||
payload = json.dumps(
|
||||
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
|
||||
separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
|
||||
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode(token, scope, secret):
|
||||
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
|
||||
if not isinstance(token, str) or not token:
|
||||
raise InvalidCursor('empty cursor')
|
||||
padding = '=' * (-len(token) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(token + padding)
|
||||
except (binascii.Error, ValueError):
|
||||
raise InvalidCursor('cursor is not base64url') from None
|
||||
if len(raw) <= MAC_BYTES:
|
||||
raise InvalidCursor('cursor is too short')
|
||||
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
|
||||
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
|
||||
raise InvalidCursor('cursor signature mismatch')
|
||||
try:
|
||||
data = json.loads(payload.decode('utf-8'))
|
||||
except (ValueError, UnicodeError):
|
||||
raise InvalidCursor('cursor payload is not JSON') from None
|
||||
if not isinstance(data, dict) or data.get('v') != VERSION:
|
||||
raise InvalidCursor('unsupported cursor version')
|
||||
if data.get('scope') != scope:
|
||||
raise InvalidCursor('cursor belongs to another resource')
|
||||
created_at = data.get('created_at')
|
||||
object_id = data.get('id')
|
||||
if not isinstance(created_at, str) or not isinstance(object_id, str):
|
||||
raise InvalidCursor('cursor payload types')
|
||||
try:
|
||||
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
|
||||
identifier = uuid.UUID(object_id)
|
||||
except ValueError:
|
||||
raise InvalidCursor('cursor payload values') from None
|
||||
if stamp.tzinfo is None:
|
||||
raise InvalidCursor('cursor timestamp has no timezone')
|
||||
return stamp.astimezone(dt.timezone.utc), identifier
|
||||
@@ -0,0 +1,97 @@
|
||||
"""PostgreSQL connection pool and schema migrations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg2 import pool, OperationalError, InterfaceError
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
from .config import Config
|
||||
from .errors import Unavailable
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MIGRATIONS_DIR = _ROOT / 'migrations'
|
||||
|
||||
|
||||
class Database:
|
||||
"""Thread-safe connection pool with prepared statement timeouts.
|
||||
|
||||
All queries are parameterised (%s). Connections are returned to the pool
|
||||
after every operation; the pool itself is closed on shutdown.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Config):
|
||||
self._config = config
|
||||
self._pool = None
|
||||
|
||||
def open(self):
|
||||
if self._pool is not None:
|
||||
return
|
||||
try:
|
||||
self._pool = pool.ThreadedConnectionPool(
|
||||
minconn=self._config.pool_min,
|
||||
maxconn=self._config.pool_max,
|
||||
dsn=self._config.database_url,
|
||||
connect_timeout=self._config.connect_timeout,
|
||||
cursor_factory=RealDictCursor,
|
||||
)
|
||||
except Exception as exc: # psycopg2.Error or configuration error
|
||||
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
|
||||
|
||||
def close(self):
|
||||
if self._pool is not None:
|
||||
self._pool.closeall()
|
||||
self._pool = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
"""Yield a connection; rollback on exception, return to pool on exit."""
|
||||
if self._pool is None:
|
||||
raise Unavailable('База данных не инициализирована')
|
||||
try:
|
||||
conn = self._pool.getconn()
|
||||
except (pool.PoolError, OperationalError, InterfaceError):
|
||||
raise Unavailable('Пул соединений PostgreSQL недоступен') from None
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute('SET statement_timeout = %s',
|
||||
(self._config.statement_timeout * 1000,))
|
||||
yield conn
|
||||
except Exception as exc:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(exc, (OperationalError, InterfaceError)):
|
||||
raise Unavailable('PostgreSQL недоступен') from None
|
||||
raise
|
||||
finally:
|
||||
self._pool.putconn(conn, close=bool(conn.closed))
|
||||
|
||||
@contextmanager
|
||||
def cursor(self):
|
||||
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
|
||||
with self.connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
yield cur
|
||||
conn.commit()
|
||||
|
||||
def ping(self):
|
||||
with self.cursor() as cur:
|
||||
cur.execute('SELECT 1')
|
||||
cur.fetchone()
|
||||
|
||||
def run_migrations(self):
|
||||
"""Apply versioned migrations once, under a transaction advisory lock."""
|
||||
with self.cursor() as cur:
|
||||
cur.execute('SELECT pg_advisory_xact_lock(3072026)')
|
||||
cur.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(_MIGRATIONS_DIR.glob('*.sql')):
|
||||
cur.execute('SELECT 1 FROM schema_migrations WHERE name = %s', (path.name,))
|
||||
if cur.fetchone() is not None:
|
||||
continue
|
||||
cur.execute(path.read_text(encoding='utf-8'))
|
||||
cur.execute('INSERT INTO schema_migrations(name) VALUES (%s)', (path.name,))
|
||||
@@ -0,0 +1,73 @@
|
||||
"""API error taxonomy; every message is safe to return to a client."""
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""Base class for errors that map onto the public error payload."""
|
||||
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
|
||||
def __init__(self, message=None):
|
||||
self.message = message or self.default_message
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
status = 400
|
||||
code = 'invalid_request'
|
||||
default_message = 'Некорректный запрос'
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
status = 401
|
||||
code = 'unauthorized'
|
||||
default_message = 'Требуется идентификация пользователя'
|
||||
www_authenticate = None
|
||||
|
||||
def __init__(self, message=None, www_authenticate=None):
|
||||
self.message = message or self.default_message
|
||||
self.www_authenticate = www_authenticate
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
status = 404
|
||||
code = 'not_found'
|
||||
default_message = 'Объект не найден'
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
status = 409
|
||||
code = 'conflict'
|
||||
default_message = 'Конфликт состояния'
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
status = 413
|
||||
code = 'payload_too_large'
|
||||
default_message = 'Тело запроса слишком велико'
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
status = 415
|
||||
code = 'unsupported_media_type'
|
||||
default_message = 'Поддерживается только application/json'
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
status = 429
|
||||
code = 'rate_limited'
|
||||
default_message = 'Слишком много запросов'
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
status = 503
|
||||
code = 'unavailable'
|
||||
default_message = 'Сервис временно недоступен'
|
||||
|
||||
|
||||
class InternalError(ApiError):
|
||||
status = 500
|
||||
code = 'internal_error'
|
||||
default_message = 'Внутренняя ошибка сервера'
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Structured stdout logging without payload contents."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
|
||||
'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line; message text of API payloads is never logged."""
|
||||
|
||||
def format(self, record):
|
||||
payload = {
|
||||
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
|
||||
'.%03dZ' % int(record.msecs),
|
||||
'level': record.levelname.lower(),
|
||||
'logger': record.name,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
for field in _FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None:
|
||||
payload[field] = value
|
||||
if record.exc_info and record.levelno >= logging.ERROR:
|
||||
payload['exception'] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def configure(level_name):
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
for existing in list(root.handlers):
|
||||
root.removeHandler(existing)
|
||||
root.addHandler(handler)
|
||||
root.setLevel(getattr(logging, level_name))
|
||||
return logging.getLogger('messenger')
|
||||
|
||||
|
||||
def access_logger():
|
||||
return logging.getLogger('messenger.access')
|
||||
@@ -0,0 +1,17 @@
|
||||
"""One-shot schema migration entry point; API processes never migrate."""
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
|
||||
|
||||
def main():
|
||||
database = Database(Config.from_env())
|
||||
try:
|
||||
database.open()
|
||||
database.run_migrations()
|
||||
print('PASS: migrations applied')
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Password hashing and session tokens.
|
||||
|
||||
Passwords are hashed with a library password KDF: scrypt from the standard
|
||||
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
|
||||
passwords longer than bcrypt's 72-byte limit are still supported.
|
||||
Session tokens are opaque random values (>= 256 bits of entropy); only their
|
||||
SHA-256 is stored in the database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
import bcrypt
|
||||
|
||||
try: # scrypt is part of hashlib on OpenSSL-enabled builds
|
||||
hashlib.scrypt(b'x', salt=b's', n=16, r=8, p=1, dklen=16)
|
||||
_HAS_SCRYPT = True
|
||||
except Exception:
|
||||
_HAS_SCRYPT = False
|
||||
|
||||
if not _HAS_SCRYPT:
|
||||
import bcrypt # type: ignore
|
||||
|
||||
SCRYPT_N = 16384
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
SCRYPT_DKLEN = 32
|
||||
SALT_BYTES = 16
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Hash a UTF-8 password with a library password KDF.
|
||||
|
||||
Returns a single storable string:
|
||||
scrypt:$n:$r:$p:$salt_b64:$hash_b64
|
||||
or
|
||||
bcrypt:$salt_b64:$hash_b64
|
||||
"""
|
||||
raw = password.encode('utf-8')
|
||||
if _HAS_SCRYPT:
|
||||
salt = secrets.token_bytes(SALT_BYTES)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=SCRYPT_N,
|
||||
r=SCRYPT_R,
|
||||
p=SCRYPT_P,
|
||||
dklen=SCRYPT_DKLEN,
|
||||
)
|
||||
return 'scrypt:%d:%d:%d:%s:%s' % (
|
||||
SCRYPT_N, SCRYPT_R, SCRYPT_P,
|
||||
base64.b64encode(salt).decode('ascii'),
|
||||
base64.b64encode(dk).decode('ascii'),
|
||||
)
|
||||
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
|
||||
# (up to 128 code points per contract) are fully covered.
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
|
||||
return 'bcrypt:%s' % hashed.decode('ascii')
|
||||
|
||||
|
||||
def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Verify a password against a KDF hash string. Constant-time compare."""
|
||||
try:
|
||||
raw = password.encode('utf-8')
|
||||
if password_hash.startswith('scrypt:'):
|
||||
parts = password_hash.split(':')
|
||||
if len(parts) != 6 or parts[0] != 'scrypt':
|
||||
return False
|
||||
_, n, r, p, salt_b64, hash_b64 = parts
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(hash_b64)
|
||||
dk = hashlib.scrypt(
|
||||
raw,
|
||||
salt=salt,
|
||||
n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(expected),
|
||||
)
|
||||
return hmac.compare_digest(dk, expected)
|
||||
if password_hash.startswith('bcrypt:'):
|
||||
stored = password_hash[len('bcrypt:'):].encode('ascii')
|
||||
digest = hashlib.sha256(raw).digest()
|
||||
return bcrypt.checkpw(digest, stored)
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Opaque random bearer token, at least 256 bits of entropy."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
|
||||
return hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
@@ -0,0 +1,305 @@
|
||||
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
|
||||
from __future__ import annotations
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import __version__, logging_utils
|
||||
from .api import Api, Request
|
||||
from .attachments import BinaryResponse, CHUNK_SIZE, MAX_UPLOAD_BYTES
|
||||
from .cache import SessionCache
|
||||
from .config import Config, ConfigError
|
||||
from .db import Database
|
||||
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
|
||||
from .store import Store
|
||||
|
||||
MAX_REQUEST_ID_LENGTH = 128
|
||||
MAX_LOGGED_LINE = 200
|
||||
|
||||
|
||||
def sanitize_request_id(raw):
|
||||
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
|
||||
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
|
||||
return raw
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def route_template(path):
|
||||
for _method, pattern, _handler in Api.ROUTES:
|
||||
if pattern.match(path):
|
||||
return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$')
|
||||
return 'unmatched'
|
||||
|
||||
|
||||
def encode_payload(payload):
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
|
||||
|
||||
class ApiServer(ThreadingHTTPServer):
|
||||
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address, api, config, logger, database):
|
||||
self.api = api
|
||||
self.config = config
|
||||
self.logger = logger
|
||||
self.database = database
|
||||
self.instance_id = os.environ.get("HOSTNAME") or uuid.uuid4().hex
|
||||
self.shutting_down = False
|
||||
self._active = 0
|
||||
self._idle = threading.Condition()
|
||||
super().__init__(address, ApiHandler)
|
||||
|
||||
def enter(self):
|
||||
with self._idle:
|
||||
self._active += 1
|
||||
|
||||
def leave(self):
|
||||
with self._idle:
|
||||
self._active -= 1
|
||||
if self._active == 0:
|
||||
self._idle.notify_all()
|
||||
|
||||
def wait_idle(self, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
with self._idle:
|
||||
while self._active > 0:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
self._idle.wait(remaining)
|
||||
return True
|
||||
|
||||
|
||||
class ApiHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
server_version = 'mtusi-messenger/' + __version__
|
||||
sys_version = ''
|
||||
timeout = 30
|
||||
|
||||
# --------------------------------------------------------- entry points
|
||||
def do_GET(self):
|
||||
self._handle()
|
||||
|
||||
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
"""Base-class chatter stays debug-only; access logs are structured."""
|
||||
self.server.logger.debug('http_note', extra={
|
||||
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
|
||||
|
||||
# ------------------------------------------------------------- pipeline
|
||||
def _handle(self):
|
||||
started = time.monotonic()
|
||||
server = self.server
|
||||
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
|
||||
head_only = self.command == 'HEAD'
|
||||
path = urlsplit(self.path).path
|
||||
status, error_code = 500, None
|
||||
self._last_exc = None
|
||||
server.enter()
|
||||
try:
|
||||
if server.shutting_down:
|
||||
raise Unavailable('Сервер завершает работу')
|
||||
if self.command == 'POST' and Api.UPLOAD_PATH.fullmatch(path):
|
||||
# Rejected uploads may leave unread bytes in the connection.
|
||||
self.close_connection = True
|
||||
body = self._upload_chunks
|
||||
else:
|
||||
body = self._read_body()
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
request = Request('GET' if head_only else self.command, path, query, self.headers,
|
||||
body, request_id, self.client_address[0])
|
||||
status, payload = server.api.dispatch(request)
|
||||
encoded = payload if isinstance(payload, BinaryResponse) else encode_payload(payload)
|
||||
except ApiError as exc:
|
||||
self._last_exc = exc
|
||||
status, error_code = exc.status, exc.code
|
||||
encoded = self._error_payload(exc, request_id)
|
||||
except Exception: # last resort: log privately, answer generically
|
||||
server.logger.exception('unhandled_error', extra={
|
||||
'event': 'unhandled_error', 'request_id': request_id,
|
||||
'method': self.command, 'route': route_template(path)})
|
||||
status, error_code = 500, 'internal_error'
|
||||
encoded = self._error_payload(ApiError(), request_id)
|
||||
finally:
|
||||
server.logger.info('http_request', extra={
|
||||
'event': 'http_request', 'request_id': request_id, 'method': self.command,
|
||||
'route': route_template(path), 'status': status,
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3),
|
||||
'error_code': error_code, 'client': self.client_address[0]})
|
||||
try:
|
||||
self._send(status, encoded, request_id, head_only)
|
||||
finally:
|
||||
if isinstance(encoded, BinaryResponse):
|
||||
encoded.close()
|
||||
server.leave()
|
||||
|
||||
def _read_body(self):
|
||||
if self.headers.get('Transfer-Encoding'):
|
||||
self.close_connection = True
|
||||
raise InvalidRequest('Transfer-Encoding не поддерживается')
|
||||
raw_length = self.headers.get('Content-Length')
|
||||
if raw_length is None:
|
||||
return b''
|
||||
try:
|
||||
length = int(raw_length.strip())
|
||||
except ValueError:
|
||||
raise InvalidRequest('Некорректный Content-Length') from None
|
||||
if length < 0:
|
||||
raise InvalidRequest('Некорректный Content-Length')
|
||||
if length > self.server.config.max_body_bytes:
|
||||
self.close_connection = True
|
||||
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
|
||||
if length == 0:
|
||||
return b''
|
||||
return self.rfile.read(length)
|
||||
|
||||
def _upload_chunks(self):
|
||||
if self.headers.get('Transfer-Encoding'):
|
||||
raise InvalidRequest('Требуется Content-Length')
|
||||
lengths = self.headers.get_all('Content-Length') or []
|
||||
try:
|
||||
length = int(lengths[0]) if len(lengths) == 1 else -1
|
||||
except ValueError:
|
||||
length = -1
|
||||
if length < 0:
|
||||
raise InvalidRequest('Некорректный Content-Length')
|
||||
if length > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLarge()
|
||||
deadline = time.monotonic() + 30
|
||||
while length:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise InvalidRequest('Истекло время загрузки')
|
||||
self.connection.settimeout(min(5, remaining))
|
||||
try:
|
||||
chunk = self.rfile.read1(min(CHUNK_SIZE, length))
|
||||
except OSError:
|
||||
raise InvalidRequest('Загрузка прервана') from None
|
||||
finally:
|
||||
self.connection.settimeout(self.timeout)
|
||||
if not chunk:
|
||||
raise InvalidRequest('Загрузка прервана')
|
||||
length -= len(chunk)
|
||||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
def _error_payload(exc, request_id):
|
||||
return encode_payload({'error': {
|
||||
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
|
||||
|
||||
def _send(self, status, encoded, request_id, head_only):
|
||||
try:
|
||||
self.send_response(status)
|
||||
binary = isinstance(encoded, BinaryResponse)
|
||||
self.send_header('Content-Type', encoded.content_type if binary else 'application/json; charset=utf-8')
|
||||
self.send_header('X-Request-Id', request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
if status == 204:
|
||||
self.send_header('Content-Length', '0')
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
self.end_headers()
|
||||
return
|
||||
self.send_header('Content-Length', str(encoded.size if binary else len(encoded)))
|
||||
if status == 503:
|
||||
self.send_header('Retry-After', '1')
|
||||
if status == 401 and self._last_exc is not None:
|
||||
www_auth = getattr(self._last_exc, 'www_authenticate', None)
|
||||
if www_auth:
|
||||
self.send_header('WWW-Authenticate', www_auth)
|
||||
if self.close_connection:
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
if not head_only and encoded:
|
||||
if binary:
|
||||
for chunk in encoded.chunks():
|
||||
self.wfile.write(chunk)
|
||||
else:
|
||||
self.wfile.write(encoded)
|
||||
except OSError:
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def build_server(config, logger, api, database):
|
||||
"""Create the listening server; raises OSError when the port is taken."""
|
||||
return ApiServer((config.host, config.port), api, config, logger, database)
|
||||
|
||||
|
||||
def run(config, logger, database, cache):
|
||||
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
|
||||
api = Api(config, Store(config, database, cache), database)
|
||||
try:
|
||||
server = build_server(config, logger, api, database)
|
||||
except OSError as exc:
|
||||
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
|
||||
return 1
|
||||
stop = threading.Event()
|
||||
|
||||
def handle_signal(signum, _frame):
|
||||
if stop.is_set():
|
||||
return
|
||||
stop.set()
|
||||
server.shutting_down = True
|
||||
logger.warning('shutdown_signal', extra={
|
||||
'event': 'shutdown_signal', 'signal': signum,
|
||||
'detail': 'stop accepting new requests'})
|
||||
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
|
||||
|
||||
for name in ('SIGTERM', 'SIGINT'):
|
||||
if hasattr(signal, name):
|
||||
signal.signal(getattr(signal, name), handle_signal)
|
||||
|
||||
logger.info('server_started', extra=dict(
|
||||
{'event': 'server_started', 'version': __version__,
|
||||
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
|
||||
**config.as_log_fields()))
|
||||
started = time.monotonic()
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
finally:
|
||||
server.server_close()
|
||||
drained = server.wait_idle(config.shutdown_grace_seconds)
|
||||
database.close()
|
||||
logger.info('server_stopped', extra={
|
||||
'event': 'server_stopped',
|
||||
'outcome': 'graceful' if drained else 'grace_period_exceeded',
|
||||
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
config = Config.from_env()
|
||||
except ConfigError as exc:
|
||||
print('configuration error: %s' % exc, file=sys.stderr)
|
||||
return 2
|
||||
logger = logging_utils.configure(config.log_level)
|
||||
database = Database(config)
|
||||
cache = SessionCache(config)
|
||||
try:
|
||||
database.open()
|
||||
cache.open()
|
||||
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
|
||||
logger.error('database_start_failed', extra={
|
||||
'event': 'database_start_failed', 'detail': str(exc)})
|
||||
database.close()
|
||||
cache.close()
|
||||
return 1
|
||||
try:
|
||||
return run(config, logger, database, cache)
|
||||
finally:
|
||||
cache.close()
|
||||
@@ -0,0 +1,377 @@
|
||||
"""PostgreSQL-backed state store. All writes go through parameterised queries
|
||||
inside explicit transactions; reads use keyset pagination over indexed columns."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from psycopg2.extras import Json
|
||||
from psycopg2.errors import UniqueViolation, ForeignKeyViolation
|
||||
import uuid
|
||||
|
||||
from . import cursor as cursor_codec
|
||||
from . import security
|
||||
from .cache import SessionCache
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .errors import Conflict, InvalidRequest, NotFound
|
||||
|
||||
|
||||
def utc_now():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def rfc3339(stamp):
|
||||
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
|
||||
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL state: users, chats, membership, messages, sessions."""
|
||||
|
||||
def __init__(self, config: Config, database: Database, cache: SessionCache):
|
||||
self._config = config
|
||||
self._db = database
|
||||
self._cache = cache
|
||||
|
||||
# ---------------------------------------------------------------- reads
|
||||
def stats(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
|
||||
' (SELECT count(*) FROM chats) AS chats,'
|
||||
' (SELECT count(*) FROM messages) AS messages')
|
||||
row = cur.fetchone()
|
||||
return {
|
||||
'users': row['users'],
|
||||
'chats': row['chats'],
|
||||
'messages': row['messages'],
|
||||
}
|
||||
|
||||
def user(self, user_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
|
||||
(str(user_id),))
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row) if row else None
|
||||
|
||||
def user_by_username(self, username):
|
||||
"""Return user row including password_hash, or None."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, username, password_hash, display_name, created_at FROM users '
|
||||
'WHERE username = %s', (username,))
|
||||
return cur.fetchone()
|
||||
|
||||
def users_exist(self, identifiers):
|
||||
if not identifiers:
|
||||
return True
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
|
||||
([str(i) for i in identifiers],))
|
||||
row = cur.fetchone()
|
||||
return row['c'] == len(identifiers)
|
||||
|
||||
# --------------------------------------------------------------- writes
|
||||
def create_user(self, username, display_name, password_hash):
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, username, display_name, created_at',
|
||||
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
|
||||
except UniqueViolation:
|
||||
# Unique violation on username -> conflict, rolled back by cursor context.
|
||||
raise Conflict('username уже занят') from None
|
||||
row = cur.fetchone()
|
||||
return self._public_user(row)
|
||||
|
||||
def create_chat(self, title, member_ids):
|
||||
"""Atomically create a chat and its membership. Rollback on any failure."""
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
|
||||
(identifier, title, stamp))
|
||||
for member_id in member_ids:
|
||||
cur.execute(
|
||||
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
|
||||
(identifier, str(member_id)))
|
||||
except ForeignKeyViolation:
|
||||
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
|
||||
return {
|
||||
'id': identifier,
|
||||
'title': title,
|
||||
'member_ids': list(member_ids),
|
||||
'created_at': rfc3339(stamp),
|
||||
}
|
||||
|
||||
def reserve_attachment(self, aid, chat_id, owner_id, filename, mime, size, digest, key):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, "
|
||||
"size_bytes, sha256, object_key, status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')",
|
||||
(aid, chat_id, owner_id, filename, mime, size, digest, key))
|
||||
|
||||
def finish_attachment(self, aid, put_object):
|
||||
"""Keep a row lock from before PUT until the ready transaction commits."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute("SELECT object_key FROM attachments WHERE id=%s "
|
||||
"AND status='pending' FOR UPDATE", (aid,))
|
||||
pending = cur.fetchone()
|
||||
if pending is None:
|
||||
raise Conflict('Загрузка вложения не может быть завершена')
|
||||
put_object(pending['object_key'])
|
||||
cur.execute("UPDATE attachments SET status='ready', updated_at=now() "
|
||||
"WHERE id=%s RETURNING *", (aid,))
|
||||
return cur.fetchone()
|
||||
|
||||
def clean_pending_attachment(self, grace_seconds, delete_object):
|
||||
"""Remove one expired reservation and its object, or leave it for retry."""
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status='pending' "
|
||||
"AND updated_at < now() - make_interval(secs => %s) "
|
||||
"ORDER BY updated_at, id LIMIT 1 FOR UPDATE SKIP LOCKED",
|
||||
(grace_seconds,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
delete_object(row['object_key'])
|
||||
cur.execute('DELETE FROM attachments WHERE id=%s', (row['id'],))
|
||||
return True
|
||||
|
||||
def visible_attachment(self, aid, user_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s AND a.status='ready'", (aid, user_id))
|
||||
return cur.fetchone()
|
||||
|
||||
@staticmethod
|
||||
def public_attachment(row):
|
||||
return {
|
||||
'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
|
||||
'filename': row['filename'], 'content_type': row['content_type'],
|
||||
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
|
||||
'error_code': row['error_code'], 'variants': [],
|
||||
'created_at': rfc3339(row['created_at']), 'updated_at': rfc3339(row['updated_at']),
|
||||
}
|
||||
|
||||
def add_message(self, chat_id, sender_id, text, key, attachment_ids=()):
|
||||
"""One transaction owns both the message and its replayable result.
|
||||
|
||||
The advisory lock serializes only the same user/method/path/key across
|
||||
processes. Hash collisions serialize extra requests, never alias keys:
|
||||
the complete key is still checked against the primary key in the table.
|
||||
A crash before COMMIT rolls back the message, links and replay record. A lost reply after COMMIT
|
||||
is recovered by replay. Transient failures are never recorded.
|
||||
"""
|
||||
path = '/api/v1/chats/%s/messages' % chat_id
|
||||
body = {'text': text}
|
||||
# Keep the lab3 digest for text-only requests, including explicit [].
|
||||
if attachment_ids:
|
||||
body['attachment_ids'] = list(attachment_ids)
|
||||
body_hash = hashlib.sha256(json.dumps(body, ensure_ascii=False,
|
||||
sort_keys=True).encode('utf-8')).hexdigest()
|
||||
scope = json.dumps([str(sender_id), 'POST', path, key], separators=(',', ':'))
|
||||
lock_id = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
|
||||
identifier = str(uuid.uuid4())
|
||||
stamp = utc_now()
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('SELECT pg_advisory_xact_lock(%s)', (lock_id,))
|
||||
cur.execute('SELECT request_hash, response_body FROM message_idempotency '
|
||||
'WHERE user_id=%s AND method=%s AND path=%s AND key=%s',
|
||||
(str(sender_id), 'POST', path, key))
|
||||
replay = cur.fetchone()
|
||||
if replay is not None:
|
||||
if replay['request_hash'] != body_hash:
|
||||
raise Conflict('Idempotency-Key уже использован с другим телом запроса')
|
||||
return {'attachment_ids': [], **replay['response_body']}
|
||||
if attachment_ids:
|
||||
cur.execute(
|
||||
'SELECT id, status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[]) FOR SHARE',
|
||||
(str(chat_id), list(attachment_ids)))
|
||||
attachments = cur.fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
raise NotFound('Вложение не найдено')
|
||||
if any(row['status'] != 'ready' for row in attachments):
|
||||
raise Conflict('Вложение не готово')
|
||||
cur.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
|
||||
'VALUES (%s, %s, %s, %s, %s) '
|
||||
'RETURNING id, chat_id, sender_id, text, created_at',
|
||||
(identifier, str(chat_id), str(sender_id), text, stamp))
|
||||
result = self._public_message(cur.fetchone())
|
||||
if attachment_ids:
|
||||
cur.executemany(
|
||||
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
|
||||
'VALUES (%s,%s,%s,%s)',
|
||||
[(identifier, aid, str(chat_id), pos) for pos, aid in enumerate(attachment_ids)])
|
||||
result['attachment_ids'] = list(attachment_ids)
|
||||
cur.execute('INSERT INTO message_idempotency '
|
||||
'(user_id, method, path, key, request_hash, response_status, response_body) '
|
||||
'VALUES (%s,%s,%s,%s,%s,201,%s)',
|
||||
(str(sender_id), 'POST', path, key, body_hash, Json(result)))
|
||||
return result
|
||||
|
||||
# --------------------------------------------------------------- pages
|
||||
def page_messages(self, chat_id, after_key, limit):
|
||||
"""Return (public messages, has_more) for a keyset window."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s', (str(chat_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
|
||||
'WHERE chat_id = %s '
|
||||
'AND (created_at, id) > (%s, %s) '
|
||||
'ORDER BY created_at, id '
|
||||
'LIMIT %s',
|
||||
(str(chat_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
items = [self._public_message(row) for row in rows[:limit]]
|
||||
if items:
|
||||
by_id = {item['id']: item for item in items}
|
||||
cur.execute(
|
||||
'SELECT message_id, attachment_id FROM message_attachments '
|
||||
'WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position',
|
||||
(list(by_id),))
|
||||
for row in cur.fetchall():
|
||||
by_id[str(row['message_id'])]['attachment_ids'].append(str(row['attachment_id']))
|
||||
has_more = len(rows) > limit
|
||||
return items, has_more
|
||||
|
||||
def page_chats(self, user_id, after_key, limit):
|
||||
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
|
||||
with self._db.cursor() as cur:
|
||||
if after_key is None:
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s', (str(user_id), limit + 1))
|
||||
else:
|
||||
stamp, identifier = after_key
|
||||
cur.execute(
|
||||
'SELECT c.id, c.title, c.created_at FROM chats c '
|
||||
'JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE cm.user_id = %s '
|
||||
'AND (c.created_at, c.id) > (%s, %s) '
|
||||
'ORDER BY c.created_at, c.id '
|
||||
'LIMIT %s',
|
||||
(str(user_id), stamp, str(identifier), limit + 1))
|
||||
rows = cur.fetchall()
|
||||
has_more = len(rows) > limit
|
||||
items = [self._public_chat(dict(row)) for row in rows[:limit]]
|
||||
return items, has_more
|
||||
|
||||
# ----------------------------------------------------------- projection
|
||||
def chat(self, chat_id):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
chat = self._public_chat(dict(row))
|
||||
cur.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
|
||||
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
|
||||
return chat
|
||||
|
||||
@staticmethod
|
||||
def is_member(chat, user_id):
|
||||
return str(user_id) in chat['member_ids']
|
||||
|
||||
@staticmethod
|
||||
def _public_user(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'username': row['username'],
|
||||
'display_name': row['display_name'],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_chat(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'title': row['title'],
|
||||
'member_ids': [], # filled by API when needed
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _public_message(row):
|
||||
return {
|
||||
'id': str(row['id']),
|
||||
'chat_id': str(row['chat_id']),
|
||||
'sender_id': str(row['sender_id']),
|
||||
'text': row['text'],
|
||||
'attachment_ids': [],
|
||||
'created_at': rfc3339(row['created_at']),
|
||||
}
|
||||
|
||||
# --------------------------------------------------------------- sessions
|
||||
def create_session(self, token_hash, user_id, expires_at):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
|
||||
'VALUES (%s, %s, %s, %s)',
|
||||
(token_hash, str(user_id), expires_at, utc_now()))
|
||||
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, str(user_id), ttl)
|
||||
|
||||
def get_session(self, token_hash):
|
||||
"""Return user_id for a valid non-expired session, or None.
|
||||
|
||||
The cache is consulted first, but every cache hit is re-validated
|
||||
against PostgreSQL so that a revoked or expired session is never
|
||||
trusted from a stale Redis key. On miss, read PostgreSQL and
|
||||
backfill the cache. Expired sessions found during lookup are deleted.
|
||||
"""
|
||||
cached = self._cache.get(token_hash)
|
||||
if cached is not None:
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
|
||||
(token_hash, utc_now()))
|
||||
if cur.fetchone() is None:
|
||||
self._cache.delete(token_hash)
|
||||
return None
|
||||
return cached
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute(
|
||||
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
|
||||
(token_hash,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
if row['expires_at'] <= utc_now():
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
return None
|
||||
user_id = str(row['user_id'])
|
||||
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
|
||||
self._cache.set(token_hash, user_id, ttl)
|
||||
return user_id
|
||||
|
||||
def delete_session(self, token_hash):
|
||||
self._cache.delete(token_hash)
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
|
||||
|
||||
def cleanup_expired_sessions(self):
|
||||
with self._db.cursor() as cur:
|
||||
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Strict request payload validation; any deviation is 400 invalid_request."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from .errors import InvalidRequest
|
||||
|
||||
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
|
||||
DIGITS_PATTERN = re.compile(r'[0-9]+')
|
||||
|
||||
|
||||
def safe_label(value):
|
||||
"""Render untrusted text for an error message without breaking its encoding.
|
||||
|
||||
Field names come straight from the client JSON, so they may contain the same
|
||||
lone surrogates that the payload itself is rejected for.
|
||||
"""
|
||||
text = value if isinstance(value, str) else repr(value)
|
||||
return text.encode('utf-8', 'replace').decode('utf-8')
|
||||
|
||||
|
||||
def require_encodable(value, name):
|
||||
"""Reject text that cannot survive a UTF-8 round trip.
|
||||
|
||||
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
|
||||
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
|
||||
otherwise the write is committed and the response encoding fails afterwards.
|
||||
"""
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise InvalidRequest(
|
||||
'Поле %s содержит недопустимые символы Unicode' % name) from None
|
||||
return value
|
||||
|
||||
|
||||
def require_object(payload):
|
||||
if not isinstance(payload, dict):
|
||||
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
|
||||
return payload
|
||||
|
||||
|
||||
def require_fields(payload, required, allowed):
|
||||
"""Reject missing required fields and any unknown field."""
|
||||
missing = [name for name in required if name not in payload]
|
||||
if missing:
|
||||
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
|
||||
extra = [name for name in payload if name not in allowed]
|
||||
if extra:
|
||||
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
|
||||
return payload
|
||||
|
||||
|
||||
def require_text_field(payload, name, minimum, maximum, strip_check=True):
|
||||
value = payload.get(name)
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой' % name)
|
||||
require_encodable(value, name)
|
||||
if strip_check and not value.strip():
|
||||
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
|
||||
if len(value) < minimum:
|
||||
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
|
||||
if len(value) > maximum:
|
||||
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
|
||||
return value
|
||||
|
||||
|
||||
def require_username(payload):
|
||||
username = payload.get('username')
|
||||
# fullmatch, not match: a trailing "$" would also accept a final newline.
|
||||
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
|
||||
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
|
||||
return require_encodable(username, 'username')
|
||||
|
||||
|
||||
def require_uuid_string(value, name):
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
|
||||
|
||||
|
||||
def require_attachment_ids(payload):
|
||||
value = payload.get('attachment_ids', [])
|
||||
if not isinstance(value, list) or len(value) > 10:
|
||||
raise InvalidRequest('attachment_ids должен содержать от 0 до 10 UUID')
|
||||
identifiers = [require_uuid_string(item, 'attachment_ids') for item in value]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise InvalidRequest('attachment_ids содержит повторяющиеся UUID')
|
||||
return identifiers
|
||||
|
||||
|
||||
def require_member_ids(payload, maximum, minimum=1):
|
||||
value = payload.get('member_ids')
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest('member_ids должен быть массивом UUID')
|
||||
if not minimum <= len(value) <= maximum:
|
||||
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
|
||||
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise InvalidRequest('member_ids содержит повторяющихся участников')
|
||||
return identifiers
|
||||
|
||||
|
||||
def parse_limit(raw, default, maximum):
|
||||
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
|
||||
|
||||
An empty value is a client error rather than an absent parameter, and the
|
||||
digit run is length-bounded before int() because Python refuses to convert
|
||||
strings longer than its integer-string limit, which would surface as a 500.
|
||||
"""
|
||||
if raw is None:
|
||||
return default
|
||||
if not DIGITS_PATTERN.fullmatch(raw):
|
||||
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
|
||||
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
|
||||
# significant digits against the maximum first rejects them without converting.
|
||||
significant = raw.lstrip('0') or '0'
|
||||
if len(significant) > len(str(maximum)):
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
value = int(significant)
|
||||
if not 1 <= value <= maximum:
|
||||
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
|
||||
return value
|
||||
|
||||
|
||||
def parse_cursor(raw):
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, str) or not raw:
|
||||
raise InvalidRequest('cursor должен быть непустой строкой')
|
||||
return raw
|
||||
|
||||
|
||||
def require_idempotency_key(value):
|
||||
"""Only the contract's bounded ASCII token is accepted."""
|
||||
if not isinstance(value, str) or re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', value) is None:
|
||||
raise InvalidRequest('Требуется корректный Idempotency-Key (1–128 ASCII символов)')
|
||||
return value
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Run cleanup scenarios against a disposable PostgreSQL database."""
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import sql
|
||||
from psycopg2.extensions import make_dsn
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.errors import Unavailable
|
||||
from messenger.store import Store
|
||||
|
||||
GRACE_SECONDS = 900
|
||||
|
||||
|
||||
def unavailable(_key):
|
||||
raise Unavailable('synthetic S3 failure')
|
||||
|
||||
|
||||
def expect_unavailable(operation):
|
||||
try:
|
||||
operation()
|
||||
except Unavailable:
|
||||
return
|
||||
raise AssertionError('Expected Unavailable')
|
||||
|
||||
|
||||
def exercise(database):
|
||||
store = Store(None, database, None)
|
||||
user, chat = str(uuid4()), str(uuid4())
|
||||
with database.cursor() as cur:
|
||||
cur.execute("INSERT INTO users VALUES (%s,'cleanup','unused','Cleanup')", (user,))
|
||||
cur.execute("INSERT INTO chats VALUES (%s,'Cleanup')", (chat,))
|
||||
|
||||
def reserve(expired=False):
|
||||
aid = str(uuid4())
|
||||
key = 'attachments/' + aid
|
||||
store.reserve_attachment(aid, chat, user, 'x.png', 'image/png', 1, 'a' * 64, key)
|
||||
if expired:
|
||||
with database.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (aid,))
|
||||
return aid, key
|
||||
|
||||
objects = set()
|
||||
fresh, fresh_key = reserve()
|
||||
objects.add(fresh_key)
|
||||
ready, ready_key = reserve(expired=True)
|
||||
store.finish_attachment(ready, objects.add)
|
||||
with database.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-interval '16 minutes' WHERE id=%s", (ready,))
|
||||
|
||||
active, active_key = reserve(expired=True)
|
||||
def active_put(key):
|
||||
objects.add(key)
|
||||
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
store.finish_attachment(active, active_put)
|
||||
|
||||
failed, failed_key = reserve(expired=True)
|
||||
def put_then_fail(key):
|
||||
objects.add(key)
|
||||
# Inject a failure after the object exists and before the ready update.
|
||||
raise Unavailable('synthetic failure after PUT')
|
||||
expect_unavailable(lambda: store.finish_attachment(failed, put_then_fail))
|
||||
assert failed_key in objects
|
||||
expect_unavailable(lambda: store.clean_pending_attachment(GRACE_SECONDS, unavailable))
|
||||
with database.cursor() as cur:
|
||||
cur.execute('SELECT status FROM attachments WHERE id=%s', (failed,))
|
||||
assert cur.fetchone()['status'] == 'pending'
|
||||
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
assert objects == {fresh_key, ready_key, active_key}
|
||||
|
||||
reserve(expired=True) # Reservation whose PUT never created an object.
|
||||
assert store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
assert not store.clean_pending_attachment(GRACE_SECONDS, objects.discard)
|
||||
with database.cursor() as cur:
|
||||
cur.execute('SELECT id::text AS id, status FROM attachments')
|
||||
assert {row['id']: row['status'] for row in cur.fetchall()} == {
|
||||
fresh: 'pending', ready: 'ready', active: 'ready',
|
||||
}
|
||||
print('PASS: PostgreSQL row lock protects active PUT; fresh and ready attachments preserved')
|
||||
print('PASS: interrupted PUT and failure after PUT cleaned; S3 failure retries safely')
|
||||
|
||||
|
||||
def main():
|
||||
dsn = os.environ['LAB4_TEST_DATABASE_URL']
|
||||
schema = 'cleanup_' + uuid4().hex
|
||||
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(sql.SQL('CREATE SCHEMA {}').format(sql.Identifier(schema)))
|
||||
config = Config.from_env({'DATABASE_URL': make_dsn(dsn, options='-csearch_path=' + schema)})
|
||||
database = Database(config)
|
||||
try:
|
||||
database.open()
|
||||
database.run_migrations()
|
||||
exercise(database)
|
||||
finally:
|
||||
database.close()
|
||||
with psycopg2.connect(dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(schema)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Check upload errors through Nginx and exercise an actual S3 outage."""
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=45)
|
||||
alice, _, eve, chat = client.bootstrap()
|
||||
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments"
|
||||
good = path + '?filename=sample.png'
|
||||
item, _ = client.attachment(alice, chat)
|
||||
content = f"/api/v1/attachments/{item['id']}/content"
|
||||
|
||||
def upload(url, payload, expected, mime='image/png', actor=alice):
|
||||
return client.call('POST', url, expected, actor=actor, raw=payload,
|
||||
headers={'Content-Type': mime})
|
||||
|
||||
huge = BytesIO()
|
||||
Image.new('1', (8001, 5000)).save(huge, 'PNG')
|
||||
for payload, expected, mime in [
|
||||
(b'x' * (10 * 1024 * 1024 + 1), 413, 'image/png'),
|
||||
(huge.getvalue(), 413, 'image/png'),
|
||||
(data[:30], 415, 'image/png'),
|
||||
(data, 415, 'image/jpeg'),
|
||||
]:
|
||||
upload(good, payload, expected, mime)
|
||||
upload(path + '?filename=a%2Fb.png', data, 400)
|
||||
upload(good, data, 404, actor=eve)
|
||||
client.call('GET', content, 404, actor=eve)
|
||||
print('PASS: size, pixel count, corruption, MIME, filename and chat ACL over Nginx')
|
||||
|
||||
def s3_command(action):
|
||||
subprocess.run(['docker', 'compose', action, 's3'], cwd=ROOT, check=True, timeout=30)
|
||||
|
||||
try:
|
||||
s3_command('stop')
|
||||
upload(good, data, 503)
|
||||
client.call('GET', content, 503, actor=alice)
|
||||
client.message(alice, chat, 'Текст при недоступном S3')
|
||||
finally:
|
||||
s3_command('start')
|
||||
|
||||
started = time.monotonic()
|
||||
for _ in range(40):
|
||||
try:
|
||||
downloaded = client.call('GET', content, actor=alice)
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(.25)
|
||||
else:
|
||||
raise Failure('S3 download did not recover after restart')
|
||||
check(downloaded['bytes'] == data, 'original changed after S3 restart')
|
||||
print(f'PASS: S3 outage returned 503, text remained available, original recovered in {time.monotonic()-started:.2f}s')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Stop the actual upload replica and check original bytes after failover and restart."""
|
||||
from hashlib import sha256
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=55)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
||||
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
reply = client.call('POST', path, 201, actor=alice, raw=original,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
aid = reply['json']['id']
|
||||
upload_instance = reply['headers']['X-Instance-Id']
|
||||
digest = sha256(original).hexdigest()
|
||||
content = '/api/v1/attachments/' + aid + '/content'
|
||||
text = client.message(alice, chat, '', attachments=[aid])['json']
|
||||
|
||||
def verify():
|
||||
received = client.call('GET', content, actor=bob)
|
||||
check(received['bytes'] == original and sha256(received['bytes']).hexdigest() == digest,
|
||||
'original changed')
|
||||
history = client.call('GET', f"/api/v1/chats/{chat['id']}/messages",
|
||||
actor=bob, schema='MessagePage')['json']
|
||||
check(text in history['items'], 'message attachment link lost')
|
||||
return received['headers']['X-Instance-Id']
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
subprocess.run(['scripts/stop-one.sh'], cwd=ROOT, check=True, timeout=30,
|
||||
env={**os.environ, 'TARGET_INSTANCE_ID': upload_instance})
|
||||
instance = verify()
|
||||
check(instance != upload_instance, 'download was served by the stopped uploader')
|
||||
print(f'PASS: {upload_instance} -> {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
|
||||
finally:
|
||||
subprocess.run([sys.executable, 'scripts/manage.py', 'restore'], cwd=ROOT, check=True, timeout=100)
|
||||
|
||||
start = time.monotonic()
|
||||
subprocess.run(['scripts/restart.sh'], cwd=ROOT, check=True, timeout=100)
|
||||
instance = verify()
|
||||
print(f'PASS: after full restart via {instance}, SHA-256 {digest}, {time.monotonic()-start:.2f}s')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,141 @@
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
import logging
|
||||
import os
|
||||
import http.client
|
||||
import socket
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import uuid4
|
||||
import json
|
||||
import sys
|
||||
|
||||
from botocore.response import StreamingBody
|
||||
from PIL import Image
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.environ['S3_BUCKET'] = 'test'
|
||||
sys.path.insert(0, str(root / 'src'))
|
||||
from messenger.api import Api
|
||||
from messenger.config import Config
|
||||
from messenger.security import hash_token
|
||||
from messenger.server import build_server
|
||||
from messenger.store import Store
|
||||
|
||||
chat, member, outsider = [str(uuid4()) for _ in range(3)]
|
||||
|
||||
class Metadata:
|
||||
rows = {}
|
||||
def get_session(self, token):
|
||||
return {hash_token('member'): member, hash_token('outsider'): outsider}.get(token)
|
||||
def user(self, uid):
|
||||
return {'id': uid}
|
||||
def chat(self, cid):
|
||||
return {'id': cid} if cid == chat else None
|
||||
def is_member(self, _chat, uid):
|
||||
return uid == member
|
||||
def reserve_attachment(self, aid, cid, uid, filename, mime, size, digest, key):
|
||||
now = datetime.now(timezone.utc)
|
||||
self.rows[aid] = dict(id=aid, chat_id=cid, owner_id=uid, filename=filename,
|
||||
content_type=mime, size_bytes=size, sha256=digest,
|
||||
object_key=key, status='pending', error_code=None,
|
||||
created_at=now, updated_at=now)
|
||||
def finish_attachment(self, aid, put_object):
|
||||
put_object(self.rows[aid]['object_key'])
|
||||
self.rows[aid]['status'] = 'ready'
|
||||
return self.rows[aid]
|
||||
def visible_attachment(self, aid, uid):
|
||||
row = self.rows.get(aid)
|
||||
if uid == member and row and row['status'] == 'ready':
|
||||
return row
|
||||
return None
|
||||
public_attachment = staticmethod(Store.public_attachment)
|
||||
|
||||
class S3:
|
||||
def __init__(self):
|
||||
self.objects = {}
|
||||
self.fail = False
|
||||
def put_object(self, **kw):
|
||||
if self.fail:
|
||||
from botocore.exceptions import EndpointConnectionError
|
||||
raise EndpointConnectionError(endpoint_url='http://s3.invalid')
|
||||
data = kw['Body'].read()
|
||||
assert len(data) == kw['ContentLength']
|
||||
self.objects[kw['Key']] = data
|
||||
def get_object(self, **kw):
|
||||
data = self.objects[kw['Key']]
|
||||
return {'ContentLength': len(data), 'Body': StreamingBody(BytesIO(data), len(data))}
|
||||
|
||||
config = Config.from_env({'APP_HOST':'127.0.0.1','APP_PORT':'8080'})
|
||||
config.port = 0
|
||||
api = Api(config, Metadata(), None)
|
||||
s3 = S3()
|
||||
api.attachments.client = s3
|
||||
server = build_server(config, logging.getLogger('lab4-check'), api, None)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = 'http://127.0.0.1:' + str(server.server_address[1])
|
||||
upload = f'/api/v1/chats/{chat}/attachments?filename=x.png'
|
||||
|
||||
def call(path, method='GET', data=None, mime='image/png', token='member'):
|
||||
headers = {'Authorization': 'Bearer ' + token, 'Content-Type': mime}
|
||||
try:
|
||||
response = urlopen(Request(base + path, data=data, headers=headers, method=method), timeout=12)
|
||||
except HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
return response.code, response.headers, response.read()
|
||||
|
||||
try:
|
||||
for image_format, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]:
|
||||
buf = BytesIO()
|
||||
Image.new('RGB', (10, 10), 'green').save(buf, format=image_format)
|
||||
data = buf.getvalue()
|
||||
status, _, raw = call(upload, 'POST', data, mime)
|
||||
assert status == 201, (status, raw)
|
||||
item = json.loads(raw)
|
||||
assert item['status'] == 'ready' and item['sha256'] == sha256(data).hexdigest()
|
||||
assert item['size_bytes'] == len(data) and 'object_key' not in item
|
||||
path = '/api/v1/attachments/' + item['id']
|
||||
assert json.loads(call(path)[2]) == item
|
||||
status, headers, actual = call(path + '/content')
|
||||
assert (status, headers['Content-Type'], actual) == (200, mime, data)
|
||||
assert call(path, token='outsider')[0] == 404
|
||||
assert call(path + '/content', token='outsider')[0] == 404
|
||||
assert call(path + '/content?variant=thumbnail')[0] == 404
|
||||
png = (root / 'tests/fixtures/sample.png').read_bytes()
|
||||
assert call(upload, 'POST', png)[0] == 201
|
||||
assert call(upload, 'POST', png, 'image/jpeg')[0] == 415
|
||||
assert call(upload, 'POST', png[:30])[0] == 415
|
||||
assert call(upload.replace('x.png','a%2Fb.png'), 'POST', png)[0] == 400
|
||||
before = len(api.store.rows)
|
||||
for length, partial, expected in [(10 * 1024 * 1024 + 1, b'', 413), (64, b'abc', 400)]:
|
||||
conn = http.client.HTTPConnection(*server.server_address, timeout=5)
|
||||
try:
|
||||
conn.request('POST', upload, body=partial, headers={
|
||||
'Authorization': 'Bearer member', 'Content-Type': 'image/png',
|
||||
'Content-Length': str(length)})
|
||||
if partial:
|
||||
conn.sock.shutdown(socket.SHUT_WR)
|
||||
with conn.getresponse() as response:
|
||||
assert response.status == expected, response.status
|
||||
response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
assert len(api.store.rows) == before
|
||||
buf = BytesIO()
|
||||
Image.new('1', (8001, 5000)).save(buf, 'PNG')
|
||||
assert call(upload, 'POST', buf.getvalue())[0] == 413
|
||||
s3.fail = True
|
||||
assert call(upload, 'POST', png)[0] == 503
|
||||
assert list(api.store.rows.values())[-1]['status'] == 'pending'
|
||||
print('PASS: local HTTP PNG/JPEG round trip, checksum, metadata, ACL and missing variant')
|
||||
print('PASS: malformed image, mismatched MIME, unsafe filename, byte/pixel limits and S3 error mapping')
|
||||
print('NOTE: metadata and S3 are test substitutes; real Compose integration not exercised')
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Exercise attachment links and message replay over the running lab4 API."""
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Message attachment links and replay')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
first, _ = client.attachment(alice, chat)
|
||||
second, _ = client.attachment(bob, chat)
|
||||
ids = [second['id'], first['id']]
|
||||
key = uuid.uuid4().hex
|
||||
sent = client.message(bob, chat, '', key=key, attachments=ids)['json']
|
||||
replay = client.message(bob, chat, '', key=key, attachments=ids)['json']
|
||||
check(sent == replay and sent['attachment_ids'] == ids, 'ordered attachment replay mismatch')
|
||||
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
client.call('POST', path, 409, actor=bob, body={'text': '', 'attachment_ids': ids[::-1]},
|
||||
headers={'Idempotency-Key': key})
|
||||
for invalid in [[ids[0]] * 2, [str(uuid.uuid4()) for _ in range(11)], ['bad UUID'], None]:
|
||||
client.call('POST', path, 400, actor=bob, body={'text': '', 'attachment_ids': invalid},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
client.call('POST', path, 400, actor=bob, body={'text': ''},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
client.call('POST', path, 404, actor=bob,
|
||||
body={'text': '', 'attachment_ids': [str(uuid.uuid4())]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
|
||||
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]},
|
||||
schema='Chat')['json']
|
||||
foreign, _ = client.attachment(alice, other)
|
||||
client.call('POST', path, 404, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [foreign['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
|
||||
plain_key = uuid.uuid4().hex
|
||||
plain = client.message(alice, chat, 'Обычный текст', key=plain_key)['json']
|
||||
check(client.message(alice, chat, 'Обычный текст', key=plain_key, attachments=[])['json'] == plain,
|
||||
'omitted and empty attachment_ids must replay identically')
|
||||
page = client.call('GET', path, actor=alice, schema='MessagePage')['json']
|
||||
check([item for item in page['items'] if item['id'] == sent['id']] == [sent],
|
||||
'ordered attachment IDs missing or duplicated in history')
|
||||
print('PASS: ordered attachments, replay, invalid input, cross-chat rejection and history')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Run via docker compose exec -T api-a python - with this file on stdin."""
|
||||
from hashlib import sha256
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from PIL import Image
|
||||
|
||||
from messenger.attachments import AttachmentStorage
|
||||
from messenger.clean_attachments import GRACE_SECONDS, clean_batch
|
||||
from messenger.config import Config
|
||||
from messenger.db import Database
|
||||
from messenger.errors import Unavailable
|
||||
from messenger.store import Store
|
||||
|
||||
|
||||
def main():
|
||||
config = Config.from_env()
|
||||
db = Database(config)
|
||||
db.open()
|
||||
store = Store(config, db, None)
|
||||
s3 = AttachmentStorage()
|
||||
user, chat, aid = [str(uuid4()) for _ in range(3)]
|
||||
key = 'attachments/' + aid
|
||||
payload = BytesIO()
|
||||
Image.new('RGB', (2, 2), 'green').save(payload, format='PNG')
|
||||
data = payload.getvalue()
|
||||
|
||||
try:
|
||||
with db.cursor() as cur:
|
||||
cur.execute('INSERT INTO users (id,username,password_hash,display_name) VALUES (%s,%s,%s,%s)',
|
||||
(user, 'gc_' + user[:8], 'unused', 'Cleanup check'))
|
||||
cur.execute("INSERT INTO chats (id,title) VALUES (%s,'Cleanup check')", (chat,))
|
||||
store.reserve_attachment(aid, chat, user, 'probe.png', 'image/png',
|
||||
len(data), sha256(data).hexdigest(), key)
|
||||
|
||||
def put_then_fail(object_key):
|
||||
s3.put(object_key, BytesIO(data), len(data), 'image/png')
|
||||
raise Unavailable('synthetic failure after successful PUT')
|
||||
|
||||
try:
|
||||
store.finish_attachment(aid, put_then_fail)
|
||||
except Unavailable as exc:
|
||||
assert str(exc) == 'synthetic failure after successful PUT'
|
||||
else:
|
||||
raise AssertionError('fault did not run')
|
||||
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
|
||||
with obj['Body'] as body:
|
||||
assert body.read() == data
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT status FROM attachments WHERE id=%s', (aid,))
|
||||
assert cur.fetchone()['status'] == 'pending'
|
||||
cur.execute("UPDATE attachments SET updated_at=now()-make_interval(secs => %s) WHERE id=%s",
|
||||
(GRACE_SECONDS + 1, aid))
|
||||
clean_batch(store, s3)
|
||||
with db.cursor() as cur:
|
||||
cur.execute('SELECT 1 FROM attachments WHERE id=%s', (aid,))
|
||||
assert cur.fetchone() is None
|
||||
try:
|
||||
obj = s3.client.get_object(Bucket=s3.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
assert exc.response['ResponseMetadata']['HTTPStatusCode'] == 404, exc
|
||||
else:
|
||||
obj['Body'].close()
|
||||
raise AssertionError('orphan object still exists')
|
||||
print('PASS: actual MinIO object existed after failed commit and was removed by cleanup')
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Measure sampled cgroup memory during ten simultaneous HTTP uploads."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
from threading import Barrier
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
COUNT = 10
|
||||
SAMPLER = """
|
||||
import select, sys, time
|
||||
from pathlib import Path
|
||||
usage = Path('/sys/fs/cgroup/memory.current')
|
||||
limit = Path('/sys/fs/cgroup/memory.max')
|
||||
cpu = Path('/sys/fs/cgroup/cpu.stat')
|
||||
print(limit.read_text().strip(), flush=True)
|
||||
while True:
|
||||
stats = dict(line.split() for line in cpu.read_text().splitlines())
|
||||
print(usage.read_text().strip(), stats['usage_usec'], flush=True)
|
||||
if select.select([sys.stdin], [], [], .05)[0]:
|
||||
break
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file, timeout=60)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
width = 1700
|
||||
source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3))
|
||||
buffer = BytesIO()
|
||||
source.save(buffer, format='PNG', compress_level=0)
|
||||
payload = buffer.getvalue()
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit')
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png"
|
||||
barrier = Barrier(COUNT + 1)
|
||||
samplers = {}
|
||||
|
||||
def upload():
|
||||
local = Client(args.base_url, args.ca_file, timeout=60)
|
||||
barrier.wait(timeout=30)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
reply = local.call('POST', path, 201, actor=alice, raw=payload,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
check(reply['json']['sha256'] == digest, 'upload hash mismatch')
|
||||
return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'],
|
||||
'seconds': time.monotonic() - start}
|
||||
except Failure as exc:
|
||||
return {'error': str(exc), 'seconds': time.monotonic() - start}
|
||||
|
||||
try:
|
||||
for service in ('api-a', 'api-b'):
|
||||
proc = subprocess.Popen(
|
||||
['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
|
||||
samplers[service] = proc
|
||||
with ThreadPoolExecutor(max_workers=COUNT) as pool:
|
||||
futures = [pool.submit(upload) for _ in range(COUNT)]
|
||||
limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()}
|
||||
initial = {service: tuple(map(int, proc.stdout.readline().split()))
|
||||
for service, proc in samplers.items()}
|
||||
start = time.monotonic()
|
||||
barrier.wait(timeout=30)
|
||||
requests = [future.result() for future in futures]
|
||||
elapsed = time.monotonic() - start
|
||||
finally:
|
||||
for proc in samplers.values():
|
||||
if proc.poll() is None:
|
||||
proc.stdin.close()
|
||||
resources = {}
|
||||
for service, proc in samplers.items():
|
||||
samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout]
|
||||
check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed')
|
||||
resources[service] = {
|
||||
'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples),
|
||||
'limit_bytes': limits[service], 'samples': len(samples),
|
||||
'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6,
|
||||
}
|
||||
successes = [row for row in requests if 'id' in row]
|
||||
durations = sorted(row['seconds'] for row in successes)
|
||||
percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1]
|
||||
for p in (50, 95, 99)} if durations else {}
|
||||
print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest,
|
||||
'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed,
|
||||
'latency_seconds': percentiles, 'requests': requests,
|
||||
'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2))
|
||||
check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user