1 Commits
Author SHA1 Message Date
samsik 11c8c03984 Собрать стенд lab4 с вложениями в приватном S3-bucket.
Метаданные остаются в PostgreSQL, байты — в RustFS, чтобы скачивание пережило остановку одной реплики API.
2026-11-06 08:49:44 +03:00
32 changed files with 2348 additions and 50 deletions
+16 -11
View File
@@ -1,20 +1,25 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
# Скопируйте в .env. Файл .env не коммитить.
# Пароли ниже — учебные заглушки локального стенда, не боевые секреты.
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
SESSION_STORE=valkey
POSTGRES_USER=messenger
POSTGRES_PASSWORD=local-only-change-me
POSTGRES_DB=messenger
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
DRAIN_SECONDS=3
# Не задавайте INSTANCE_ID: иначе все реплики получат один X-Instance-Id.
# DATABASE_URL и REDIS_URL собирает compose.yaml.
# Если задан DATABASE_URL_FILE или CACHE_URL_FILE и файл существует, приложение читает его.
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Ключ доступа без символа «/»: его требует SigV4. Секрет — отдельная строка.
S3_ACCESS_KEY_ID=LAB4LOCALACCESS
S3_SECRET_ACCESS_KEY=local-only-change-me-s3
# Необязательные файлы вместо переменных выше, если путь существует:
# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
+5
View File
@@ -0,0 +1,5 @@
*.sh text eol=lf
*.cfg text eol=lf
compose.yaml text eol=lf
Dockerfile text eol=lf
.env.example text eol=lf
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Самсонов Александр, БВТ2303
Целевая оценка (3/4/5): 3
Трек 5, если заявлен: нет
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: стенд lab3 (Traefik, две реплики, PostgreSQL, Valkey, идемпотентность) плюс приватный bucket RustFS 1.0.4 и синхронные вложения PNG/JPEG.
Команды запуска из чистого clone: см. REPORT.md
Результаты проверок и ссылка на REPORT.md: smoke, persistence и failover PASS; отчёт REPORT.md
Ограничения / известные ошибки: нет отдельного S3-пользователя и нет сборщика объектов, оставшихся после сбоя между PUT и COMMIT
Источники и использование ИИ: RustFS, библиотека minio, помощь ИИ в Cursor
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+31
View File
@@ -0,0 +1,31 @@
FROM python:3.12.6-slim-bookworm
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY --chown=app:app src/ /app/
COPY --chown=app:app migrations/ /app/migrations/
USER app
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab
EXPOSE 8080
ARG BUILD_ID=dev
LABEL lab.build-id=$BUILD_ID
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=8s --retries=5 \
CMD python3 -c "import urllib.request,sys; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2); sys.exit(0)"
CMD ["python3", "/app/server.py"]
+56 -25
View File
@@ -1,37 +1,68 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Самсонов Александр, БВТ2303
- Целевая оценка: 3
- Commit/PR: локально, до публикации
- Источники и помощь ИИ: контракт лабораторной, RustFS 1.0.4 как S3-совместимое хранилище (образы MinIO на Docker Hub к ноябрю 2026 недоступны), библиотека minio 7.2.15, перенос стенда lab3. Сборку сервера и проверки помог сделать ИИ в Cursor.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine с Compose v2 и Python 3.9+ для публичных тестов. Образ API — Python 3.12.6. Проверка выполнена в WSL Ubuntu 22.04: Docker 29.1.3, Compose 2.40.3.
## Архитектура и решения
```sh
cp .env.example .env
chmod +x scripts/up.sh scripts/restart.sh scripts/stop-one.sh scripts/restore-api.sh scripts/check-private-bucket.sh
docker compose --env-file .env -f compose.yaml up -d --build --scale api=2 --remove-orphans
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Тот же запуск: `sh scripts/up.sh`. Дождитесь `GET http://127.0.0.1:8080/health/ready` → `{"status":"ok"}`.
## Доказательства критериев
Публичный порт только у `lb`: `127.0.0.1:8080`. У API и RustFS порты наружу не опубликованы. Миграции выполняет одноразовый `migrate`, приватный bucket создаёт одноразовый `s3-init` (`mc` не нужен: клиент S3 вызывает `make_bucket`, если bucket ещё нет). Повторный запуск bucket не публикует. Политика anonymous не ставится.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
Остановка без удаления данных:
```sh
docker compose --env-file .env -f compose.yaml stop
```
Удаление учебных volumes:
```sh
docker compose --env-file .env -f compose.yaml down -v
```
После `scripts/stop-one.sh` остановленная реплика не поднимается сама. Вернуть её: `sh scripts/restore-api.sh`.
Образы: `python:3.12.6-slim-bookworm`, `postgres:16.6-bookworm`, `valkey/valkey:8.0.2`, `traefik:v3.7.16`, `rustfs/rustfs:1.0.4`.
## Архитектура
```
Клиент → Traefik → API ×2 → PostgreSQL (пользователи, чаты, сообщения, метаданные вложений)
→ Valkey (сессии, AOF)
→ RustFS (байты оригинала, volume s3data)
```
Из lab3 сохранены две реплики, общий PostgreSQL и Valkey, Traefik без sticky, `X-Instance-Id`, идемпотентность POST сообщения, drain по SIGTERM. API не хранит оригинал на своём диске: ключ объекта `o/<uuid>` назначает сервер, `filename` только в метаданных.
`/health/ready` отвечает 200, только если живы PostgreSQL, Valkey и bucket. Иначе 503, и Traefik перестаёт слать трафик на реплику. Период проверки Traefik — 1 с, timeout — 1 с, верхняя оценка исключения мёртвого backend — 2 с.
201 на загрузку возвращается после PUT в bucket и COMMIT строки `attachments`. Если запись в БД не прошла, процесс удаляет только что положенный объект. Если процесс умрёт между PUT и COMMIT, объект может остаться без строки в PostgreSQL: фонового сборщика мусора нет.
Лимит байт проверяется по `Content-Length` до чтения тела. Размеры PNG/JPEG читаются из IHDR/SOF, пиксели не распаковываются. Больше 10 MiB или 40 млн пикселей — 413. Не PNG/JPEG или несовпадение байтов и `Content-Type` — 415. Имя с `/`, `\` или управляющим символом — 400.
## Доказательства
| Критерий | Команда | Результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke, checksum, чужой пользователь | `python3 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS | evidence/smoke.txt |
| Restart API, PostgreSQL, Valkey и RustFS, тот же токен и тот же файл | `python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080` | PASS | evidence/persistence.txt |
| Upload, остановка обслужившей реплики, чтение через другую | `python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080` | 2 реплики, 139 успешных записей, 4 ошибки, первое восстановление 3.63 с, PASS | evidence/failover.txt |
| Опасное имя, подмена MIME, 50 млн пикселей, чужой чат | `python3 scripts/grade3_extra.py --base-url http://127.0.0.1:8080` | PASS, коды 400/415/413/404 | evidence/grade3-extra.txt |
| Content-Length 10485761 при теле в 1 байт, прямо в API | `docker compose cp scripts/check_size_header.py api:/tmp/check_size_header.py` и `exec api python3 /tmp/check_size_header.py` | 413 за 0.001 с | evidence/grade3-extra.txt |
| Приватный bucket | `sh scripts/check-private-bucket.sh` | анонимный GET 403 | evidence/grade3-extra.txt |
| Недоступность S3 | `docker compose stop s3`, затем запрос `/health/ready` | 503; после `start s3` снова 200 | evidence/s3-down.txt |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Ограничения
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Уровень 4 не делался: нет отдельного пользователя S3 с урезанными правами, нет фоновой сверки осиротевших объектов, нет замера RAM на 10 одновременных загрузках. Консоль RustFS выключена (`RUSTFS_CONSOLE_ENABLE=false`) и наружу не опубликована. Один и тот же ключ из `.env` является корневым ключом RustFS и ключом API. Это учебный локальный стенд.
+209
View File
@@ -0,0 +1,209 @@
# Локальный стенд lab4: клиент → Traefik → API ×2 → PostgreSQL + Valkey + RustFS.
# Публичный порт только у lb. У API и S3 нет ports и нет container_name.
# Миграции и создание bucket — одноразовые сервисы, не каждая реплика.
name: mtusi-lab4
services:
postgres:
image: postgres:16.6-bookworm
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
POSTGRES_DB: ${POSTGRES_DB:-messenger}
volumes:
- pgdata:/var/lib/postgresql/data
shm_size: 128mb
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-messenger} -d ${POSTGRES_DB:-messenger}"]
interval: 2s
timeout: 3s
retries: 20
start_period: 5s
restart: unless-stopped
stop_grace_period: 20s
networks: [lab]
deploy:
resources:
limits:
cpus: "0.75"
memory: 512M
valkey:
image: valkey/valkey:8.0.2
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--save", "60", "1"]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 2s
timeout: 3s
retries: 20
start_period: 3s
restart: unless-stopped
stop_grace_period: 10s
networks: [lab]
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
s3:
image: rustfs/rustfs:1.0.4
environment:
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_VOLUMES: /data
RUSTFS_CHECK_UPDATE: "false"
volumes:
- s3data:/data
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"]
interval: 2s
timeout: 3s
retries: 20
start_period: 5s
restart: unless-stopped
stop_grace_period: 15s
networks: [lab]
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
migrate:
build: .
image: mtusi-messenger:lab4
command: ["python3", "/app/migrate.py"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
depends_on:
postgres:
condition: service_healthy
restart: "no"
networks: [lab]
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
s3-init:
image: mtusi-messenger:lab4
command: ["python3", "/app/s3_init.py"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
depends_on:
s3:
condition: service_healthy
restart: "no"
networks: [lab]
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
api:
image: mtusi-messenger:lab4
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: valkey
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
REDIS_URL: redis://valkey:6379/0
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
valkey:
condition: service_healthy
expose:
- "8080"
labels:
- traefik.enable=true
- traefik.docker.network=mtusi-lab4-net
- traefik.http.routers.messenger.rule=PathPrefix(`/`)
- traefik.http.routers.messenger.entrypoints=web
- traefik.http.services.messenger.loadbalancer.server.port=8080
- traefik.http.services.messenger.loadbalancer.healthcheck.path=/health/ready
- traefik.http.services.messenger.loadbalancer.healthcheck.interval=1s
- traefik.http.services.messenger.loadbalancer.healthcheck.timeout=1s
- traefik.http.services.messenger.loadbalancer.healthcheck.scheme=http
- traefik.http.services.messenger.loadbalancer.passhostheader=true
healthcheck:
test:
- CMD
- python3
- -c
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)
interval: 2s
timeout: 2s
retries: 8
start_period: 8s
init: true
restart: unless-stopped
stop_grace_period: 10s
stop_signal: SIGTERM
networks: [lab]
deploy:
replicas: 2
resources:
limits:
cpus: "0.50"
memory: 512M
lb:
image: traefik:v3.7.16
command: ["--configFile=/etc/traefik/traefik.yml"]
volumes:
- ./infra/traefik.yml:/etc/traefik/traefik.yml:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
ports:
- "127.0.0.1:8080:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8082/ping"]
interval: 2s
timeout: 2s
retries: 8
start_period: 3s
restart: unless-stopped
stop_grace_period: 5s
networks: [lab]
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
networks:
lab:
name: mtusi-lab4-net
driver: bridge
volumes:
pgdata:
valkeydata:
s3data:
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 4, "first_recovery_seconds": 3.63}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+3
View File
@@ -0,0 +1,3 @@
PASS: filename, MIME, pixels, outsider, foreign chat
oversized Content-Length -> 413 in 0.001s
anonymous S3 status 403
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+4
View File
@@ -0,0 +1,4 @@
docker compose stop s3
HTTP/1.1 503 Service Unavailable
После docker compose start s3 готовность /health/ready снова 200.
Анонимный GET http://s3:9000/messenger-attachments/ из сети стенда: 403.
+1
View File
@@ -0,0 +1 @@
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+45
View File
@@ -0,0 +1,45 @@
# Traefik 3.7.16. Без sticky cookie.
# Адреса API Traefik берёт из Docker и обновляет по событиям start/die,
# поэтому новая реплика и остановленная не зависят от кэша одного DNS-ответа.
# Healthcheck: период 1s, timeout 1s, одна неуспешная проверка исключает backend.
# Верхняя оценка обнаружения отказа: 1s до следующей проверки + 1s timeout = 2s.
# Таймауты чтения запроса и ответа — 60s, чтобы прошёл оригинал до 10 MiB.
# Обнаружение мёртвой реплики от этого не зависит: healthcheck по-прежнему 1s+1s.
global:
checkNewVersion: false
sendAnonymousUsage: false
entryPoints:
web:
address: ":8080"
http:
aliasHeadersStrategy: reject
transport:
respondingTimeouts:
readTimeout: 60s
writeTimeout: 60s
idleTimeout: 30s
ping:
address: "127.0.0.1:8082"
http:
aliasHeadersStrategy: delete
ping:
entryPoint: ping
providers:
docker:
endpoint: unix:///var/run/docker.sock
exposedByDefault: false
network: mtusi-lab4-net
watch: true
serversTransport:
forwardingTimeouts:
dialTimeout: 1s
responseHeaderTimeout: 60s
idleConnTimeout: 1s
log:
level: INFO
+49
View File
@@ -0,0 +1,49 @@
-- Пользователи, чаты, участники, сообщения.
-- Сессии здесь не хранятся: их источник истины — Valkey (AOF + volume).
-- Идемпотентность отправки — в 002_idempotency.sql.
--
-- Транзакции приложения:
-- регистрация — один INSERT, уникальность username держит UNIQUE
-- создание чата — одна транзакция: чат и все chat_members, иначе откат
-- сообщение и запись идемпотентности — одна транзакция
--
-- Индекс истории messages_history_idx покрывает keyset-запрос
-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
-- ORDER BY created_at, id
-- LIMIT $4
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL,
CONSTRAINT users_username_unique UNIQUE (username)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX chats_created_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position integer NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
sender_id uuid NOT NULL REFERENCES users (id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
+18
View File
@@ -0,0 +1,18 @@
-- Общий результат POST сообщения.
-- Область: user_id + http_method + path + idem_key.
-- Строка пишется в той же транзакции, что и messages, поэтому 5xx
-- до COMMIT не оставляет ни сообщения, ни «окончательного» ключа.
-- Повтор с другим request_hash в пределах TTL — отказ без второй вставки.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
http_method text NOT NULL,
path text NOT NULL,
idem_key text NOT NULL,
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id),
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, http_method, path, idem_key)
);
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+27
View File
@@ -0,0 +1,27 @@
-- Метаданные вложения. Байты лежат в приватном S3-bucket, не в этой таблице.
-- object_key назначает сервер. filename только для отображения.
-- Связь сообщения и вложения — message_attachments, порядок задаёт position.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
created_at timestamptz NOT NULL,
CONSTRAINT attachments_object_key_unique UNIQUE (object_key)
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
attachment_id uuid NOT NULL REFERENCES attachments (id),
position integer NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
+4
View File
@@ -0,0 +1,4 @@
psycopg[binary]==3.2.3
psycopg-pool==3.2.3
redis==5.1.1
minio==7.2.15
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env -f compose.yaml exec -T api python3 - <<'PY'
from urllib.request import urlopen
try:
urlopen("http://s3:9000/messenger-attachments/", timeout=3)
raise SystemExit("anonymous bucket read was allowed")
except Exception as exc:
code = getattr(exc, "code", None)
if code not in (401, 403):
raise SystemExit("unexpected result %s %s" % (type(exc).__name__, code))
print("anonymous S3 status", code)
PY
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Проверка лимита размера на самом API, мимо балансировщика. Токен не печатает."""
import http.client
import importlib
import json
import time
import uuid
import urllib.request
urlerr = importlib.import_module("urllib.error")
BASE = "http://127.0.0.1:8080"
def call(method, path, body=None, headers=None, timeout=5):
req = urllib.request.Request(BASE + path, data=body, headers=headers or {}, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
return response.status, response.read()
except urlerr.HTTPError as exc:
return exc.code, exc.read()
def main():
username = "u_sz_" + uuid.uuid4().hex[:12]
password = "T3st-" + uuid.uuid4().hex
status, raw = call(
"POST", "/api/v1/users",
json.dumps({"username": username, "display_name": "Size", "password": password}).encode(),
{"Content-Type": "application/json"},
)
if status != 201:
raise SystemExit("register %s" % status)
user = json.loads(raw)
import base64
token = base64.b64encode((username + ":" + password).encode()).decode()
status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + token})
if status != 201:
raise SystemExit("login %s" % status)
bearer = json.loads(raw)["token"]
auth = {"Authorization": "Bearer " + bearer, "Content-Type": "application/json"}
status, raw = call(
"POST", "/api/v1/chats",
json.dumps({"title": "limit", "member_ids": [user["id"]]}).encode(),
auth,
)
if status != 201:
raise SystemExit("chat %s" % status)
chat = json.loads(raw)["id"]
# Заголовок больше лимита, тело из одного байта: API должен ответить до чтения остальных байт.
started = time.monotonic()
conn = http.client.HTTPConnection("127.0.0.1", 8080, timeout=3)
conn.request(
"POST",
"/api/v1/chats/" + chat + "/attachments?filename=big.png",
body=b"x",
headers={
"Authorization": "Bearer " + bearer,
"Content-Type": "image/png",
"Content-Length": "10485761",
},
)
response = conn.getresponse()
status = response.status
elapsed = time.monotonic() - started
conn.close()
if status != 413:
raise SystemExit("expected 413, got %s" % status)
if elapsed > 2:
raise SystemExit("limit check took too long: %.3f" % elapsed)
print("oversized Content-Length -> %s in %.3fs" % (status, elapsed))
if __name__ == "__main__":
main()
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Негативные проверки вложений для уровня 3. Токены не печатает."""
import hashlib
import struct
import sys
import uuid
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def png(width: int, height: int) -> bytes:
signature = b"\x89PNG\r\n\x1a\n"
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
chunk = b"IHDR" + ihdr
crc = zlib.crc32(chunk) & 0xFFFFFFFF
return signature + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc)
def main():
args = arguments("Lab4 grade-3 attachment checks")
base = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = base.bootstrap()
other = base.call(
"POST", "/api/v1/chats", 201, actor=alice,
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
sample = (ROOT / "tests/fixtures/sample.png").read_bytes()
att, content = base.attachment(alice, chat)
linked = base.message(alice, chat, "", attachments=[att["id"]])["json"]
check(linked["attachment_ids"] == [att["id"]], "message lost attachment")
downloaded = base.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)
check(downloaded["bytes"] == content, "bob download differs")
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == att["sha256"], "checksum differs")
for suffix in ("", "/content"):
base.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
base.call(
"POST", "/api/v1/chats/" + other["id"] + "/messages", 404, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
base.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=a/b.png", 400,
actor=alice, raw=sample, headers={"Content-Type": "image/png"},
)
base.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=photo.png", 415,
actor=alice, raw=sample, headers={"Content-Type": "image/jpeg"},
)
base.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=huge.png", 413,
actor=alice, raw=png(10000, 5000), headers={"Content-Type": "image/png"},
)
print("PASS: filename, MIME, pixels, outsider, foreign chat")
if __name__ == "__main__":
run(main)
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Перезапуск API и всех хранилищ. Volumes не удаляются, схема и объекты остаются.
docker compose --env-file .env -f compose.yaml restart api postgres valkey s3
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
if [ -z "${replicas}" ]; then
replicas=2
fi
docker compose --env-file .env -f compose.yaml up -d --no-deps --scale "api=${replicas}" --no-recreate api
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
cd "$(dirname "$0")/.."
found=""
for cid in $(docker compose --env-file .env -f compose.yaml ps -q api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$cid")
short=$(printf '%s' "$cid" | cut -c1-12)
if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then
found=$cid
break
fi
done
if [ -z "$found" ]; then
echo "API replica was not found" >&2
exit 1
fi
# docker stop + restart: unless-stopped оставляет реплику выключенной.
docker stop -t 8 "$found" >/dev/null
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
echo "Copy .env.example to .env" >&2
exit 1
fi
replicas=$(sed -n 's/^API_REPLICAS=//p' .env | tr -d '\r')
if [ -z "${replicas}" ]; then
replicas=2
fi
docker compose --env-file .env -f compose.yaml up -d --build --scale "api=${replicas}" --remove-orphans
+603
View File
@@ -0,0 +1,603 @@
"""PostgreSQL: пользователи, чаты, участники, сообщения, идемпотентность."""
from __future__ import annotations
import hashlib
import os
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
import psycopg
from psycopg.errors import UniqueViolation
from psycopg_pool import ConnectionPool, PoolTimeout
from errors import ApiError
_pool: ConnectionPool | None = None
def migrations_dir() -> Path:
here = Path(__file__).resolve().parent
for candidate in (here / "migrations", here.parent / "migrations"):
if candidate.is_dir():
return candidate
return here.parent / "migrations"
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE", "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get("DATABASE_URL", "").strip()
def idempotency_ttl() -> int:
raw = os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")
try:
value = int(raw)
except ValueError:
return 86400
return value if value >= 1 else 86400
def init() -> None:
global _pool
url = database_url()
if not url:
raise RuntimeError("database is not configured")
_pool = ConnectionPool(
url,
min_size=0,
max_size=12,
timeout=3,
open=True,
kwargs={
"connect_timeout": 3,
"autocommit": False,
"options": "-c statement_timeout=5000",
},
)
def migrate_with_retry() -> None:
last = None
for _ in range(30):
try:
migrate()
return
except Exception as exc:
last = exc
time.sleep(1)
raise RuntimeError("migrations failed") from last
def migrate() -> None:
url = database_url()
if not url:
raise RuntimeError("database is not configured")
with psycopg.connect(url, connect_timeout=3, autocommit=False) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (842021,))
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT now()
)
"""
)
conn.commit()
applied = {
row[0]
for row in conn.execute("SELECT version FROM schema_migrations").fetchall()
}
for path in sorted(migrations_dir().glob("*.sql")):
if path.name in applied:
continue
sql = path.read_text(encoding="utf-8")
with conn.transaction():
for statement in sql.split(";"):
if statement.strip():
conn.execute(statement)
conn.execute(
"INSERT INTO schema_migrations (version) VALUES (%s)",
(path.name,),
)
@contextmanager
def connection():
if _pool is None:
raise ApiError(503, "unavailable", "База данных недоступна")
try:
with _pool.connection() as conn:
yield conn
except ApiError:
raise
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc:
raise ApiError(503, "unavailable", "База данных недоступна") from exc
def ping() -> bool:
try:
with connection() as conn:
conn.execute("SELECT 1")
return True
except ApiError:
return False
def close() -> None:
if _pool is not None:
_pool.close()
def user_json(row) -> dict:
return {
"id": str(row[0]),
"username": row[1],
"display_name": row[2],
"created_at": format_ts(row[3]),
}
def format_ts(value: datetime) -> str:
value = value.astimezone(timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def create_user(username: str, display_name: str, password_hash: str, created: datetime) -> dict:
uid = uuid.uuid4()
try:
with connection() as conn:
with conn.transaction():
row = conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(uid, username, display_name, password_hash, created),
).fetchone()
except UniqueViolation as exc:
raise ApiError(409, "conflict", "username занят") from exc
return user_json(row)
def user_for_login(username: str):
with connection() as conn:
return conn.execute(
"""
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = %s
""",
(username,),
).fetchone()
def get_user(user_id: str) -> dict | None:
with connection() as conn:
row = conn.execute(
"""
SELECT id, username, display_name, created_at
FROM users
WHERE id = %s
""",
(uuid.UUID(user_id),),
).fetchone()
if row is None:
return None
return user_json(row)
def create_chat(title: str, member_ids: list[str], created: datetime) -> dict:
cid = uuid.uuid4()
members = [uuid.UUID(item) for item in member_ids]
with connection() as conn:
with conn.transaction():
conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(cid, title, created),
)
found = conn.execute(
"SELECT id FROM users WHERE id = ANY(%s::uuid[])",
(members,),
).fetchall()
if {row[0] for row in found} != set(members):
raise ApiError(400, "invalid_request", "неизвестный участник")
for position, member in enumerate(members):
conn.execute(
"""
INSERT INTO chat_members (chat_id, user_id, position)
VALUES (%s, %s, %s)
""",
(cid, member, position),
)
return {
"id": str(cid),
"title": title,
"member_ids": list(member_ids),
"created_at": format_ts(created),
}
def _chat_row(row) -> dict:
return {
"id": str(row[0]),
"title": row[1],
"created_at": format_ts(row[2]),
"member_ids": [str(item) for item in row[3]],
}
def get_chat(chat_id: str, user_id: str) -> dict:
with connection() as conn:
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at,
array_agg(cm.user_id ORDER BY cm.position)
FROM chats c
JOIN chat_members mine
ON mine.chat_id = c.id AND mine.user_id = %s
JOIN chat_members cm ON cm.chat_id = c.id
WHERE c.id = %s
GROUP BY c.id
""",
(uuid.UUID(user_id), uuid.UUID(chat_id)),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Чат не найден")
chat = _chat_row(row)
return {
"id": chat["id"],
"title": chat["title"],
"member_ids": chat["member_ids"],
"created_at": chat["created_at"],
}
def list_chats(user_id: str, limit: int, cursor) -> dict:
params: list = [uuid.UUID(user_id)]
where = ""
if cursor is not None:
created, chat_id = cursor
where = "WHERE (c.created_at, c.id) > (%s, %s)"
params.extend([created, uuid.UUID(chat_id)])
params.append(limit + 1)
sql = f"""
SELECT c.id, c.title, c.created_at,
array_agg(cm.user_id ORDER BY cm.position)
FROM chats c
JOIN chat_members mine
ON mine.chat_id = c.id AND mine.user_id = %s
JOIN chat_members cm ON cm.chat_id = c.id
{where}
GROUP BY c.id
ORDER BY c.created_at, c.id
LIMIT %s
"""
with connection() as conn:
rows = conn.execute(sql, params).fetchall()
page = rows[:limit]
next_cursor = None
if len(rows) > limit and page:
last = page[-1]
next_cursor = (last[2], str(last[0]))
return {
"items": [
{
"id": str(row[0]),
"title": row[1],
"member_ids": [str(item) for item in row[3]],
"created_at": format_ts(row[2]),
}
for row in page
],
"next_cursor_parts": next_cursor,
}
def _lock_key(scope: str) -> int:
digest = hashlib.sha256(scope.encode("utf-8")).digest()
return int.from_bytes(digest[:8], "big", signed=True)
def _is_member(conn, chat_id: str, user_id: str) -> bool:
row = conn.execute(
"""
SELECT 1 FROM chat_members
WHERE chat_id = %s AND user_id = %s
""",
(uuid.UUID(chat_id), uuid.UUID(user_id)),
).fetchone()
return row is not None
def _attachment_ids(value) -> list:
if not value:
return []
return [str(item) for item in value]
def _load_message(conn, message_id) -> dict:
row = conn.execute(
"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
COALESCE(
array_agg(ma.attachment_id ORDER BY ma.position)
FILTER (WHERE ma.attachment_id IS NOT NULL),
'{}'::uuid[]
)
FROM messages m
LEFT JOIN message_attachments ma ON ma.message_id = m.id
WHERE m.id = %s
GROUP BY m.id
""",
(message_id,),
).fetchone()
if row is None:
raise ApiError(500, "internal_error", "Внутренняя ошибка")
return message_json(row)
def _require_chat_attachments(conn, chat_id: str, attachment_ids: list[str]) -> None:
if not attachment_ids:
return
rows = conn.execute(
"""
SELECT id::text, chat_id::text
FROM attachments
WHERE id = ANY(%s::uuid[])
""",
(attachment_ids,),
).fetchall()
found = {row[0]: row[1] for row in rows}
for att_id in attachment_ids:
if found.get(att_id) != chat_id:
raise ApiError(404, "not_found", "Вложение не найдено")
def create_message_idempotent(
chat_id: str,
sender_id: str,
text: str,
attachment_ids: list[str],
created: datetime,
method: str,
path: str,
idem_key: str,
request_hash: str,
) -> dict:
"""Один ключ — одно сообщение, в том числе при гонке двух реплик.
pg_advisory_xact_lock сериализует одинаковую область.
INSERT сообщения, связей и ключа коммитятся вместе.
"""
scope = "%s|%s|%s|%s" % (sender_id, method, path, idem_key)
ttl = idempotency_ttl()
with connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s::bigint)", (_lock_key(scope),))
existing = conn.execute(
"""
SELECT request_hash, message_id, created_at
FROM idempotency_keys
WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s
""",
(uuid.UUID(sender_id), method, path, idem_key),
).fetchone()
if existing is not None:
age = conn.execute("SELECT now()").fetchone()[0] - existing[2]
if age.total_seconds() > ttl:
conn.execute(
"""
DELETE FROM idempotency_keys
WHERE user_id = %s AND http_method = %s AND path = %s AND idem_key = %s
""",
(uuid.UUID(sender_id), method, path, idem_key),
)
existing = None
if existing is not None:
if existing[0] != request_hash:
raise ApiError(409, "conflict", "повторный ключ с другим сообщением")
if not _is_member(conn, chat_id, sender_id):
raise ApiError(404, "not_found", "Чат не найден")
return _load_message(conn, existing[1])
if not _is_member(conn, chat_id, sender_id):
raise ApiError(404, "not_found", "Чат не найден")
_require_chat_attachments(conn, chat_id, attachment_ids)
mid = uuid.uuid4()
conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(mid, uuid.UUID(chat_id), uuid.UUID(sender_id), text, created),
)
for position, att_id in enumerate(attachment_ids):
conn.execute(
"""
INSERT INTO message_attachments (message_id, attachment_id, position)
VALUES (%s, %s, %s)
""",
(mid, uuid.UUID(att_id), position),
)
conn.execute(
"""
INSERT INTO idempotency_keys (
user_id, http_method, path, idem_key, request_hash, message_id, created_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(uuid.UUID(sender_id), method, path, idem_key, request_hash, mid, created),
)
return _load_message(conn, mid)
def list_messages(chat_id: str, user_id: str, limit: int, cursor) -> dict:
params: list = [uuid.UUID(chat_id), uuid.UUID(chat_id), uuid.UUID(user_id)]
cursor_sql = ""
if cursor is not None:
created, message_id = cursor
cursor_sql = "AND (m.created_at, m.id) > (%s, %s)"
params.extend([created, uuid.UUID(message_id)])
params.append(limit + 1)
sql = f"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
COALESCE(
array_agg(ma.attachment_id ORDER BY ma.position)
FILTER (WHERE ma.attachment_id IS NOT NULL),
'{{}}'::uuid[]
)
FROM messages m
LEFT JOIN message_attachments ma ON ma.message_id = m.id
WHERE m.chat_id = %s
AND EXISTS (
SELECT 1 FROM chat_members
WHERE chat_id = %s AND user_id = %s
)
{cursor_sql}
GROUP BY m.id
ORDER BY m.created_at, m.id
LIMIT %s
"""
with connection() as conn:
rows = conn.execute(sql, params).fetchall()
if not rows:
with connection() as conn:
member = conn.execute(
"""
SELECT 1 FROM chat_members
WHERE chat_id = %s AND user_id = %s
""",
(uuid.UUID(chat_id), uuid.UUID(user_id)),
).fetchone()
if member is None:
raise ApiError(404, "not_found", "Чат не найден")
page = rows[:limit]
next_cursor = None
if len(rows) > limit and page:
last = page[-1]
next_cursor = (last[4], str(last[0]))
return {
"items": [message_json(row) for row in page],
"next_cursor_parts": next_cursor,
}
def message_json(row) -> dict:
return {
"id": str(row[0]),
"chat_id": str(row[1]),
"sender_id": str(row[2]),
"text": row[3],
"created_at": format_ts(row[4]),
"attachment_ids": _attachment_ids(row[5]),
}
def is_member(chat_id: str, user_id: str) -> bool:
try:
with connection() as conn:
return _is_member(conn, chat_id, user_id)
except ApiError:
raise
def insert_attachment(
att_id: str,
chat_id: str,
owner_id: str,
filename: str,
content_type: str,
size_bytes: int,
digest: str,
object_key: str,
width: int,
height: int,
created: datetime,
) -> dict:
with connection() as conn:
with conn.transaction():
if not _is_member(conn, chat_id, owner_id):
raise ApiError(404, "not_found", "Чат не найден")
row = conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, width, height, created_at
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, created_at
""",
(
uuid.UUID(att_id),
uuid.UUID(chat_id),
uuid.UUID(owner_id),
filename,
content_type,
size_bytes,
digest,
object_key,
width,
height,
created,
),
).fetchone()
return _attachment_public(row)
def get_attachment(att_id: str, user_id: str) -> dict:
row = _attachment_row(att_id, user_id)
return _attachment_public(row)
def get_attachment_record(att_id: str, user_id: str) -> dict:
row = _attachment_row(att_id, user_id)
return {
"content_type": row[4],
"size_bytes": row[5],
"sha256": row[6],
"object_key": row[7],
}
def _attachment_row(att_id: str, user_id: str):
with connection() as conn:
row = conn.execute(
"""
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.object_key, a.created_at
FROM attachments a
WHERE a.id = %s
AND EXISTS (
SELECT 1 FROM chat_members
WHERE chat_id = a.chat_id AND user_id = %s
)
""",
(uuid.UUID(att_id), uuid.UUID(user_id)),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
def _attachment_public(row) -> dict:
created = format_ts(row[8])
return {
"id": str(row[0]),
"chat_id": str(row[1]),
"owner_id": str(row[2]),
"filename": row[3],
"content_type": row[4],
"size_bytes": row[5],
"sha256": row[6],
"status": "ready",
"variants": [],
"error_code": None,
"created_at": created,
"updated_at": created,
}
+8
View File
@@ -0,0 +1,8 @@
"""Ошибки API. Текст не содержит SQL, паролей и токенов."""
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str):
self.status = status
self.code = code
self.message = message
+74
View File
@@ -0,0 +1,74 @@
"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей."""
from __future__ import annotations
from errors import ApiError
PNG_SIG = b"\x89PNG\r\n\x1a\n"
# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC.
_JPEG_SOF = {
0xC0, 0xC1, 0xC2, 0xC3,
0xC5, 0xC6, 0xC7,
0xC9, 0xCA, 0xCB,
0xCD, 0xCE, 0xCF,
}
def sniff(data: bytes) -> str:
if data.startswith(PNG_SIG):
return "image/png"
if data.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
def dimensions(data: bytes, content_type: str) -> tuple[int, int]:
if content_type == "image/png":
return _png_size(data)
if content_type == "image/jpeg":
return _jpeg_size(data)
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
def _png_size(data: bytes) -> tuple[int, int]:
if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR":
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
if width < 1 or height < 1:
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
return width, height
def _jpeg_size(data: bytes) -> tuple[int, int]:
if len(data) < 4 or data[:2] != b"\xff\xd8":
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
index = 2
size = len(data)
while index < size:
if data[index] != 0xFF:
break
while index < size and data[index] == 0xFF:
index += 1
if index >= size:
break
marker = data[index]
index += 1
if marker in (0xD8, 0xD9) or marker == 0xDA:
break
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
continue
if index + 2 > size:
break
segment = int.from_bytes(data[index:index + 2], "big")
if segment < 2 or index + segment > size:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
if marker in _JPEG_SOF:
if segment < 7:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
height = int.from_bytes(data[index + 3:index + 5], "big")
width = int.from_bytes(data[index + 5:index + 7], "big")
if width < 1 or height < 1:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
return width, height
index += segment
raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров")
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env python3
"""Однократные миграции. Реплики API этот файл не запускают."""
import sys
import db
def main() -> None:
db.migrate_with_retry()
sys.stderr.write("migrations applied\n")
if __name__ == "__main__":
main()
+43
View File
@@ -0,0 +1,43 @@
"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах."""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
N = 2**14
R = 8
P = 1
DKLEN = 32
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
digest = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN,
maxmem=64 * 1024 * 1024)
return "scrypt$%d$%d$%d$%s$%s" % (
N, R, P,
base64.b64encode(salt).decode("ascii"),
base64.b64encode(digest).decode("ascii"),
)
def verify_password(password: str, stored: str) -> bool:
try:
algo, n, r, p, salt_b64, digest_b64 = stored.split("$")
if algo != "scrypt":
return False
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(digest_b64)
actual = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p),
dklen=len(expected), maxmem=64 * 1024 * 1024)
except (ValueError, TypeError):
return False
return hmac.compare_digest(actual, expected)
# Одинаковая стоимость проверки, когда пользователя нет.
DUMMY_HASH = hash_password("dummy-password-not-used")
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Однократное создание приватного bucket. Повторный запуск ничего не публикует."""
import sys
import time
import storage
def main() -> None:
storage.init()
last = None
for _ in range(30):
try:
storage.ensure_bucket()
sys.stderr.write("bucket ready\n")
return
except Exception as exc:
last = exc
time.sleep(1)
sys.stderr.write("bucket bootstrap failed\n")
raise SystemExit(1) from last
if __name__ == "__main__":
main()
+701
View File
@@ -0,0 +1,701 @@
#!/usr/bin/env python3
"""Messenger lab4. Реплики за Traefik, метаданные в PostgreSQL, байты в S3."""
from __future__ import annotations
import base64
import hashlib
import json
import os
import re
import secrets
import signal
import socket
import socketserver
import sys
import threading
import time
import unicodedata
import uuid
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler
from urllib.parse import parse_qs, urlparse
import db
import images
import passwords
import sessions
import storage
from errors import ApiError
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
APP_ENV = os.environ.get("APP_ENV", "lab")
INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z")
UUID_RE = re.compile(
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z")
TIMESTAMP_RE = re.compile(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z")
MAX_BODY = 1024 * 1024
_draining = threading.Event()
_inflight = 0
_inflight_lock = threading.Lock()
def drain_seconds() -> float:
raw = os.environ.get("DRAIN_SECONDS", "3")
try:
value = float(raw)
except ValueError:
return 3.0
if value < 0:
return 0.0
if value > 6:
return 6.0
return value
def now_utc() -> datetime:
current = datetime.now(timezone.utc)
return current.replace(microsecond=(current.microsecond // 1000) * 1000)
def is_blank(value: str) -> bool:
return value.strip() == ""
def require_object(body):
if not isinstance(body, dict):
raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом")
def reject_extra(body: dict, allowed: set):
extra = set(body) - allowed
if extra:
raise ApiError(400, "invalid_request",
"Лишние поля: " + ", ".join(sorted(extra)))
def parse_uuid(value, field: str) -> str:
if not isinstance(value, str) or not UUID_RE.match(value):
raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}")
return value.lower()
def parse_display_name(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "display_name должен быть строкой")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "display_name: 1..100 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "display_name не может быть пустым")
return value
def parse_username(value) -> str:
if not isinstance(value, str) or not USERNAME_RE.match(value):
raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$")
return value
def parse_password(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "password должен быть строкой")
if not (12 <= len(value) <= 128):
raise ApiError(400, "invalid_request", "password: 12..128 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "password не может быть пустым")
return value
def parse_title(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "title должен быть строкой")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "title: 1..100 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "title не может быть пустым")
return value
def max_upload_bytes() -> int:
raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760")
try:
value = int(raw)
except ValueError:
return 10485760
return value if value >= 1 else 10485760
def max_image_pixels() -> int:
raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000")
try:
value = int(raw)
except ValueError:
return 40000000
return value if value >= 1 else 40000000
def parse_text(value, has_attachments: bool) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "text должен быть строкой")
if len(value) > 4000:
raise ApiError(400, "invalid_request", "text: не больше 4000 code points")
if is_blank(value) and not has_attachments:
raise ApiError(400, "invalid_request", "text не может быть пустым без вложений")
return value
def parse_attachment_ids(value) -> list:
if value is None:
return []
if not isinstance(value, list):
raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом")
if len(value) > 10:
raise ApiError(400, "invalid_request", "attachment_ids: не больше 10")
seen = set()
result = []
for item in value:
att_id = parse_uuid(item, "attachment_ids[]")
if att_id in seen:
raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты")
seen.add(att_id)
result.append(att_id)
return result
def parse_filename(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
if not (1 <= len(value) <= 255):
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
for char in value:
if char in "/\\" or unicodedata.category(char) == "Cc":
raise ApiError(400, "invalid_request", "filename содержит запрещённый символ")
return value
def parse_member_ids(value) -> list:
if not isinstance(value, list):
raise ApiError(400, "invalid_request", "member_ids должен быть массивом")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "member_ids: 1..100")
seen = set()
result = []
for item in value:
uid = parse_uuid(item, "member_ids[]")
if uid in seen:
raise ApiError(400, "invalid_request", "member_ids содержит дубликаты")
seen.add(uid)
result.append(uid)
return result
def parse_limit(raw, default=50) -> int:
if raw is None:
return default
if isinstance(raw, str) and raw.strip() == "":
raise ApiError(400, "invalid_request", "limit должен быть целым")
try:
value = int(raw)
except (TypeError, ValueError):
raise ApiError(400, "invalid_request", "limit должен быть целым")
if not (1 <= value <= 100):
raise ApiError(400, "invalid_request", "limit: 1..100")
return value
def parse_optional_cursor(raw):
if raw is None:
return None
if not isinstance(raw, str) or raw.strip() == "":
raise ApiError(400, "invalid_request", "Некорректный cursor")
return raw
def parse_cursor_timestamp(value: str) -> datetime:
if not isinstance(value, str) or not TIMESTAMP_RE.match(value):
raise ApiError(400, "invalid_request", "Некорректный cursor")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
return parsed
def parse_idempotency_key(headers) -> str:
raw = headers.get("Idempotency-Key")
if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw):
raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key")
return raw
def canonical_hash(text: str, attachment_ids: list) -> str:
# Порядок ключей JSON не важен. Отсутствующий attachment_ids равен [].
# Порядок элементов массива сохраняется: это часть тела запроса.
payload = {"attachment_ids": attachment_ids, "text": text}
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def challenge_for(path: str) -> str:
if path == "/api/v1/auth/sessions":
return 'Basic realm="messenger", charset="UTF-8"'
return 'Bearer realm="messenger"'
def bearer_token(headers) -> str:
auth = headers.get("Authorization") or ""
parts = auth.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
raise ApiError(401, "unauthorized", "Требуется Bearer")
return parts[1].strip()
def current_user(headers) -> dict:
token = bearer_token(headers)
user = db.get_user(sessions.user_id(token))
if user is None:
raise ApiError(401, "unauthorized", "Сессия недействительна")
return user
def basic_credentials(headers) -> tuple[str, str]:
auth = headers.get("Authorization") or ""
parts = auth.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip():
raise ApiError(401, "unauthorized", "Требуется Basic")
try:
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
except (ValueError, UnicodeDecodeError) as exc:
raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc
if ":" not in decoded:
raise ApiError(401, "unauthorized", "Неверные учётные данные")
username, password = decoded.split(":", 1)
return username, password
def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str:
raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(cursor: str, chat_id: str):
pad = "=" * (-len(cursor) % 4)
try:
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
except Exception as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
parts = raw.split("|")
if len(parts) != 3:
raise ApiError(400, "invalid_request", "Некорректный cursor")
c_chat, c_created, c_id = parts
if not UUID_RE.match(c_chat):
raise ApiError(400, "invalid_request", "Некорректный cursor")
if c_chat.lower() != chat_id:
raise ApiError(400, "invalid_request", "cursor от другого чата")
return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor")
def encode_chat_cursor(created_at: str, chat_id: str) -> str:
raw = f"chat|{created_at}|{chat_id}".encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_chat_cursor(cursor: str):
pad = "=" * (-len(cursor) % 4)
try:
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
except Exception as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
parts = raw.split("|")
if len(parts) != 3 or parts[0] != "chat":
raise ApiError(400, "invalid_request", "Некорректный cursor")
return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor")
def create_user(body) -> dict:
require_object(body)
reject_extra(body, {"username", "display_name", "password"})
username = parse_username(body.get("username"))
display_name = parse_display_name(body.get("display_name"))
password = parse_password(body.get("password"))
return db.create_user(username, display_name, passwords.hash_password(password), now_utc())
def create_session(headers) -> dict:
username, password = basic_credentials(headers)
row = None
try:
row = db.user_for_login(username)
except ApiError:
raise
stored = row[3] if row is not None else passwords.DUMMY_HASH
password_ok = passwords.verify_password(password, stored)
if row is None or not password_ok:
raise ApiError(401, "unauthorized", "Неверные учётные данные")
token = secrets.token_urlsafe(32)
ttl = sessions.session_ttl()
expires = now_utc() + timedelta(seconds=ttl)
sessions.create(token, str(row[0]), ttl)
return {
"token": token,
"token_type": "Bearer",
"expires_at": db.format_ts(expires),
"user": db.user_json((row[0], row[1], row[2], row[4])),
}
def logout(headers) -> None:
token = bearer_token(headers)
sessions.user_id(token)
sessions.revoke(token)
def create_chat(body, actor: dict) -> dict:
require_object(body)
reject_extra(body, {"title", "member_ids"})
title = parse_title(body.get("title"))
member_ids = parse_member_ids(body.get("member_ids"))
if actor["id"] not in member_ids:
raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids")
return db.create_chat(title, member_ids, now_utc())
def list_chats(actor: dict, limit: int, cursor) -> dict:
decoded = decode_chat_cursor(cursor) if cursor else None
page = db.list_chats(actor["id"], limit, decoded)
next_cursor = None
if page["next_cursor_parts"] is not None:
created, chat_id = page["next_cursor_parts"]
next_cursor = encode_chat_cursor(db.format_ts(created), chat_id)
return {"items": page["items"], "next_cursor": next_cursor}
def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict:
cid = parse_uuid(chat_id, "chat_id")
require_object(body)
reject_extra(body, {"text", "attachment_ids"})
attachment_ids = parse_attachment_ids(body.get("attachment_ids"))
text = parse_text(body.get("text"), bool(attachment_ids))
path = "/api/v1/chats/%s/messages" % cid
return db.create_message_idempotent(
cid,
actor["id"],
text,
attachment_ids,
now_utc(),
"POST",
path,
idem_key,
canonical_hash(text, attachment_ids),
)
def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict:
detected = images.sniff(data)
if detected != declared:
raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type")
width, height = images.dimensions(data, detected)
if width * height > max_image_pixels():
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
att_id = str(uuid.uuid4())
object_key = "o/" + att_id
digest = hashlib.sha256(data).hexdigest()
created = now_utc()
storage.put_bytes(object_key, data, detected)
try:
return db.insert_attachment(
att_id, chat_id, actor["id"], filename, detected, len(data), digest,
object_key, width, height, created,
)
except Exception:
storage.delete_quiet(object_key)
raise
def read_attachment(attachment_id: str, actor: dict) -> dict:
return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"])
def read_attachment_bytes(attachment_id: str, actor: dict, variant: str):
record = db.get_attachment_record(parse_uuid(attachment_id, "attachment_id"), actor["id"])
if variant != "original":
raise ApiError(404, "not_found", "Вариант не найден")
data = storage.get_bytes(record["object_key"])
if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными")
return record["content_type"], data
def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict:
cid = parse_uuid(chat_id, "chat_id")
decoded = decode_cursor(cursor, cid) if cursor else None
page = db.list_messages(cid, actor["id"], limit, decoded)
next_cursor = None
if page["next_cursor_parts"] is not None:
created, message_id = page["next_cursor_parts"]
next_cursor = encode_cursor(cid, db.format_ts(created), message_id)
return {"items": page["items"], "next_cursor": next_cursor}
class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
daemon_threads = True
block_on_close = False
allow_reuse_address = True
request_queue_size = 256
class Handler(BaseHTTPRequestHandler):
server_version = "mtusi-messenger/4.0"
protocol_version = "HTTP/1.0"
def log_message(self, fmt, *args):
return
def _request_id(self) -> str:
return self.headers.get("X-Request-Id") or str(uuid.uuid4())
def _send_json(self, status: int, payload, request_id: str, extra_headers=None):
body = b""
if status != 204:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
if status != 204:
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("Connection", "close")
for key, value in (extra_headers or {}).items():
self.send_header(key, value)
self.end_headers()
if body:
self.wfile.write(body)
def _send_bytes(self, content_type: str, payload: bytes, request_id: str):
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(payload)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("Connection", "close")
self.end_headers()
if payload:
self.wfile.write(payload)
def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
def _read_body(self):
length = self.headers.get("Content-Length")
if not length:
return None
try:
size = int(length)
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
if size > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело слишком большое")
raw = self.rfile.read(size)
ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip()
if ctype and ctype != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
if not raw:
raise ApiError(400, "invalid_request", "Пустое тело")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ApiError(400, "invalid_request", "Malformed JSON") from exc
def _read_upload(self) -> tuple[str, bytes]:
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Нужен Content-Length")
try:
size = int(length)
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
if size > max_upload_bytes():
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
if size < 1:
raise ApiError(415, "unsupported_media_type", "Пустое тело")
declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
if declared not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
data = self.rfile.read(size)
if len(data) != size:
raise ApiError(400, "invalid_request", "Тело короче Content-Length")
return declared, data
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
global _inflight
with _inflight_lock:
_inflight += 1
rid = self._request_id()
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
query = parse_qs(parsed.query, keep_blank_values=True)
try:
self._route(method, path, query, rid)
except ApiError as exc:
headers = {}
if exc.status == 401:
headers["WWW-Authenticate"] = challenge_for(path)
self._send_error(exc.status, exc.code, exc.message, rid, headers)
except BrokenPipeError:
pass
except Exception:
sys.stderr.write("internal error on %s %s\n" % (method, path))
self._send_error(500, "internal_error", "Внутренняя ошибка", rid)
finally:
with _inflight_lock:
_inflight -= 1
def _route(self, method: str, path: str, query: dict, rid: str):
if method == "GET" and path == "/health/live":
return self._send_json(200, {"status": "ok"}, rid)
if method == "GET" and path == "/health/ready":
if _draining.is_set():
raise ApiError(503, "unavailable", "Реплика останавливается")
if db.ping() and sessions.ping() and storage.ping():
return self._send_json(200, {"status": "ok"}, rid)
raise ApiError(503, "unavailable", "Зависимость недоступна")
if _draining.is_set():
raise ApiError(503, "unavailable", "Реплика останавливается")
if method == "POST" and path == "/api/v1/users":
return self._send_json(201, create_user(self._read_body()), rid)
if method == "POST" and path == "/api/v1/auth/sessions":
return self._send_json(201, create_session(self.headers), rid)
if method == "DELETE" and path == "/api/v1/auth/sessions/current":
logout(self.headers)
return self._send_json(204, None, rid)
if method == "GET" and path == "/api/v1/users/me":
return self._send_json(200, current_user(self.headers), rid)
if method == "POST" and path == "/api/v1/chats":
actor = current_user(self.headers)
return self._send_json(201, create_chat(self._read_body(), actor), rid)
if method == "GET" and path == "/api/v1/chats":
actor = current_user(self.headers)
limit = parse_limit(query.get("limit", [None])[0])
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
return self._send_json(200, list_chats(actor, limit, cursor), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)", path)
if match and method == "GET":
actor = current_user(self.headers)
return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path)
if match and method == "POST":
actor = current_user(self.headers)
chat_id = parse_uuid(match.group(1), "chat_id")
if not db.is_member(chat_id, actor["id"]):
raise ApiError(404, "not_found", "Чат не найден")
names = query.get("filename") or []
if len(names) != 1:
raise ApiError(400, "invalid_request", "Нужен один filename")
filename = parse_filename(names[0])
declared, data = self._read_upload()
body = upload_attachment(chat_id, actor, filename, declared, data)
return self._send_json(201, body, rid)
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path)
if match and method == "GET":
actor = current_user(self.headers)
variants = query.get("variant") or ["original"]
if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "":
raise ApiError(400, "invalid_request", "Некорректный variant")
content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0])
return self._send_bytes(content_type, payload, rid)
match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path)
if match and method == "GET":
actor = current_user(self.headers)
return self._send_json(200, read_attachment(match.group(1), actor), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path)
if match and method == "POST":
actor = current_user(self.headers)
idem_key = parse_idempotency_key(self.headers)
body = create_message(match.group(1), self._read_body(), actor, idem_key)
return self._send_json(201, body, rid)
if match and method == "GET":
actor = current_user(self.headers)
limit = parse_limit(query.get("limit", [None])[0])
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid)
raise ApiError(404, "not_found", "Маршрут не найден")
def _inflight_now() -> int:
with _inflight_lock:
return _inflight
def main():
try:
db.init()
except Exception:
sys.stderr.write("database is not configured\n")
try:
sessions.init()
except Exception:
sys.stderr.write("session store is not configured\n")
try:
storage.init()
except Exception:
sys.stderr.write("object storage is not configured\n")
server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler)
def drain_and_stop():
time.sleep(drain_seconds())
deadline = time.monotonic() + 2
while time.monotonic() < deadline and _inflight_now() > 0:
time.sleep(0.05)
server.shutdown()
def shutdown(signum, frame):
if _draining.is_set():
return
_draining.set()
sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID)
threading.Thread(target=drain_and_stop, daemon=True).start()
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
sys.stderr.write(
"listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID)
)
try:
server.serve_forever(poll_interval=0.1)
finally:
server.server_close()
db.close()
sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID)
if __name__ == "__main__":
main()
+85
View File
@@ -0,0 +1,85 @@
"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии.
Источник истины сессий — общий Valkey, не память процесса API.
AOF и volume сохраняют ключ при перезапуске хранилища.
"""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import redis
from errors import ApiError
_client: redis.Redis | None = None
def session_ttl() -> int:
raw = os.environ.get("SESSION_TTL_SECONDS", "3600")
try:
value = int(raw)
except ValueError:
return 3600
return value if value >= 1 else 1
def redis_url() -> str:
path = os.environ.get("CACHE_URL_FILE", "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get("REDIS_URL", "").strip()
def init() -> None:
global _client
url = redis_url()
if not url:
_client = None
return
_client = redis.Redis.from_url(
url, socket_connect_timeout=0.5, socket_timeout=0.5,
decode_responses=True)
def _key(token: str) -> str:
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()
return "session:" + digest
def _require() -> redis.Redis:
if _client is None:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно")
return _client
def ping() -> bool:
try:
return bool(_require().ping())
except (ApiError, redis.RedisError):
return False
def create(token: str, user_id: str, ttl: int) -> None:
try:
_require().set(_key(token), user_id, ex=ttl)
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
def user_id(token: str) -> str:
try:
value = _require().get(_key(token))
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
if not value:
raise ApiError(401, "unauthorized", "Сессия недействительна")
return value
def revoke(token: str) -> None:
try:
_require().delete(_key(token))
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
+110
View File
@@ -0,0 +1,110 @@
"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент."""
from __future__ import annotations
import io
import os
import sys
from pathlib import Path
from minio import Minio
from errors import ApiError
_client: Minio | None = None
def _read_setting(env_name: str, file_env: str) -> str:
path = os.environ.get(file_env, "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get(env_name, "").strip()
def endpoint() -> str:
return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
def bucket() -> str:
return os.environ.get("S3_BUCKET", "messenger-attachments").strip()
def _host_and_secure() -> tuple[str, bool]:
raw = endpoint()
secure = raw.startswith("https://")
host = raw.split("://", 1)[-1].rstrip("/")
return host, secure
def init() -> None:
global _client
host, secure = _host_and_secure()
access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
if not host or not access or not secret or not bucket():
raise RuntimeError("object storage is not configured")
_client = Minio(
host,
access_key=access,
secret_key=secret,
secure=secure,
region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
)
def _api() -> Minio:
if _client is None:
raise ApiError(503, "unavailable", "Хранилище недоступно")
return _client
def ping() -> bool:
try:
return _api().bucket_exists(bucket())
except Exception:
return False
def ensure_bucket() -> None:
api = _api()
name = bucket()
if not api.bucket_exists(name):
api.make_bucket(name)
def put_bytes(key: str, data: bytes, content_type: str) -> None:
try:
_api().put_object(
bucket(),
key,
io.BytesIO(data),
length=len(data),
content_type=content_type,
)
except ApiError:
raise
except Exception as exc:
sys.stderr.write("s3 put failed\n")
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
def get_bytes(key: str) -> bytes:
response = None
try:
response = _api().get_object(bucket(), key)
return response.read()
except ApiError:
raise
except Exception as exc:
sys.stderr.write("s3 get failed\n")
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
finally:
if response is not None:
response.close()
response.release_conn()
def delete_quiet(key: str) -> None:
try:
_api().remove_object(bucket(), key)
except Exception:
sys.stderr.write("s3 delete failed\n")