Compare commits

3 Commits
Author SHA1 Message Date
romashko 8f951e7bbc report.md 2026-10-30 20:24:13 +03:00
romashko 959eee7f9d Grade 4 2026-10-30 20:08:17 +03:00
romashko 4c7a796795 Add third lab code 2026-10-24 01:03:18 +03:00
53 changed files with 2153 additions and 52 deletions
+13 -16
View File
@@ -1,20 +1,17 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env. # Copy to .env and choose local credentials. Never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080 APP_PORT=8080
APP_ENV=lab POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace-with-local-password
SESSION_TTL_SECONDS=3600 SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values. CACHE_TTL_SECONDS=30
DATABASE_URL_FILE=/run/secrets/database_url MAX_REQUEST_BYTES=65536
# Optional on 3, required use on 4 in lab2; Redis OR Valkey: MAX_INFLIGHT=32
SESSION_STORE=postgres MINIO_ROOT_USER=local-admin
CACHE_URL_FILE=/run/secrets/cache_url MINIO_ROOT_PASSWORD=replace-with-admin-password
API_REPLICAS=2 S3_ACCESS_KEY_ID=messenger-api
IDEMPOTENCY_TTL_SECONDS=86400 S3_SECRET_ACCESS_KEY=replace-with-api-password
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760 MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000 MAX_IMAGE_PIXELS=40000000
# Compose sets internal PostgreSQL, Redis and S3 addresses.
# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init.
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 4 ## Лабораторная 4
Автор / группа: TODO Автор / группа: не указаны
Целевая оценка (3/4/5): TODO Целевая оценка (3/4/5): 4
Трек 5, если заявлен: TODO Трек 5, если заявлен: не заявлен
Что реализовано и что перенесено: TODO Что реализовано и что перенесено: перенесены PostgreSQL, Redis, Nginx и две API-реплики из lab3; добавлены private MinIO bucket, Attachment, upload/download через API, миграция, очистка orphan-объектов и проверки вложений.
Команды запуска из чистого clone: TODO Команды запуска из чистого clone: `cp .env.example .env`, задать локальные пароли, затем `docker compose --env-file .env up -d --build` и `make test`.
Результаты проверок и ссылка на REPORT.md: TODO Результаты проверок и ссылка на REPORT.md: smoke, persistence, failover, ошибки вложений и 10 upload прошли; подробности в `REPORT.md`.
Ограничения / известные ошибки: TODO Ограничения / известные ошибки: S3, PostgreSQL и Redis по одному экземпляру; cleanup удаляет только orphan-объекты старше одного часа.
Источники и использование ИИ: TODO Источники и использование ИИ: FastAPI, SQLAlchemy, boto3, Pillow, MinIO; Codex использовался для реализации и проверок.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены. - [x] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод. - [x] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. - [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. - [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. - [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.12.8-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY contracts ./contracts
COPY src ./src
COPY migrations ./migrations
COPY tests ./tests
COPY student-tests ./student-tests
COPY scripts ./scripts
COPY lab.json .
RUN useradd --create-home --shell /usr/sbin/nologin appuser
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
STOPSIGNAL SIGTERM
CMD ["python", "-m", "src"]
+4 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?= ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover .PHONY: help check validate test load upload-load test-persistence test-failover
help: help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -21,6 +21,9 @@ test:
load: load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
upload-load:
$(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)"
test-persistence: test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+25 -21
View File
@@ -1,37 +1,41 @@
# Отчёт — лабораторная 4 # Отчёт — лабораторная 4
- Автор, группа: TODO - Автор, группа: не указаны
- Целевая оценка; выбранный трек 5, если нужен: TODO - Целевая оценка: 4
- Commit/PR: TODO после публикации - Commit/PR: не опубликованы
- Использованные источники, библиотеки и помощь ИИ: TODO - Использованные библиотеки и помощь ИИ: FastAPI, SQLAlchemy, Redis, boto3, Pillow, MinIO; Codex использовался для реализации и проверок.
## Запуск из чистого clone ## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. ```sh
cp .env.example .env
# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD и S3_SECRET_ACCESS_KEY.
docker compose --env-file .env up -d --build
make test
python student-tests/test_attachments.py
python scripts/upload_load.py --concurrency 10 --output evidence/upload-load.json
```
Стенд запускает PostgreSQL, Redis, MinIO, миграции, две API-реплики и Nginx. Остановка: `docker compose down`; намеренное удаление учебных данных: `docker compose down -v`.
## Архитектура и решения ## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. Из lab3 перенесены PostgreSQL, Redis-сессии, Nginx и две API-реплики. Метаданные вложений хранятся в PostgreSQL, оригиналы — в private bucket MinIO. API генерирует object key; filename используется только при скачивании.
Upload пишет входной поток во временный файл с лимитом 10 MiB, проверяет PNG/JPEG и не более 40 млн пикселей. Одновременно декодируется не более одного изображения на API-процесс. Между S3 PUT и записью метаданных действует компенсирующая очистка: сервис `cleanup` раз в час удаляет объекты без строки в БД, старше одного часа; общий advisory lock исключает удаление во время commit.
## Доказательства критериев ## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO | | Публичный smoke | `make test` | PASS: lab4 public API scenario | console |
| Остальные пункты уровня 3 | TODO | TODO | TODO | | Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | прежняя сессия, сообщение и оригинал сохранены | console |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | | Failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | PASS; восстановление записи за 6.361 с, файл читается через другую API-реплику | console |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO | | Ошибки вложений | `python student-tests/test_attachments.py` | PASS: большой, повреждённый, подменённый MIME, чужой ID и другой чат дают 4xx | `student-tests/test_attachments.py` |
| 10 upload | `python scripts/upload_load.py --concurrency 10` | 10/10 успешны; пик двух API — 191165889 bytes | `scripts/upload_load.py` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. | Private bucket | `docker compose exec -T nginx wget -S -O /dev/null http://s3:9000/messenger-attachments` | unauthenticated GET возвращает 403 | console |
| S3 unavailable | pause S3, `GET /health/ready`, unpause S3 | через LB: 503; после восстановления: 200 | console |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги ## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. MinIO, PostgreSQL и Redis запущены по одному экземпляру. Cleanup предназначен для orphan-объектов; он не восстанавливает отсутствующие объекты для существующих метаданных. Административная консоль MinIO не публикуется в сеть.
+175
View File
@@ -0,0 +1,175 @@
x-api: &api
image: mtusi2026lab4-api:local
build: .
dns_opt:
- timeout:1
- attempts:1
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
POSTGRES_HOST: postgres
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
networks: [backend]
cpus: 1.0
mem_limit: 512m
stop_grace_period: 10s
services:
api:
<<: *api
deploy:
replicas: 2
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-init:
condition: service_completed_successfully
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
interval: 2s
timeout: 2s
start_period: 5s
retries: 2
migrate:
<<: *api
command: [python, -m, src.database]
healthcheck:
disable: true
restart: 'no'
depends_on:
postgres:
condition: service_healthy
s3:
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
command: [server, /data, --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
MINIO_BROWSER: 'off'
volumes:
- s3_data:/data
networks: [backend]
healthcheck:
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
s3-init:
# The pinned server image also contains mc; no second image is needed.
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
entrypoint: [sh, /bootstrap.sh]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
volumes:
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
- ./infra/s3-policy.json:/policy.json:ro
depends_on:
s3:
condition: service_healthy
networks: [backend]
restart: 'no'
cpus: 0.25
mem_limit: 128m
cleanup:
<<: *api
command: [python, -m, src.cleanup]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
nginx:
image: nginx:1.28.0-alpine
entrypoint: [sh, /etc/nginx/readiness.sh]
dns_opt:
- timeout:1
- attempts:1
ports:
- '127.0.0.1:${APP_PORT:-8080}:8080'
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/readiness.sh:/etc/nginx/readiness.sh:ro
networks: [edge, backend]
depends_on:
api:
condition: service_healthy
healthcheck:
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
interval: 2s
timeout: 2s
retries: 3
cpus: 0.5
mem_limit: 128m
stop_grace_period: 10s
postgres:
image: postgres:17.4-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
volumes:
- postgres_data:/var/lib/postgresql/data
networks: [backend]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
redis:
image: redis:7.4.2-bookworm
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
volumes:
- redis_data:/data
networks: [backend]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 256m
networks:
edge:
backend:
internal: true
volumes:
postgres_data:
redis_data:
s3_data:
+34
View File
@@ -0,0 +1,34 @@
# Перенесённая основа lab3
На этом этапе перенесена только работающая основа: FastAPI, PostgreSQL,
миграции 001–003, сессии и кеш Redis, идемпотентность сообщений, две API-реплики
и Nginx с проверкой готовности. S3 и вложения ещё не реализованы.
Контракт и публичные тесты остаются от lab4; полный smoke lab4 пока не пройдёт.
## Запуск основы
Скопируйте `.env.example` в `.env` и задайте свой `POSTGRES_PASSWORD`.
Параметры `*_FILE` и S3 пока являются настройками будущей реализации;
текущая основа использует `POSTGRES_*` и Redis из Compose.
```sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
```
API доступен через http://localhost:8080. При одновременно запущенной lab3
укажите другой `APP_PORT` в `.env`. Имя образа lab4 отдельное, volumes
изолируются именем Compose-проекта. Миграции выполняет сервис `migrate`.
Проверка перенесённой логики после запуска:
```sh
docker compose exec -T api python -m unittest discover -s student-tests -v
```
В `student-tests` перенесены проверки базового API/сессий и идемпотентности.
Эксперименты масштабирования, старые результаты и отчёт lab3 не переносились.
Скрипты `scripts/restart.*` и `scripts/stop-one.*` работают с текущим проектом;
для остановки реплики требуется `TARGET_INSTANCE_ID` из `X-Instance-Id`.
При добавлении S3 потребуется включить его в сценарий перезапуска и адаптировать
лимиты тела запроса в API и Nginx для загрузки изображений.
+38
View File
@@ -0,0 +1,38 @@
worker_processes 1;
worker_shutdown_timeout 8s;
events { worker_connections 1024; }
http {
upstream messenger {
zone messenger 64k;
include /etc/nginx/ready-backends.conf;
}
log_format upstream '$request_method $uri $status upstream=$upstream_addr '
'upstream_status=$upstream_status time=$request_time';
access_log /var/log/nginx/access.log upstream;
server {
listen 8080;
# API enforces the byte limit and returns the contracted JSON error.
client_max_body_size 0;
client_body_timeout 5s;
client_header_timeout 5s;
keepalive_timeout 5s;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_request_buffering off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header Connection close;
proxy_connect_timeout 1s;
proxy_send_timeout 35s;
proxy_read_timeout 120s;
# A 503 is an API response (for example, S3 is unavailable), so
# preserve it for the client instead of marking every replica down.
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 8s;
# Never transparently retry a POST already sent to an API.
# A client retries messages using the original Idempotency-Key.
}
}
}
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
# Only ready API addresses enter the upstream, including at nginx startup.
update_backends() {
next=/etc/nginx/ready-backends.conf.next
: > "$next"
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
echo "server $address:8080;" >> "$next"
fi
done
# If every running API temporarily loses a shared dependency (for example
# S3), retain the last addresses. The API can then return the contracted
# 503 instead of Nginx replacing it with a 502. At initial startup there
# is no previous healthy set, so use an inert placeholder.
if [ ! -s "$next" ]; then
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
rm -f "$next"
return
fi
echo 'server 127.0.0.1:9 down;' > "$next"
fi
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
mv "$next" /etc/nginx/ready-backends.conf
if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then
nginx -s reload
fi
else
rm -f "$next"
fi
}
update_backends
(
while sleep 2; do
update_backends reload
done
) &
exec nginx -g 'daemon off;'
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
mc mb --ignore-existing local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
}
]
}
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
CREATE INDEX chats_order_idx ON chats (created_at, id);
+1
View File
@@ -0,0 +1 @@
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
+9
View File
@@ -0,0 +1,9 @@
-- Additive migration: no rewrite of existing lab2 tables or messages.
CREATE TABLE message_idempotency (
actor_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (actor_id, chat_id, key)
);
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
object_key text NOT NULL UNIQUE,
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
+19
View File
@@ -0,0 +1,19 @@
fastapi==0.115.6
uvicorn==0.34.0
pydantic==2.13.5
starlette==0.41.3
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
colorama==0.4.6; sys_platform == "win32"
h11==0.16.0
idna==3.20
pydantic_core==2.46.5
typing-inspection==0.4.4
typing_extensions==4.16.0
psycopg[binary]==3.2.6
SQLAlchemy==2.0.41
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.70
Pillow==12.0.0
+79
View File
@@ -0,0 +1,79 @@
"""Project-scoped Docker operations; never read/export container environments."""
import json
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, timeout=120).strip()
def compose(*args):
return docker('compose', *args)
def snapshot(service=None):
ids = compose('ps', '-a', '-q', *([service] if service else [])).split()
if not ids:
return []
# Select fields before returning; no secrets in evidence.
template = ('{"id":{{json .Id}},"name":{{json .Name}},"instance":{{json .Config.Hostname}},'
'"image":{{json .Image}},"state":{{json .State.Status}},'
'"started":{{json .State.StartedAt}},'
'"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}},'
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
'"ports":{{json .HostConfig.PortBindings}}}')
return [json.loads(line) for line in docker('inspect', '--format', template, *ids).splitlines()]
def wait_api(count=2, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
rows = snapshot('api')
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
time.sleep(3) # Allow the nginx readiness poll (2s) and reload.
return rows
time.sleep(0.5)
raise RuntimeError('API replicas did not become healthy')
def save(name, value):
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
def stop_instance(instance):
before = snapshot('api')
matches = [r for r in before if r['instance'] == instance and r['state'] == 'running']
if len(matches) != 1 or sum(r['state'] == 'running' for r in before) < 2:
raise RuntimeError('Target must identify one running API with a running peer')
docker('stop', '--time', '8', matches[0]['id'])
result = {'target': instance, 'before': before, 'after': snapshot('api')}
if next(r for r in result['after'] if r['instance'] == instance)['state'] != 'exited':
raise RuntimeError('Target was not stopped')
return result
def restore():
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
return wait_api()
def rolling():
before = wait_api()
if len(before) != 2:
raise RuntimeError('This rolling scenario requires exactly two replicas')
steps = []
for old in before:
stop_instance(old['instance'])
# Only remove the selected, already stopped API; never remove a volume.
docker('rm', old['id'])
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
steps.append({'replaced': old['id'], 'ready': wait_api()})
result = {'before': before, 'steps': steps, 'after': snapshot('api')}
if {r['id'] for r in before} & {r['id'] for r in result['after']}:
raise RuntimeError('Not all original API containers were replaced')
return result
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
python scripts\restart.py
+28
View File
@@ -0,0 +1,28 @@
"""Restart the whole lab stack, retaining volumes and recording actual start times."""
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def snapshot():
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
return subprocess.check_output(
['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids],
cwd=ROOT, text=True,
)
def main():
before = snapshot()
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'],
cwd=ROOT, check=True, timeout=60)
after = snapshot()
(ROOT / 'evidence').mkdir(exist_ok=True)
(ROOT / 'evidence' / 'restart.txt').write_text(
'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before
+ 'After (UTC):\n' + after, encoding='utf-8')
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/restart.py
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
python scripts\stop-one.py
+5
View File
@@ -0,0 +1,5 @@
import os
from ops import stop_instance
if __name__ == '__main__':
stop_instance(os.environ['TARGET_INSTANCE_ID'])
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+71
View File
@@ -0,0 +1,71 @@
"""Upload one fixture concurrently and record the peak RSS of API containers."""
import argparse
import json
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def api_memory_bytes():
text = subprocess.check_output(
['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'],
cwd=ROOT, text=True,
)
values = []
for line in text.splitlines():
amount = line.split()[1]
number, unit = float(amount[:-3]), amount[-3:].lower()
values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit]))
return sum(values)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--output', default='evidence/upload-load.json')
args = parser.parse_args()
check(args.concurrency >= 10, 'concurrency must be at least 10')
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
start = threading.Event()
done = threading.Event()
peak = [0]
def monitor():
while not done.is_set():
peak[0] = max(peak[0], api_memory_bytes())
time.sleep(.2)
def upload(index):
start.wait()
return Client(args.base_url, timeout=60).call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index,
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
watcher = threading.Thread(target=monitor, daemon=True)
watcher.start()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
jobs = [pool.submit(upload, item) for item in range(args.concurrency)]
start.set()
results = [job.result() for job in jobs]
done.set()
watcher.join(timeout=1)
report = {'concurrency': args.concurrency, 'fixture_bytes': len(content),
'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]}
destination = ROOT / args.output
destination.parent.mkdir(exist_ok=True)
destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps(report, ensure_ascii=False))
if __name__ == '__main__':
main()
View File
+13
View File
@@ -0,0 +1,13 @@
import os
import uvicorn
if __name__ == '__main__':
uvicorn.run(
'src.main:app',
host=os.getenv('APP_HOST', '0.0.0.0'),
port=int(os.getenv('APP_PORT', '8080')),
workers=1,
timeout_graceful_shutdown=5,
)
+82
View File
@@ -0,0 +1,82 @@
import os
import unicodedata
from uuid import uuid4
from PIL import Image, UnidentifiedImageError
from sqlalchemy import func, select
from .errors import APIError
from .models import Attachment, ChatMember
from .object_storage import BUCKET, PREFIX, s3_client
from .store import ensure_uuid, serialize, store
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
if MAX_BYTES < 1 or MAX_PIXELS < 1:
raise ValueError('Upload limits must be positive')
def validate_filename(filename):
if (not filename or not filename.strip() or len(filename) > 255
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
raise APIError(400, 'invalid_request', 'Неверное имя файла')
def validate_image(source, content_type):
try:
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
if picture.width * picture.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
if Image.MIME[picture.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
picture.verify()
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
except Image.DecompressionBombError as exc:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
source.seek(0)
def lock_object(session, key):
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
def metadata(attachment):
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
'content_type', 'size_bytes', 'sha256', 'created_at')
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
'variants': [], 'error_code': None}
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
validate_image(source, content_type)
attachment_id = str(uuid4())
key = PREFIX + attachment_id
with store.sessions.begin() as session:
store._chat(session, chat_id, actor_id)
lock_object(session, key)
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
ContentLength=size, ContentType=content_type)
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
object_key=key, filename=filename, content_type=content_type,
size_bytes=size, sha256=digest)
session.add(attachment)
session.flush()
result = metadata(attachment)
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
return result
def get_attachment(attachment_id, actor_id):
with store.sessions() as session:
attachment = session.scalar(select(Attachment).join(
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
if attachment is None:
raise APIError(404, 'not_found', 'Объект не найден')
return attachment
+68
View File
@@ -0,0 +1,68 @@
import hashlib
import json
import os
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from redis import Redis
from .errors import APIError
password_hasher = PasswordHasher()
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
def verify_password(encoded, password):
try:
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
except VerificationError:
return False
def unauthorized():
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
class Sessions:
def __init__(self):
self.client = Redis.from_url(
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
retry_on_timeout=False,
)
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
if self.ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
@staticmethod
def key(token):
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise unauthorized()
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
def create(self, user):
token = secrets.token_urlsafe(32)
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
pxat=int(expires.timestamp() * 1000))
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
def user_id(self, token):
raw = self.client.get(self.key(token))
if raw is None:
raise unauthorized()
session = json.loads(raw)
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
raise unauthorized()
return session['user_id']
def delete(self, token):
if not self.client.delete(self.key(token)):
raise unauthorized()
sessions = Sessions()
+40
View File
@@ -0,0 +1,40 @@
"""History pages are disposable; PostgreSQL owns both data and their version."""
import hashlib
import json
import os
from redis.exceptions import RedisError
from .auth import sessions
class HistoryCache:
def __init__(self, client, ttl=30, prefix='history'):
self.client = client
self.ttl = ttl
self.prefix = prefix
if ttl < 1:
raise ValueError('CACHE_TTL_SECONDS must be positive')
def key(self, chat_id, version, limit, after):
page = json.dumps([limit, after], separators=(',', ':'))
digest = hashlib.sha256(page.encode()).hexdigest()
return f'{self.prefix}:{chat_id}:{version}:{digest}'
def get_or_load(self, chat_id, version, limit, after, load):
key = self.key(chat_id, version, limit, after)
try:
cached = self.client.get(key)
except RedisError:
return load()
if cached is not None:
return json.loads(cached)
page = load()
try:
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
except RedisError:
pass
return page
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
+61
View File
@@ -0,0 +1,61 @@
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, PREFIX, s3_client
GRACE = timedelta(hours=1)
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
for obj in page.get('Contents', []):
if obj['LastModified'] >= cutoff:
continue
key = obj['Key']
with sessions.begin() as session:
lock_object(session, key)
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
return deleted
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
try:
while True:
try:
count = cleanup(sessionmaker(engine), s3_client())
logging.info('Cleanup completed: deleted=%d', count)
except Exception as exc:
logging.error('Cleanup deferred: %s', type(exc).__name__)
if args.once:
raise SystemExit(1) from None
if args.once:
break
time.sleep(3600)
finally:
engine.dispose()
if __name__ == '__main__':
main()
+51
View File
@@ -0,0 +1,51 @@
import os
from pathlib import Path
from sqlalchemy import create_engine, func, insert, select
from sqlalchemy.engine import URL, make_url
from .models import SchemaMigration
def database_url():
if os.getenv('DATABASE_URL'):
url = make_url(os.environ['DATABASE_URL'])
if url.drivername in ('postgres', 'postgresql'):
url = url.set(drivername='postgresql+psycopg')
if url.drivername != 'postgresql+psycopg':
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
return url
return URL.create(
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
database=os.getenv('POSTGRES_DB', 'messenger'),
username=os.getenv('POSTGRES_USER', 'messenger'),
password=os.environ['POSTGRES_PASSWORD'],
)
def create_db_engine():
return create_engine(
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
connect_args={'connect_timeout': 2,
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
)
def migrate(engine):
with engine.begin() as conn:
conn.execute(select(func.pg_advisory_xact_lock(202602)))
SchemaMigration.__table__.create(conn, checkfirst=True)
applied = set(conn.scalars(select(SchemaMigration.version)))
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
if path.name not in applied:
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
conn.execute(insert(SchemaMigration).values(version=path.name))
if __name__ == '__main__':
engine = create_db_engine()
try:
migrate(engine)
print('Migrations applied')
finally:
engine.dispose()
+31
View File
@@ -0,0 +1,31 @@
from typing import Optional
from fastapi import Depends, Header, Query
from .auth import sessions, unauthorized
from .errors import APIError
from .store import store
def page_limit(limit: str = Query(default="50")) -> int:
if not limit.isascii() or not limit.isdecimal() or len(limit) > 3:
raise APIError(400, "invalid_request", "Неверный limit")
number = int(limit)
if not 1 <= number <= 100:
raise APIError(400, "invalid_request", "Неверный limit")
return number
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
parts = (authorization or '').split()
if len(parts) != 2 or parts[0].lower() != 'bearer':
raise unauthorized()
return parts[1]
def current_user(token: str = Depends(bearer_token)) -> dict:
user = store.get_user(sessions.user_id(token))
if user is None:
raise unauthorized()
return user
+60
View File
@@ -0,0 +1,60 @@
from fastapi import Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from .middleware import INSTANCE_ID
class APIError(Exception):
def __init__(self, status_code: int, code: str, message: str):
self.status_code = status_code
self.code = code
self.message = message
def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse:
request_id = getattr(request.state, "request_id", "")
return JSONResponse(
status_code=status_code,
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
content={
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
},
)
def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
response = error_response(request, exc.status_code, exc.code, exc.message)
if exc.status_code == 429:
response.headers['Retry-After'] = '1'
if exc.status_code == 401:
response.headers['WWW-Authenticate'] = (
'Basic realm="messenger", charset="UTF-8"'
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
)
return response
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
return error_response(request, 400, "invalid_request", "Неверный запрос")
def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
if exc.status_code == 404:
return error_response(request, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request, 400, "invalid_request", "Метод не поддерживается")
else:
return error_response(request, exc.status_code, "invalid_request", "Неверный запрос")
def internal_error_handler(request: Request, exc: Exception) -> JSONResponse:
return error_response(request, 500, "internal_error", "Внутренняя ошибка")
+99
View File
@@ -0,0 +1,99 @@
import asyncio
import os
from dataclasses import dataclass
from starlette.requests import Request
from .errors import error_response
from .middleware import is_upload
@dataclass(frozen=True)
class Limits:
max_users: int = 1000
max_chats: int = 1000
max_messages: int = 10000
max_text_bytes: int = 16 * 1024 * 1024
max_request_bytes: int = 64 * 1024
max_inflight: int = 32
message_rate: int = 1000
message_burst: int = 1000
@classmethod
def from_env(cls):
values = {}
for name, default in cls().__dict__.items():
value = int(os.getenv(name.upper(), str(default)))
if value < 1:
raise ValueError(f'{name.upper()} must be positive')
values[name] = value
return cls(**values)
limits = Limits.from_env()
class ResourceLimitMiddleware:
def __init__(self, app, settings=limits):
self.app = app
self.settings = settings
self.active = 0
async def __call__(self, scope, receive, send):
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
return await self.app(scope, receive, send)
request = Request(scope)
async def reject(status, code, message):
response = error_response(request, status, code, message)
if status == 503:
response.headers['Retry-After'] = '1'
await response(scope, receive, send)
if self.active >= self.settings.max_inflight:
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
self.active += 1
try:
# The upload endpoint authenticates and bounds its stream itself.
if is_upload(scope['method'], scope['path']):
return await self.app(scope, receive, send)
length = request.headers.get('content-length')
if length is not None:
try:
size = int(length)
if size < 0:
raise ValueError()
except ValueError:
return await reject(400, 'invalid_request', 'Неверный Content-Length')
if size > self.settings.max_request_bytes:
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
body = bytearray()
try:
async with asyncio.timeout(5):
while True:
message = await receive()
if message['type'] == 'http.disconnect':
return
chunk = message.get('body', b'')
if len(body) + len(chunk) > self.settings.max_request_bytes:
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
body.extend(chunk)
if not message.get('more_body', False):
break
except TimeoutError:
return await reject(503, 'unavailable', 'Истекло время чтения запроса')
delivered = False
async def buffered_receive():
nonlocal delivered
if not delivered:
delivered = True
return {'type': 'http.request', 'body': bytes(body), 'more_body': False}
return await receive()
await self.app(scope, buffered_receive, send)
finally:
self.active -= 1
+48
View File
@@ -0,0 +1,48 @@
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from starlette.exceptions import HTTPException
from contextlib import asynccontextmanager
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
from sqlalchemy.orm import sessionmaker
from redis.exceptions import RedisError
from botocore.exceptions import BotoCoreError, ClientError
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
from .database import create_db_engine
from .store import store
from .auth import sessions
from .middleware import RequestIdMiddleware
from .limits import ResourceLimitMiddleware
from .routers.health import router as health_router
from .routers.users import router as users_router
from .routers.chats import router as chats_router
from .routers.auth import router as auth_router
from .routers.attachments import router as attachments_router
@asynccontextmanager
async def lifespan(app):
engine = create_db_engine()
try:
store.sessions = sessionmaker(engine, expire_on_commit=False)
yield
finally:
engine.dispose()
sessions.client.close()
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
app.add_middleware(ResourceLimitMiddleware)
app.add_middleware(RequestIdMiddleware)
app.add_exception_handler(APIError, api_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(HTTPException, http_error_handler)
app.add_exception_handler(Exception, internal_error_handler)
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
app.add_exception_handler(error, dependency_error_handler)
app.include_router(health_router)
app.include_router(users_router)
app.include_router(chats_router)
app.include_router(auth_router)
app.include_router(attachments_router)
+43
View File
@@ -0,0 +1,43 @@
from uuid import uuid4
from socket import gethostname
from starlette.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
import re
def is_upload(method, path):
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
INSTANCE_ID = gethostname()
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
request.state.request_id = request_id
if (request.method in {"POST", "PUT", "PATCH"}
and request.url.path.startswith("/api/")
and not is_upload(request.method, request.url.path)
and request.url.path != '/api/v1/auth/sessions'):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
response = JSONResponse(
status_code=415,
content={
"error": {
"code": "unsupported_media_type",
"message": "Ожидается Content-Type application/json",
"request_id": request_id,
}
},
)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
response = await call_next(request)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
+85
View File
@@ -0,0 +1,85 @@
from datetime import datetime
from uuid import uuid4
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.dialects.postgresql import ARRAY
class Base(DeclarativeBase):
pass
class User(Base):
__tablename__ = 'users'
__table_args__ = (
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
username: Mapped[str] = mapped_column(Text, unique=True)
display_name: Mapped[str] = mapped_column(Text)
password_hash: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class Chat(Base):
__tablename__ = 'chats'
__table_args__ = (
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
Index('chats_order_idx', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
title: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
class ChatMember(Base):
__tablename__ = 'chat_members'
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
class Message(Base):
__tablename__ = 'messages'
__table_args__ = (
CheckConstraint('char_length(text) <= 4000'),
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
text: Mapped[str] = mapped_column(Text)
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class SchemaMigration(Base):
__tablename__ = 'schema_migrations'
version: Mapped[str] = mapped_column(Text, primary_key=True)
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
class Attachment(Base):
__tablename__ = 'attachments'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
object_key: Mapped[str] = mapped_column(Text, unique=True)
filename: Mapped[str] = mapped_column(Text)
content_type: Mapped[str] = mapped_column(Text)
size_bytes: Mapped[int] = mapped_column(BigInteger)
sha256: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class IdempotencyRecord(Base):
__tablename__ = 'message_idempotency'
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
key: Mapped[str] = mapped_column(Text, primary_key=True)
message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE'))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
+23
View File
@@ -0,0 +1,23 @@
import os
from functools import lru_cache
import boto3
from botocore.config import Config
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
PREFIX = 'originals/'
@lru_cache(maxsize=1)
def s3_client():
return boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
config=Config(connect_timeout=2, read_timeout=5,
retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'},
request_checksum_calculation='when_required',
response_checksum_validation='when_required'),
)
View File
+84
View File
@@ -0,0 +1,84 @@
import asyncio
import hashlib
from tempfile import TemporaryFile
from urllib.parse import quote
from fastapi import APIRouter, Depends, Query, Request
from starlette.background import BackgroundTask
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
from ..dependencies import current_user
from ..errors import APIError
from ..object_storage import BUCKET, s3_client
from ..store import ensure_uuid, store
router = APIRouter(prefix='/api/v1', tags=['attachments'])
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
decode_slot = asyncio.Semaphore(1)
@router.post('/chats/{chat_id}/attachments', status_code=201)
async def upload(chat_id: str, request: Request, filename: str = Query(...),
user: dict = Depends(current_user)):
chat_id = ensure_uuid(chat_id)
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
validate_filename(filename)
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if content_type not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
try:
length = int(length)
if length < 0:
raise ValueError()
except ValueError as exc:
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
size = 0
digest = hashlib.sha256()
with TemporaryFile() as source:
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
digest.update(chunk)
await run_in_threadpool(source.write, chunk)
except (TimeoutError, ClientDisconnect) as exc:
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
if length is not None and size != length:
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
async with decode_slot:
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
filename, content_type, size, digest.hexdigest())
@router.get('/attachments/{attachment_id}')
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
return metadata(get_attachment(attachment_id, user['id']))
@router.get('/attachments/{attachment_id}/content')
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
attachment = get_attachment(attachment_id, user['id'])
if variant != 'original':
raise APIError(404, 'not_found', 'Вариант не найден')
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
def chunks():
try:
yield from body.iter_chunks(chunk_size=64 * 1024)
finally:
body.close()
return StreamingResponse(chunks(), media_type=attachment.content_type,
headers={'Content-Length': str(attachment.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
'X-Content-Type-Options': 'nosniff'},
background=BackgroundTask(body.close))
+34
View File
@@ -0,0 +1,34 @@
import base64
from fastapi import APIRouter, Depends, Header, Request, Response
from starlette.concurrency import run_in_threadpool
from ..auth import sessions, unauthorized
from ..dependencies import bearer_token, current_user
from ..errors import APIError
from ..store import store
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
@router.post('', status_code=201)
async def create_session(request: Request, authorization: str = Header(default='')):
if await request.body():
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
try:
scheme, encoded = authorization.split()
if scheme.lower() != 'basic':
raise ValueError()
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
raise ValueError()
except (ValueError, UnicodeError):
raise unauthorized() from None
user = await run_in_threadpool(store.login, username, password)
return await run_in_threadpool(sessions.create, user)
@router.delete('/current', status_code=204)
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
sessions.delete(token)
return Response(status_code=204)
+49
View File
@@ -0,0 +1,49 @@
from typing import Optional
import re
from fastapi import APIRouter, Depends, Header, Query, status
from ..errors import APIError
from ..dependencies import current_user, page_limit
from ..schemas.chat import Chat, ChatPage, CreateChat
from ..schemas.message import CreateMessage, Message, MessagePage
from ..store import store
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED)
def create_chat(payload: CreateChat, user: dict = Depends(current_user)):
return store.create_chat(user["id"], payload.title, payload.member_ids)
@router.get("", response_model=ChatPage)
def list_chats(
limit: int = Depends(page_limit),
cursor: Optional[str] = Query(default=None, min_length=1),
user: dict = Depends(current_user),
):
return store.list_chats(user["id"], limit, cursor)
@router.get("/{chat_id}", response_model=Chat)
def get_chat(chat_id: str, user: dict = Depends(current_user)):
return store.get_chat_for_user(chat_id, user["id"])
@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED)
def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user),
idempotency_key: Optional[str] = Header(default=None)):
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
@router.get("/{chat_id}/messages", response_model=MessagePage)
def list_messages(
chat_id: str,
limit: int = Depends(page_limit),
cursor: Optional[str] = Query(default=None, min_length=1),
user: dict = Depends(current_user),
):
return store.list_messages(chat_id, user["id"], limit, cursor)
+23
View File
@@ -0,0 +1,23 @@
from fastapi import APIRouter
from sqlalchemy import select
from ..schemas.common import Health
from ..auth import sessions
from ..store import store
from ..object_storage import BUCKET, s3_client
router = APIRouter(prefix='/health')
@router.get("/live", response_model=Health)
def health_live():
return {"status": "ok"}
@router.get("/ready", response_model=Health)
def health_ready():
with store.sessions() as session:
session.execute(select(1))
sessions.client.ping()
s3_client().head_bucket(Bucket=BUCKET)
return {"status": "ok"}
+19
View File
@@ -0,0 +1,19 @@
from fastapi import APIRouter, Depends, status
from ..dependencies import current_user
from ..schemas.user import CreateUser, User
from ..store import store
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
def create_user(payload: CreateUser):
return store.create_user(payload.username, payload.display_name, payload.password)
@router.get("/me", response_model=User)
def get_current_user(user: dict = Depends(current_user)):
return user
View File
+35
View File
@@ -0,0 +1,35 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator
from .common import ContractModel
class CreateChat(ContractModel):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: List[StrictStr] = Field(min_length=1, max_length=100)
@field_validator("title")
def validate_title(cls, value):
if not value.strip():
raise ValueError("blank title")
return value
@field_validator("member_ids")
def validate_member_ids(cls, value):
if len(value) != len(set(value)):
raise ValueError("duplicate member_ids")
return value
class Chat(ContractModel):
id: str
title: str
member_ids: List[str]
created_at: str
class ChatPage(ContractModel):
items: List[Chat]
next_cursor: Optional[str]
+17
View File
@@ -0,0 +1,17 @@
from pydantic import BaseModel, ConfigDict, field_validator
class ContractModel(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator('*')
def validate_text_encoding(cls, value, info):
if isinstance(value, str):
value.encode('utf-8')
if info.field_name != 'password' and '\x00' in value:
raise ValueError('NUL is not supported in PostgreSQL text')
return value
class Health(ContractModel):
status: str
+39
View File
@@ -0,0 +1,39 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator, model_validator
from uuid import UUID
from .common import ContractModel
class CreateMessage(ContractModel):
text: StrictStr = Field(max_length=4000)
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
def validate_ids(cls, value):
ids = [str(UUID(item)) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('duplicate attachments')
return ids
@model_validator(mode='after')
def validate_content(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('empty message')
return self
class Message(ContractModel):
id: str
chat_id: str
sender_id: str
text: str
attachment_ids: List[str]
created_at: str
class MessagePage(ContractModel):
items: List[Message]
next_cursor: Optional[str]
+34
View File
@@ -0,0 +1,34 @@
import re
from pydantic import Field, StrictStr, field_validator
from .common import ContractModel
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
class CreateUser(ContractModel):
username: StrictStr
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("username")
def validate_username(cls, value):
if not USERNAME_RE.fullmatch(value):
raise ValueError("invalid username")
return value
@field_validator("display_name")
def validate_display_name(cls, value):
if not value.strip():
raise ValueError("blank display_name")
return value
class User(ContractModel):
id: str
username: str
display_name: str = Field(min_length=1, max_length=100)
created_at: str
+226
View File
@@ -0,0 +1,226 @@
import base64
import json
from datetime import datetime, timezone
from typing import Tuple
from uuid import UUID
from sqlalchemy import select, tuple_, update
from sqlalchemy.exc import IntegrityError
from .auth import password_hasher, verify_password, unauthorized
from .errors import APIError
from .cache import history_cache
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
def ensure_uuid(value):
try:
if not isinstance(value, str):
raise ValueError()
return str(UUID(value))
except (TypeError, ValueError) as exc:
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
def serialize(model, *fields):
if model is None:
return None
result = {}
for field in fields:
value = getattr(model, field)
if isinstance(value, datetime):
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
result[field] = value
return result
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
CHAT_FIELDS = ('id', 'title', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
def after_condition(model, after):
return tuple_(model.created_at, model.id) > (
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
def history_query(chat_id, limit, after):
query = select(Message).where(Message.chat_id == chat_id)
if after:
query = query.where(after_condition(Message, after))
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
class PostgresStore:
def __init__(self, cache=history_cache):
self.sessions = None
self.cache = cache
def create_user(self, username, display_name, password):
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
try:
with self.sessions.begin() as session:
session.add(user)
session.flush()
result = serialize(user, *USER_FIELDS)
return result
except IntegrityError as exc:
if getattr(exc.orig, 'sqlstate', None) == '23505':
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
raise
def get_user(self, user_id):
with self.sessions() as session:
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
def login(self, username, password):
with self.sessions() as session:
user = session.scalar(select(User).where(User.username == username))
encoded = user.password_hash if user else None
result = serialize(user, *USER_FIELDS)
if not verify_password(encoded, password):
raise unauthorized()
return result
def create_chat(self, actor_id, title, member_ids):
members = [ensure_uuid(member) for member in member_ids]
if len(members) != len(set(members)) or actor_id not in members:
raise APIError(400, 'invalid_request', 'Неверный список участников')
try:
with self.sessions.begin() as session:
chat = Chat(title=title)
session.add(chat)
session.flush()
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
result = serialize(chat, *CHAT_FIELDS)
result['member_ids'] = sorted(members)
return result
except IntegrityError as exc:
if getattr(exc.orig, 'sqlstate', None) == '23503':
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
raise
@staticmethod
def _chat(session, chat_id, actor_id):
chat = session.scalar(select(Chat).join(ChatMember).where(
Chat.id == chat_id, ChatMember.user_id == actor_id))
if chat is None:
raise APIError(404, 'not_found', 'Объект не найден')
return chat
@staticmethod
def _members(session, chat):
result = serialize(chat, *CHAT_FIELDS)
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
return result
def get_chat_for_user(self, chat_id, actor_id):
with self.sessions() as session:
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
def list_chats(self, actor_id, limit, cursor):
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
if after:
query = query.where(after_condition(Chat, after))
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
with self.sessions() as session:
chats = list(session.scalars(query))
items = [self._members(session, chat) for chat in chats[:limit]]
return self._page(items, len(chats) > limit, 'chats', actor_id)
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
chat_id = ensure_uuid(chat_id)
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
with self.sessions.begin() as session:
self._chat(session, chat_id, actor_id)
if attachment_ids:
found = list(session.scalars(select(Attachment.id).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
if len(found) != len(attachment_ids):
raise APIError(404, 'not_found', 'Объект не найден')
if idempotency_key is not None:
# Serialize writers to this chat before looking up the key.
# The same row is updated below for history cache invalidation.
session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update())
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
if record is not None:
previous = session.get(Message, record.message_id)
if previous.text != text or previous.attachment_ids != attachment_ids:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
return serialize(previous, *MESSAGE_FIELDS)
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
session.add(message)
session.flush()
if idempotency_key is not None:
session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id,
key=idempotency_key, message_id=message.id))
result = serialize(message, *MESSAGE_FIELDS)
return result
def list_messages(self, chat_id, actor_id, limit, cursor):
chat_id = ensure_uuid(chat_id)
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
with self.sessions() as session:
version = self._chat(session, chat_id, actor_id).history_version
return self.cache.get_or_load(
chat_id, version, limit, after,
lambda: self._load_messages(chat_id, limit, after),
)
def _load_messages(self, chat_id, limit, after):
with self.sessions() as session:
messages = list(session.scalars(history_query(chat_id, limit, after)))
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
return self._page(items, len(messages) > limit, 'messages', chat_id)
def _page(self, items, has_more, resource, owner_id):
cursor = None
if has_more:
last = items[-1]
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
return {'items': items, 'next_cursor': cursor}
@staticmethod
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
payload = {
"resource": resource,
"owner_id": owner_id,
"created_at": created_at,
"id": item_id,
}
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
@staticmethod
def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]:
try:
padded = cursor + "=" * (-len(cursor) % 4)
payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
raise APIError(400, "invalid_request", "Неверный cursor") from exc
if not isinstance(payload, dict):
raise APIError(400, "invalid_request", "Неверный cursor")
if payload.get("resource") != resource or payload.get("owner_id") != owner_id:
raise APIError(400, "invalid_request", "Неверный cursor")
item_id = ensure_uuid(payload.get("id"))
created_at = payload.get("created_at")
if not isinstance(created_at, str) or not created_at:
raise APIError(400, "invalid_request", "Неверный cursor")
try:
parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc:
raise ValueError("Expected UTC timestamp")
created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z")
except ValueError as exc:
raise APIError(400, "invalid_request", "Неверный cursor") from exc
return created_at, item_id
store = PostgresStore()
+39
View File
@@ -0,0 +1,39 @@
"""Attachment negative cases. Run against the Compose API at localhost:8080."""
import sys
from pathlib import Path
from uuid import uuid4
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
def main():
client = Client()
alice, bob, eve, chat = client.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes()
client.call('POST', path, 415, actor=alice, raw=b'not an image',
headers={'Content-Type': 'image/png'})
client.call('POST', path, 415, actor=alice, raw=image,
headers={'Content-Type': 'image/jpeg'})
client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1),
headers={'Content-Type': 'image/png'})
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png',
400, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
attachment = client.call('POST', path, 201, actor=alice, raw=image,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve)
client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice)
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json']
client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
body={'text': '', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': uuid4().hex})
if __name__ == '__main__':
main()
print('PASS: attachment validation and access checks')
+55
View File
@@ -0,0 +1,55 @@
"""Database atomicity and concurrent conflict checks, using isolated synthetic data."""
import unittest
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import patch
from uuid import uuid4
from sqlalchemy import func, select
from sqlalchemy.orm import Session, sessionmaker
from src.database import create_db_engine
from src.errors import APIError
from src.models import Chat, IdempotencyRecord, Message
from src.store import PostgresStore
class IdempotencyTests(unittest.TestCase):
def setUp(self):
self.engine = create_db_engine()
self.store = PostgresStore()
self.store.sessions = sessionmaker(self.engine, expire_on_commit=False)
self.user = self.store.create_user('idem_' + uuid4().hex[:16], 'Idempotency', 'Synthetic-password-123')
self.chat = self.store.create_chat(self.user['id'], 'Atomic', [self.user['id']])
def tearDown(self):
self.engine.dispose()
def test_record_and_message_rollback_together(self):
real_add = Session.add
def fail_record(session, instance, *args, **kwargs):
if isinstance(instance, IdempotencyRecord):
raise RuntimeError('Simulate failure after message INSERT, before COMMIT')
return real_add(session, instance, *args, **kwargs)
args = (self.chat['id'], self.user['id'], 'atomic', 'same-key')
with patch.object(Session, 'add', fail_record):
with self.assertRaises(RuntimeError):
self.store.create_message(*args)
with Session(self.engine) as session:
self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 0)
self.assertIsNone(session.get(IdempotencyRecord, (self.user['id'], self.chat['id'], 'same-key')))
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 0)
first = self.store.create_message(*args)
self.assertEqual(self.store.create_message(*args), first)
def test_concurrent_different_payloads_have_one_winner(self):
def send(index):
try:
return self.store.create_message(self.chat['id'], self.user['id'], str(index), 'conflict'), 201
except APIError as exc:
return None, exc.status_code
with ThreadPoolExecutor(max_workers=8) as executor:
results = list(executor.map(send, range(8)))
self.assertEqual(sorted(code for _, code in results), [201] + [409] * 7)
with Session(self.engine) as session:
self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 1)
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 1)
+127
View File
@@ -0,0 +1,127 @@
"""Run inside the API container: python -m unittest discover -s student-tests -v."""
import base64
import os
import subprocess
import sys
import time
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from sqlalchemy import func, select
from sqlalchemy.orm import sessionmaker
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
from src.auth import sessions
from src.database import create_db_engine, migrate
from src.models import User, Chat, ChatMember, SchemaMigration
class Lab2Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.engine = create_db_engine()
cls.sessions = sessionmaker(cls.engine)
@classmethod
def tearDownClass(cls):
cls.engine.dispose()
def setUp(self):
self.client = Client('http://127.0.0.1:8080')
def test_registration_race(self):
body = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Race',
'password': 'Synthetic-password-123'}
def register(_):
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
with ThreadPoolExecutor(max_workers=4) as executor:
codes = list(executor.map(register, range(4)))
self.assertEqual(sorted(codes), [201, 409, 409, 409])
with self.sessions() as session:
count = session.scalar(select(func.count()).select_from(User).where(User.username == body['username']))
self.assertEqual(count, 1)
def test_chat_rollback(self):
actor = self.client.user('rollback')
title = 'rollback_' + uuid4().hex
self.client.call('POST', '/api/v1/chats', 400, actor=actor,
body={'title': title, 'member_ids': [actor['id'], str(uuid4())]})
with self.sessions() as session:
self.assertEqual(session.scalar(select(func.count()).select_from(Chat).where(Chat.title == title)), 0)
self.assertEqual(session.scalar(select(func.count()).select_from(ChatMember).where(
ChatMember.user_id == actor['id'])), 0)
def test_password_logout_and_identity(self):
actor = self.client.user('auth')
credential = base64.b64encode((actor['user']['username'] + ':Wrong-password-123').encode()).decode()
self.client.call('POST', '/api/v1/auth/sessions', 401,
headers={'Authorization': 'Basic ' + credential})
self.client.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + credential})
self.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
me = self.client.call('GET', '/api/v1/users/me', actor=actor,
headers={'X-User-Id': str(uuid4())}, schema='User')['json']
self.assertEqual(me['id'], actor['id'])
ttl = sessions.client.ttl(sessions.key(actor['token']))
self.assertGreater(ttl, 0)
self.assertLessEqual(ttl, 3600)
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
self.client.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor)
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
with self.sessions() as session:
encoded = session.get(User, actor['id']).password_hash
self.assertTrue(encoded.startswith('$argon2id$'))
self.assertNotIn(actor['registration']['password'], encoded)
def test_unicode_password(self):
username = 'unicode_' + uuid4().hex[:12]
password = '🙂:я' * 40
self.client.call('POST', '/api/v1/users', 201,
body={'username': username, 'display_name': 'Unicode', 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
result = self.client.call('POST', '/api/v1/auth/sessions', 201,
headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
self.assertEqual(result['user']['username'], username)
def test_migrations_repeat(self):
with self.sessions() as session:
before = session.scalar(select(func.count()).select_from(User))
migrate(self.engine)
migrate(self.engine)
with self.sessions() as session:
self.assertEqual(session.scalar(select(func.count()).select_from(User)), before)
versions = set(session.scalars(select(SchemaMigration.version)))
expected = {path.name for path in (Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')}
self.assertEqual(versions, expected)
def test_expiry_short_ttl(self):
process = subprocess.Popen(
[sys.executable, '-m', 'src'],
env={**os.environ, 'SESSION_TTL_SECONDS': '2', 'APP_PORT': '8081', 'APP_HOST': '127.0.0.1'},
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
try:
client = Client('http://127.0.0.1:8081')
for _ in range(100):
try:
client.call('GET', '/health/ready')
break
except AssertionError:
self.assertIsNone(process.poll(), 'Short-TTL API exited')
time.sleep(0.1)
else:
self.fail('Short-TTL API did not become ready')
actor = client.user('expiry')
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
time.sleep(2.2)
client.call('GET', '/api/v1/users/me', 401, actor=actor)
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
finally:
process.terminate()
process.wait(timeout=10)
if __name__ == '__main__':
unittest.main()