ЛР 4 - Григорян - БПИ2303 - а 4 #57
+57
-10
@@ -1,20 +1,67 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Скопируйте в .env для локального окружения; .env не коммитится.
|
||||
# Имена переменных соответствуют полям app/core/config.py.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
MAX_MESSAGES=100000
|
||||
# Лимит одновременных запросов НА ОДНУ реплику
|
||||
MAX_IN_FLIGHT=16
|
||||
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
SESSION_STORE=redis
|
||||
|
||||
REDIS_HOST=redis
|
||||
REDIS_PORT=6379
|
||||
REDIS_DB=0
|
||||
REDIS_MAX_CONNECTIONS=20
|
||||
REDIS_POOL_TIMEOUT_SECONDS=1
|
||||
REDIS_CONNECT_TIMEOUT_SECONDS=1
|
||||
REDIS_SOCKET_TIMEOUT_SECONDS=0.5
|
||||
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_NAME=messenger
|
||||
DB_USER=messenger
|
||||
|
||||
# Пул на одну реплику: суммарно API_REPLICAS * (POOL_SIZE + MAX_OVERFLOW)
|
||||
# соединений должно помещаться в max_connections PostgreSQL.
|
||||
DB_POOL_SIZE=5
|
||||
DB_MAX_OVERFLOW=10
|
||||
DB_POOL_TIMEOUT_SECONDS=3
|
||||
DB_CONNECT_TIMEOUT_SECONDS=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
|
||||
# --- Балансировка и идемпотентность ---
|
||||
API_REPLICAS=2
|
||||
API_IMAGE=mtusi-messenger:lab4
|
||||
API_BUILD_REVISION=development
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
SHUTDOWN_DRAIN_SECONDS=1
|
||||
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
READINESS_TIMEOUT_SECONDS=2
|
||||
|
||||
# Порт единственной публичной точки входа (nginx)
|
||||
LB_HOST_PORT=8080
|
||||
# Используются только тестовым стендом (compose.test.yaml)
|
||||
DB_HOST_PORT=15432
|
||||
REDIS_HOST_PORT=16379
|
||||
|
||||
# --- Lab4: MinIO, ограниченный upload и очистка журнала ---
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
S3_CONNECT_TIMEOUT_SECONDS=2
|
||||
S3_READ_TIMEOUT_SECONDS=10
|
||||
S3_UPLOAD_TIMEOUT_SECONDS=30
|
||||
S3_DOWNLOAD_TIMEOUT_SECONDS=60
|
||||
S3_IMAGE_TIMEOUT_SECONDS=10
|
||||
S3_MAX_UPLOAD_BYTES=10485760
|
||||
S3_MAX_IMAGE_PIXELS=40000000
|
||||
S3_CHUNK_BYTES=65536
|
||||
S3_MAX_CONCURRENT_UPLOADS=2
|
||||
S3_CLEANUP_GRACE_SECONDS=86400
|
||||
S3_CLEANUP_INTERVAL_SECONDS=300
|
||||
|
||||
@@ -28,5 +28,9 @@ coverage/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
venv/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
AGENTS.md
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
FROM python:3.12.15-slim-bookworm
|
||||
|
||||
LABEL io.mtusi.messenger.secrets="file-v1"
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN useradd --system --uid 10001 --no-create-home appuser
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY alembic.ini ./
|
||||
COPY migrations ./migrations
|
||||
COPY app ./app
|
||||
|
||||
ARG API_BUILD_REVISION
|
||||
LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION
|
||||
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
|
||||
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -1,13 +1,86 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
-include .env
|
||||
|
||||
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard venv/bin/python),venv/bin/python,python3))
|
||||
PYTEST_ARGS ?=
|
||||
BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080)
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
COMPOSE = docker compose --env-file .env
|
||||
|
||||
# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis
|
||||
# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh.
|
||||
TEST_PROJECT ?= mtusi-messenger-lab4-test
|
||||
TEST_LB_PORT ?= 18080
|
||||
TEST_DB_PORT ?= $(DB_HOST_PORT)
|
||||
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
|
||||
TEST_BASE_URL = http://localhost:$(TEST_LB_PORT)
|
||||
TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \
|
||||
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT)
|
||||
TEST_COMPOSE = $(TEST_ENV) $(COMPOSE)
|
||||
|
||||
TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file <secrets/db_password),$(DB_PASSWORD))
|
||||
TEST_DATABASE_URL ?= postgresql+asyncpg://$(DB_USER):$(TEST_DB_PASSWORD)@localhost:$(TEST_DB_PORT)/$(DB_NAME)_test
|
||||
TEST_REDIS_HOST ?= localhost
|
||||
TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/redis_password),$(REDIS_PASSWORD))
|
||||
TEST_REDIS_DB ?= 15
|
||||
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
|
||||
|
||||
define wait_ready
|
||||
@for i in $$(seq 1 90); do \
|
||||
curl -sf "$(1)/health/ready" >/dev/null && exit 0; sleep 1; \
|
||||
done; echo "API $(1) не готов за 90 секунд" >&2; exit 1
|
||||
endef
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover \
|
||||
init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \
|
||||
up down restart restore ps logs migrate migration instances \
|
||||
test-stand-up test-stand-down test-stand-destroy test-stand-ps \
|
||||
test-stand-smoke test-stand-persistence test-stand-failover \
|
||||
test-own test-docker test-limits test-concurrency test-migrations test-all \
|
||||
explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \
|
||||
rolling-update failover-all test-lab3-grade4
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply"
|
||||
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
|
||||
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
|
||||
@echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps"
|
||||
@echo " test-stand-smoke | test-stand-persistence | test-stand-failover"
|
||||
@echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all"
|
||||
@echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)"
|
||||
@echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)"
|
||||
@echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)"
|
||||
@echo "База lab3: rolling-update | failover-all"
|
||||
@echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)"
|
||||
@echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)"
|
||||
|
||||
# --- Проверки API и сценариев отказа (tests/) ---
|
||||
|
||||
init:
|
||||
$(PYTHON) scripts/init_local.py
|
||||
|
||||
test-grade4:
|
||||
$(MAKE) check validate
|
||||
$(MAKE) test-stand-up
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-s3:
|
||||
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
|
||||
|
||||
load-uploads:
|
||||
$(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10
|
||||
|
||||
cleanup-dry-run:
|
||||
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once
|
||||
|
||||
cleanup-apply:
|
||||
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once --apply
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -15,6 +88,22 @@ check:
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
|
||||
test-nginx:
|
||||
$(PYTHON) scripts/check_nginx.py
|
||||
|
||||
test-lab3-grade3:
|
||||
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
|
||||
$(MAKE) check validate; \
|
||||
$(MAKE) test-stand-up; \
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
|
||||
|
||||
test-lab3-grade4:
|
||||
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
|
||||
$(MAKE) check validate; \
|
||||
$(MAKE) test-stand-up; \
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -28,3 +117,112 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# --- Основной стенд ---
|
||||
|
||||
rolling-update:
|
||||
$(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
failover-all:
|
||||
$(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
up:
|
||||
$(COMPOSE) up -d --build
|
||||
$(call wait_ready,$(BASE_URL))
|
||||
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
|
||||
restart:
|
||||
scripts/restart.sh
|
||||
$(call wait_ready,$(BASE_URL))
|
||||
|
||||
# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры).
|
||||
restore:
|
||||
$(COMPOSE) up -d --wait api
|
||||
|
||||
ps:
|
||||
$(COMPOSE) ps
|
||||
|
||||
logs:
|
||||
$(COMPOSE) logs -f api nginx
|
||||
|
||||
migrate:
|
||||
$(COMPOSE) run --rm --build migrate
|
||||
|
||||
# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id).
|
||||
instances:
|
||||
@for i in $$(seq 1 10); do \
|
||||
curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \
|
||||
done
|
||||
|
||||
# Генерация миграции против БД тестового стенда (основной стенд БД не публикует).
|
||||
migration:
|
||||
DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)"
|
||||
|
||||
# --- Тестовый стенд ---
|
||||
|
||||
test-stand-up:
|
||||
$(TEST_COMPOSE) up -d --build
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
$(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
|
||||
|
||||
test-stand-down:
|
||||
$(TEST_COMPOSE) down
|
||||
|
||||
test-stand-destroy:
|
||||
$(TEST_COMPOSE) down -v
|
||||
|
||||
test-stand-ps:
|
||||
$(TEST_COMPOSE) ps
|
||||
|
||||
test-stand-smoke:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-stand-idempotency:
|
||||
$(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-stand-persistence:
|
||||
$(TEST_COMPOSE) ps
|
||||
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_COMPOSE) ps
|
||||
|
||||
test-stand-failover:
|
||||
$(TEST_COMPOSE) ps
|
||||
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_COMPOSE) ps -a api
|
||||
$(TEST_COMPOSE) up -d --wait api
|
||||
|
||||
# --- Собственные тесты (student-tests/) ---
|
||||
|
||||
test-own:
|
||||
$(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
|
||||
test-docker:
|
||||
$(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
|
||||
test-limits:
|
||||
$(PYTHON) -m pytest student-tests/test_limits.py -q
|
||||
|
||||
test-concurrency:
|
||||
$(PYTHON) -m pytest student-tests/test_concurrency.py -q
|
||||
|
||||
test-migrations:
|
||||
$(PYTHON) -m pytest student-tests/test_migrations.py -q
|
||||
|
||||
test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence
|
||||
|
||||
# --- Демо (на тестовом стенде) ---
|
||||
|
||||
explain:
|
||||
$(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py
|
||||
|
||||
expiry-demo: test-stand-up
|
||||
$(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh
|
||||
$(TEST_COMPOSE) up -d api
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
|
||||
failure-demo: test-stand-up
|
||||
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Лабораторная 4. S3-хранилище и вложения
|
||||
|
||||
В этом fork реализованы вложения на MinIO поверх базы lab3. Инструкция
|
||||
запуска — [RUNBOOK.md](RUNBOOK.md), соответствие требованиям оценки 4 —
|
||||
[LAB4_PLAN.md](LAB4_PLAN.md). Полный прогон на оценку 4 выполнен;
|
||||
результаты — в [REPORT.md](REPORT.md) и [evidence](evidence/README.md). Требования ниже сохранены как задание.
|
||||
|
||||
Разделить метаданные и бинарное содержимое, добавить изображения в сообщения и сохранить доступ к ним при переключении API-реплики.
|
||||
|
||||
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
|
||||
|
||||
@@ -1,37 +1,72 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: **Григорян Илья Мурадович, БПИ2303**. Целевая оценка: **4**, трек 5 не заявлен.
|
||||
|
||||
## Запуск из чистого clone
|
||||
Проверка: **04.11.2026**, отдельный чистый стенд `mtusi-messenger-lab4-test`.
|
||||
Базовый commit: `ec540a1a725862a8619fcb758daec4ad197f0dd5`; проверено незакоммиченное рабочее дерево.
|
||||
Точный снимок — [SHA-256 исходников](evidence/source-sha256.txt), среда и команды —
|
||||
[run.json](evidence/run.json). Исходники во время прогона не изменялись.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
## Запуск
|
||||
|
||||
## Архитектура и решения
|
||||
Нужны Python 3.12, Docker и Compose. Из корня проекта:
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make init
|
||||
make up
|
||||
```
|
||||
|
||||
## Доказательства критериев
|
||||
Вход — `http://localhost:8080`. `make init` сохраняет существующие настройки и
|
||||
секреты. MinIO/mc собираются из закреплённых исходников; первая сборка требует
|
||||
интернета. Миграции и bootstrap выполняются до API. `make down` сохраняет данные.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Повтор полного прогона с сохранением evidence и удалением тестовых volumes:
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
```sh
|
||||
.venv/bin/python -m scripts.collect_evidence
|
||||
```
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Реализация
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Nginx → две FastAPI-реплики → PostgreSQL, Redis и приватный MinIO.
|
||||
Метаданные и связи вложений находятся в PostgreSQL, оригиналы — в MinIO.
|
||||
Сохранены общие сессии и идемпотентность lab3; сообщения поддерживают `attachment_ids`.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
PNG/JPEG принимаются потоком во временный tmpfs, проверяются отдельным ограниченным
|
||||
процессом и сохраняются без перекодирования. `201 ready` возвращается после PUT
|
||||
и COMMIT. До PUT фиксируется журнал загрузки; при сбое выполняется компенсация,
|
||||
при её отказе — повторная очистка после защитного интервала 24 часа. Готовые
|
||||
вложения не удаляются автоматически.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Проверки и результаты
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**188 pytest прошли за 73,96 с**;
|
||||
контракт, OpenAPI и все сквозные сценарии завершились успешно.
|
||||
Исходный вывод — [test-output.txt](evidence/test-output.txt), методика —
|
||||
[evidence/README.md](evidence/README.md), числа — [metrics.json](evidence/metrics.json).
|
||||
|
||||
| Требование | Подтверждение |
|
||||
| --- | --- |
|
||||
| 3: сохранён минимум lab3, общий стенд | Две API, общие PostgreSQL/Redis/MinIO; 24 конкурентных повтора через две реплики создали одно сообщение. |
|
||||
| 3: вложения, SHA-256, авторизованный доступ | Smoke прошёл: 201 ready, метаданные, сообщение с файлом, побайтный оригинал; постороннему 404. |
|
||||
| 3: PNG/JPEG и ограничения | Проверены MIME, имя, повреждения, превышения и приём ровно 10 MiB / 40 млн пикселей. |
|
||||
| 3: сохранность и отказ загрузившей API | Persistence и failover прошли с прежней сессией и тем же оригиналом; перезапуск всех работающих сервисов подтверждён PID/StartedAt. |
|
||||
| 4: негативные сценарии | Слишком большой файл, повреждение, подмена MIME, неизвестный/чужой ID и файл другого чата дают предусмотренные 4xx. |
|
||||
| 4: согласованность S3/БД | Проверены отмена, неопределённый COMMIT, реальный PUT с отказом метаданных, неудачная компенсация, grace period, dry-run и повтор очистки. |
|
||||
| 4: ограниченная память | 3 серии по 10 запросов: 12 × 201, 18 × 429; пики выборок API — 154,9 / 156,1 MiB при лимите 1536 MiB. |
|
||||
| 4: приватность и bootstrap | Повтор bootstrap ×2 сохраняет данные; прямые анонимные GET/PUT запрещены, credentials API ограничены, консоль недоступна из сети API. |
|
||||
|
||||
Сохранённые проверки lab3 также прошли: SIGTERM, ресурсы/секреты, rolling update
|
||||
обеих API и последовательный отказ каждой. Восстановление записи — **1,865 / 2,042 с**.
|
||||
Nginx отдельно проверен: [nginx-output.txt](evidence/nginx-output.txt).
|
||||
Тестовые контейнеры, сети и volumes удалены — [cleanup.txt](evidence/cleanup.txt).
|
||||
|
||||
## Границы результата
|
||||
|
||||
Один хост и один короткий синтетический прогон. PostgreSQL, Redis, MinIO и Nginx
|
||||
не реплицированы. 429 ограничителя и переходные 503 при отказах/обновлении учтены;
|
||||
подтверждённые сообщения и файлы сохранились. Память cgroup измерялась через 20 мс,
|
||||
включая дочерние процессы и tmpfs; накопительный kernel peak — до 156,1 MiB.
|
||||
Это не измерение предельного RPS. Использована помощь Codex; версии и первичные.
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
[alembic]
|
||||
script_location = migrations
|
||||
prepend_sys_path = .
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,4 @@
|
||||
from app.main import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from dataclasses import asdict
|
||||
import json
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.s3 import S3Settings
|
||||
from app.domain.attachments import reconcile_attachments
|
||||
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def cleanup_once(factory, storage, apply: bool) -> bool:
|
||||
completed = True
|
||||
try:
|
||||
async with factory() as session:
|
||||
result = await reconcile_attachments(session, storage, dry_run=not apply)
|
||||
print(json.dumps(asdict(result)), flush=True)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error("Reconciliation", error)
|
||||
completed = False
|
||||
return completed
|
||||
|
||||
|
||||
async def run(once: bool, apply: bool) -> None:
|
||||
settings = Settings() # type: ignore
|
||||
s3_settings = S3Settings()
|
||||
engine = create_db_engine(settings)
|
||||
factory = create_session_factory(engine)
|
||||
storage = create_s3_storage(s3_settings)
|
||||
try:
|
||||
while True:
|
||||
completed = await cleanup_once(factory, storage, apply)
|
||||
if once:
|
||||
if not completed:
|
||||
raise SystemExit(1)
|
||||
break
|
||||
await asyncio.sleep(s3_settings.cleanup_interval_seconds)
|
||||
finally:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
|
||||
parser.add_argument("--once", action="store_true")
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="удалять безопасные кандидаты; по умолчанию dry-run",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(run(args.once, args.apply))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import Literal, Self
|
||||
|
||||
from pydantic import Field, SecretStr, model_validator
|
||||
from pydantic_settings import (
|
||||
BaseSettings,
|
||||
PydanticBaseSettingsSource,
|
||||
SettingsConfigDict,
|
||||
)
|
||||
from sqlalchemy import URL
|
||||
from app.core.s3 import S3Settings
|
||||
|
||||
ASYNC_DRIVER = "postgresql+asyncpg"
|
||||
SECRETS_DIR = next(
|
||||
(
|
||||
path
|
||||
for path in (
|
||||
Path("/run/secrets"),
|
||||
Path(__file__).resolve().parents[2] / "secrets",
|
||||
)
|
||||
if (path / "db_password").is_file()
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
class LogLevel(StrEnum):
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
s3_enabled: bool = True
|
||||
s3: S3Settings = Field(default_factory=S3Settings)
|
||||
app_host: str
|
||||
app_port: int = Field(ge=1, le=65535)
|
||||
app_env: str
|
||||
log_level: LogLevel = LogLevel.INFO
|
||||
|
||||
max_messages: int = Field(ge=1)
|
||||
max_in_flight: int = Field(ge=1)
|
||||
|
||||
session_ttl_seconds: int = Field(default=3600, ge=1)
|
||||
session_store: Literal["redis"] = "redis"
|
||||
|
||||
redis_host: str = "localhost"
|
||||
redis_port: int = Field(default=6379, ge=1, le=65535)
|
||||
redis_db: int = Field(default=0, ge=0)
|
||||
redis_password: SecretStr | None = None
|
||||
|
||||
redis_max_connections: int = Field(default=20, ge=1)
|
||||
redis_pool_timeout_seconds: float = Field(default=1.0, gt=0)
|
||||
redis_connect_timeout_seconds: float = Field(default=1.0, gt=0)
|
||||
redis_socket_timeout_seconds: float = Field(default=0.5, gt=0)
|
||||
|
||||
database_url: SecretStr | None = None
|
||||
db_host: str | None = None
|
||||
db_port: int = Field(default=5432, ge=1, le=65535)
|
||||
db_name: str | None = None
|
||||
db_user: str | None = None
|
||||
db_password: SecretStr | None = None
|
||||
|
||||
db_pool_size: int = Field(default=5, ge=1)
|
||||
db_max_overflow: int = Field(default=5, ge=0)
|
||||
db_pool_timeout_seconds: float = Field(default=3.0, gt=0)
|
||||
db_connect_timeout_seconds: int = Field(default=3, ge=1)
|
||||
db_statement_timeout_ms: int = Field(default=5000, ge=1)
|
||||
|
||||
instance_id: str | None = None
|
||||
idempotency_ttl_seconds: int = Field(default=86400, ge=1)
|
||||
shutdown_drain_seconds: float = Field(default=1, ge=0)
|
||||
graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1)
|
||||
readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5)
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore",
|
||||
secrets_dir=SECRETS_DIR,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def settings_customise_sources(
|
||||
cls,
|
||||
settings_cls: type[BaseSettings],
|
||||
init_settings: PydanticBaseSettingsSource,
|
||||
env_settings: PydanticBaseSettingsSource,
|
||||
dotenv_settings: PydanticBaseSettingsSource,
|
||||
file_secret_settings: PydanticBaseSettingsSource,
|
||||
) -> tuple[PydanticBaseSettingsSource, ...]:
|
||||
"""Явные параметры и файлы имеют приоритет над окружением."""
|
||||
return init_settings, file_secret_settings, env_settings, dotenv_settings
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_database_source(self) -> Self:
|
||||
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
|
||||
raise ValueError(
|
||||
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
|
||||
)
|
||||
if self.database_url is not None:
|
||||
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
|
||||
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
|
||||
return self
|
||||
|
||||
if not (self.db_host and self.db_name and self.db_user and self.db_password):
|
||||
raise ValueError(
|
||||
"Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password"
|
||||
)
|
||||
return self
|
||||
|
||||
def get_database_url(self) -> str:
|
||||
if self.database_url is not None:
|
||||
return self.database_url.get_secret_value()
|
||||
|
||||
assert self.db_password is not None
|
||||
url = URL.create(
|
||||
ASYNC_DRIVER,
|
||||
username=self.db_user,
|
||||
password=self.db_password.get_secret_value(),
|
||||
host=self.db_host,
|
||||
port=self.db_port,
|
||||
database=self.db_name,
|
||||
)
|
||||
return url.render_as_string(hide_password=False)
|
||||
@@ -0,0 +1,28 @@
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def create_db_engine(settings: Settings) -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
settings.get_database_url(),
|
||||
pool_size=settings.db_pool_size,
|
||||
max_overflow=settings.db_max_overflow,
|
||||
pool_timeout=settings.db_pool_timeout_seconds,
|
||||
pool_pre_ping=True,
|
||||
connect_args={
|
||||
"timeout": settings.db_connect_timeout_seconds,
|
||||
"server_settings": {
|
||||
"statement_timeout": str(settings.db_statement_timeout_ms)
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
|
||||
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
|
||||
from app.domain.image_validation import decode_image
|
||||
from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
|
||||
path, content_type, maximum = sys.argv[1:]
|
||||
status = 0
|
||||
try:
|
||||
decode_image(Path(path), content_type, int(maximum))
|
||||
except IMAGE_WORKER_ERRORS as error:
|
||||
status = image_worker_error_code(error)
|
||||
return status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Идентификатор API-реплики и заголовок X-Instance-Id."""
|
||||
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
INSTANCE_ID_HEADER = "X-Instance-Id"
|
||||
|
||||
|
||||
def resolve_instance_id(configured: str | None) -> str:
|
||||
"""Выбирает INSTANCE_ID, HOSTNAME или UUID при старте приложения."""
|
||||
if configured and configured.strip():
|
||||
return configured.strip()
|
||||
|
||||
hostname = os.getenv("HOSTNAME", "").strip()
|
||||
if hostname:
|
||||
return hostname
|
||||
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
class InstanceIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Добавляет сохранённый идентификатор реплики в ответ."""
|
||||
|
||||
def __init__(self, app, instance_id: str) -> None:
|
||||
super().__init__(app)
|
||||
self.instance_id = instance_id
|
||||
|
||||
async def dispatch(
|
||||
self,
|
||||
request: Request,
|
||||
call_next: RequestResponseEndpoint,
|
||||
) -> Response:
|
||||
response = await call_next(request)
|
||||
response.headers[INSTANCE_ID_HEADER] = self.instance_id
|
||||
|
||||
return response
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Готовность реплики и мягкое завершение сервера."""
|
||||
|
||||
import asyncio
|
||||
from types import FrameType
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
class Lifecycle:
|
||||
"""Хранит состояние завершения реплики."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._draining = False
|
||||
|
||||
@property
|
||||
def draining(self) -> bool:
|
||||
return self._draining
|
||||
|
||||
def start_draining(self) -> None:
|
||||
"""Отключает приём новых бизнес-запросов."""
|
||||
self._draining = True
|
||||
|
||||
|
||||
class GracefulServer(uvicorn.Server):
|
||||
"""Оставляет время для исключения завершающейся реплики."""
|
||||
|
||||
def __init__(
|
||||
self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float
|
||||
) -> None:
|
||||
super().__init__(config)
|
||||
self.lifecycle = lifecycle
|
||||
self.drain_seconds = drain_seconds
|
||||
self._drain_task: asyncio.Task[None] | None = None
|
||||
|
||||
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
|
||||
"""Первый сигнал запускает drain, повторный ускоряет остановку."""
|
||||
if self._drain_task is not None:
|
||||
if not self._drain_task.done():
|
||||
self._drain_task.cancel()
|
||||
super().handle_exit(sig, frame)
|
||||
return
|
||||
self.lifecycle.start_draining()
|
||||
self._drain_task = asyncio.get_running_loop().create_task(
|
||||
self._finish_draining(sig, frame)
|
||||
)
|
||||
|
||||
async def _finish_draining(self, sig: int, frame: FrameType | None) -> None:
|
||||
"""Передаёт остановку Uvicorn после drain-периода."""
|
||||
await asyncio.sleep(self.drain_seconds)
|
||||
super().handle_exit(sig, frame)
|
||||
@@ -0,0 +1,25 @@
|
||||
from redis.asyncio import BlockingConnectionPool, Redis
|
||||
from redis.asyncio.retry import Retry
|
||||
from redis.backoff import NoBackoff
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
def create_redis_client(settings: Settings) -> Redis:
|
||||
password = (
|
||||
settings.redis_password.get_secret_value() if settings.redis_password else None
|
||||
)
|
||||
pool = BlockingConnectionPool(
|
||||
host=settings.redis_host,
|
||||
port=settings.redis_port,
|
||||
db=settings.redis_db,
|
||||
password=password,
|
||||
max_connections=settings.redis_max_connections,
|
||||
timeout=settings.redis_pool_timeout_seconds,
|
||||
socket_connect_timeout=settings.redis_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis_socket_timeout_seconds,
|
||||
retry=Retry(NoBackoff(), retries=1),
|
||||
decode_responses=True,
|
||||
)
|
||||
return Redis.from_pool(pool)
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.errors.exceptions import rate_limited, unavailable
|
||||
|
||||
|
||||
class RequestCapacity:
|
||||
def __init__(self, max_in_flight: int) -> None:
|
||||
self._max_in_flight = max_in_flight
|
||||
self._in_flight = 0
|
||||
|
||||
@property
|
||||
def in_flight(self) -> int:
|
||||
return self._in_flight
|
||||
|
||||
def try_acquire(self) -> bool:
|
||||
if self._in_flight >= self._max_in_flight:
|
||||
return False
|
||||
self._in_flight += 1
|
||||
return True
|
||||
|
||||
def release(self) -> None:
|
||||
self._in_flight -= 1
|
||||
|
||||
|
||||
def create_request_capacity(max_in_flight: int) -> RequestCapacity:
|
||||
return RequestCapacity(max_in_flight)
|
||||
|
||||
|
||||
async def acquire_request_slot(request: Request) -> AsyncIterator[None]:
|
||||
if request.app.state.lifecycle.draining:
|
||||
raise unavailable("Реплика завершает работу")
|
||||
capacity: RequestCapacity = request.app.state.request_capacity
|
||||
|
||||
if not capacity.try_acquire():
|
||||
raise rate_limited()
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
capacity.release()
|
||||
@@ -0,0 +1,22 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
REQUEST_ID_HEADER = "X-Request-Id"
|
||||
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(
|
||||
self,
|
||||
request: Request,
|
||||
call_next: RequestResponseEndpoint,
|
||||
) -> Response:
|
||||
request_id = str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
|
||||
response = await call_next(request)
|
||||
response.headers[REQUEST_ID_HEADER] = request_id
|
||||
|
||||
return response
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Конфигурация MinIO; значения credentials читаются только из файлов."""
|
||||
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
class S3Settings(BaseSettings):
|
||||
endpoint: str = "http://minio:9000"
|
||||
region: str = "us-east-1"
|
||||
bucket: str = "messenger-attachments"
|
||||
access_key_id_file: Path = Path("/run/secrets/s3_access_key_id")
|
||||
secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key")
|
||||
connect_timeout_seconds: float = Field(default=2, gt=0, le=5)
|
||||
read_timeout_seconds: float = Field(default=10, gt=0, le=15)
|
||||
upload_timeout_seconds: float = Field(default=30, gt=0, le=60)
|
||||
download_timeout_seconds: float = Field(default=60, gt=0, le=120)
|
||||
image_timeout_seconds: float = Field(default=10, gt=0, le=15)
|
||||
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
|
||||
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
|
||||
chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144)
|
||||
max_concurrent_uploads: int = Field(default=2, ge=1, le=2)
|
||||
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
|
||||
cleanup_interval_seconds: int = Field(default=300, ge=1)
|
||||
|
||||
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_endpoint(self):
|
||||
url = urlsplit(self.endpoint)
|
||||
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
|
||||
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
|
||||
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
|
||||
return self
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import secrets
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
TOKEN_BYTES = 32
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
_executor = ThreadPoolExecutor(
|
||||
max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)),
|
||||
thread_name_prefix="argon2",
|
||||
)
|
||||
|
||||
|
||||
def _hash_password_sync(password: str) -> str:
|
||||
"""Блокирующий расчёт Argon2id-хеша со случайной солью в формате PHC."""
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def _verify_password_sync(password: str, password_hash: str) -> bool:
|
||||
"""Блокирующая проверка пароля по PHC-строке; False при несовпадении или повреждённом хеше."""
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES))
|
||||
|
||||
|
||||
async def hash_password(password: str) -> str:
|
||||
"""Хеширует пароль в пуле потоков, не блокируя event loop."""
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(_executor, _hash_password_sync, password)
|
||||
|
||||
|
||||
async def verify_password(password: str, password_hash: str) -> bool:
|
||||
"""Проверяет пароль в пуле потоков, не блокируя event loop."""
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(
|
||||
_executor, _verify_password_sync, password, password_hash
|
||||
)
|
||||
|
||||
|
||||
async def dummy_verify_password(password: str) -> None:
|
||||
"""Проверка против фиктивного хеша: время ответа не выдаёт отсутствие пользователя."""
|
||||
await verify_password(password, _DUMMY_HASH)
|
||||
|
||||
|
||||
def generate_token() -> str:
|
||||
"""Непрозрачный токен сессии: 32 случайных байта в base64url."""
|
||||
return secrets.token_urlsafe(TOKEN_BYTES)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
"""SHA-256 токена в hex: детерминированный ключ для хранилища сессий."""
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
@@ -0,0 +1 @@
|
||||
"""Бизнес-логика: проверки доступа, транзакции и ошибки API."""
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from contextlib import contextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from anyio import CancelScope
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.request_capacity import RequestCapacity
|
||||
from app.domain.chats import ensure_chat_member
|
||||
from app.domain.images import validate_filename, validate_image
|
||||
from app.errors.exceptions import (
|
||||
conflict,
|
||||
invalid_request,
|
||||
not_found,
|
||||
payload_too_large,
|
||||
rate_limited,
|
||||
unavailable,
|
||||
unsupported_media_type,
|
||||
)
|
||||
from app.errors.handlers import (
|
||||
RECOVERABLE_STORAGE_ERRORS,
|
||||
log_recoverable_error,
|
||||
operation_timeout,
|
||||
)
|
||||
from app.schemas.attachment import Attachment
|
||||
from app.schemas.user import User
|
||||
from app.storage import attachments as repository
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CleanupResult:
|
||||
examined: int
|
||||
candidates: int
|
||||
deleted: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OriginalFile:
|
||||
path: Path
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
|
||||
|
||||
def require_storage(storage: MinioStorage | None) -> MinioStorage:
|
||||
if storage is None:
|
||||
raise unavailable("Объектное хранилище не настроено")
|
||||
return storage
|
||||
|
||||
|
||||
def validate_declared_size(value: str | None, maximum: int) -> None:
|
||||
if value is not None:
|
||||
if not value.isascii() or not value.isdecimal() or len(value) > 20:
|
||||
raise invalid_request("Некорректный Content-Length")
|
||||
if int(value) > maximum:
|
||||
raise payload_too_large()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
|
||||
if not capacity.try_acquire():
|
||||
raise rate_limited()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
capacity.release()
|
||||
|
||||
|
||||
async def receive_attachment(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage | None,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
capacity: RequestCapacity,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
declared_length: str | None,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
storage = require_storage(storage)
|
||||
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
|
||||
timeout_error = invalid_request("Истекло время загрузки")
|
||||
with upload_slot(capacity):
|
||||
async with operation_timeout(
|
||||
storage.settings.upload_timeout_seconds, timeout_error
|
||||
):
|
||||
result = await upload_attachment(
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
chunks=chunks,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def read_original(
|
||||
chunks: AsyncIterator[bytes],
|
||||
path: Path,
|
||||
maximum: int,
|
||||
chunk_bytes: int,
|
||||
) -> OriginalFile:
|
||||
checksum = hashlib.sha256()
|
||||
size = 0
|
||||
with path.open("wb") as target:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > maximum:
|
||||
raise payload_too_large()
|
||||
checksum.update(chunk)
|
||||
view = memoryview(chunk)
|
||||
for offset in range(0, len(view), chunk_bytes):
|
||||
target.write(view[offset : offset + chunk_bytes])
|
||||
if size == 0:
|
||||
raise unsupported_media_type("Пустой файл")
|
||||
return OriginalFile(path, size, checksum.hexdigest())
|
||||
|
||||
|
||||
async def compensate_upload(
|
||||
session: AsyncSession, storage: MinioStorage, id: UUID
|
||||
) -> None:
|
||||
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
|
||||
await session.rollback()
|
||||
intent = await repository.lock_upload(session, id)
|
||||
if intent is not None:
|
||||
if await repository.get_by_key(session, intent.object_key) is None:
|
||||
await storage.delete_object(intent.object_key)
|
||||
await repository.finish_upload(session, id)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def recover_upload(
|
||||
session: AsyncSession, storage: MinioStorage, id: UUID
|
||||
) -> None:
|
||||
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
|
||||
try:
|
||||
await compensate_upload(session, storage, id)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error("Upload compensation", error)
|
||||
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
|
||||
with suppress(*RECOVERABLE_STORAGE_ERRORS):
|
||||
await session.rollback()
|
||||
|
||||
|
||||
async def publish_original(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
original: OriginalFile,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
) -> Attachment:
|
||||
id = uuid4()
|
||||
key = f"originals/{uuid4().hex}"
|
||||
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
|
||||
await repository.reserve_upload(session, id, key)
|
||||
await session.commit()
|
||||
try:
|
||||
if await repository.lock_upload(session, id) is None:
|
||||
raise unavailable("Загрузка отменена до записи оригинала")
|
||||
await storage.put_file(
|
||||
key,
|
||||
original.path,
|
||||
size_bytes=original.size_bytes,
|
||||
content_type=content_type,
|
||||
sha256=original.sha256,
|
||||
)
|
||||
record = await repository.create_attachment(
|
||||
session,
|
||||
id=id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user.id,
|
||||
object_key=key,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=original.size_bytes,
|
||||
sha256=original.sha256,
|
||||
)
|
||||
await repository.finish_upload(session, id)
|
||||
result = Attachment.model_validate(record)
|
||||
await session.commit()
|
||||
except BaseException:
|
||||
# Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT.
|
||||
with CancelScope(shield=True):
|
||||
await recover_upload(session, storage, id)
|
||||
raise
|
||||
return result
|
||||
|
||||
|
||||
async def upload_attachment(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
*,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
await ensure_chat_member(session, user, chat_id)
|
||||
validate_filename(filename)
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
|
||||
# Не держим соединение PostgreSQL во время чтения медленного клиента.
|
||||
await session.rollback()
|
||||
settings = storage.settings
|
||||
with TemporaryDirectory(prefix="messenger-upload-") as directory:
|
||||
original = await read_original(
|
||||
chunks,
|
||||
Path(directory) / "original",
|
||||
settings.max_upload_bytes,
|
||||
settings.chunk_bytes,
|
||||
)
|
||||
await validate_image(original.path, content_type, settings)
|
||||
result = await publish_original(
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
original=original,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def get_attachment(
|
||||
session: AsyncSession, user: User, attachment_id: UUID
|
||||
) -> Attachment:
|
||||
record = await repository.get_attachment(session, attachment_id)
|
||||
if record is None:
|
||||
raise not_found()
|
||||
await ensure_chat_member(session, user, record.chat_id)
|
||||
return Attachment.model_validate(record)
|
||||
|
||||
|
||||
async def download_attachment(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage | None,
|
||||
user: User,
|
||||
attachment_id: UUID,
|
||||
*,
|
||||
variant: str = "original",
|
||||
) -> tuple[Attachment, AsyncIterator[bytes]]:
|
||||
result = await get_attachment(session, user, attachment_id)
|
||||
if variant != "original":
|
||||
raise not_found()
|
||||
storage = require_storage(storage)
|
||||
record = await repository.get_attachment(session, attachment_id)
|
||||
if record is None:
|
||||
raise not_found()
|
||||
key = record.object_key
|
||||
await session.rollback()
|
||||
info = await storage.stat_object(key)
|
||||
if (
|
||||
info is None
|
||||
or info.size_bytes != result.size_bytes
|
||||
or info.content_type != result.content_type
|
||||
):
|
||||
raise unavailable("Оригинал временно недоступен")
|
||||
stream = await storage.open_object(key)
|
||||
return result, stream
|
||||
|
||||
|
||||
async def validate_message_attachments(
|
||||
session: AsyncSession,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
attachment_ids: list[UUID],
|
||||
) -> None:
|
||||
await ensure_chat_member(session, user, chat_id)
|
||||
records = await repository.get_for_chat(session, chat_id, attachment_ids)
|
||||
if len(records) != len(attachment_ids):
|
||||
raise not_found()
|
||||
if any(record.status != "ready" for record in records):
|
||||
raise conflict("Вложение ещё не готово")
|
||||
|
||||
|
||||
async def reconcile_attachments(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
*,
|
||||
batch_size: int = 20,
|
||||
dry_run: bool = True,
|
||||
) -> CleanupResult:
|
||||
if not 1 <= batch_size <= 100:
|
||||
raise invalid_request("batch_size должен быть от 1 до 100")
|
||||
intents = await repository.stale_uploads(
|
||||
session, storage.settings.cleanup_grace_seconds, batch_size
|
||||
)
|
||||
candidates = deleted = 0
|
||||
for intent in intents:
|
||||
if await repository.get_by_key(session, intent.object_key) is None:
|
||||
candidates += 1
|
||||
if not dry_run:
|
||||
await storage.delete_object(intent.object_key)
|
||||
deleted += 1
|
||||
if not dry_run:
|
||||
await repository.finish_upload(session, intent.id)
|
||||
if dry_run:
|
||||
await session.rollback()
|
||||
else:
|
||||
await session.commit()
|
||||
return CleanupResult(len(intents), candidates, deleted)
|
||||
@@ -0,0 +1,84 @@
|
||||
import re
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core import security
|
||||
from app.domain.users import to_user_schema
|
||||
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
|
||||
from app.schemas.session import AuthSession
|
||||
from app.schemas.user import (
|
||||
PASSWORD_MAX_LENGTH,
|
||||
PASSWORD_MIN_LENGTH,
|
||||
USERNAME_PATTERN,
|
||||
User,
|
||||
)
|
||||
from app.storage import sessions as sessions_storage
|
||||
from app.storage import users as users_storage
|
||||
|
||||
USERNAME_RE = re.compile(USERNAME_PATTERN)
|
||||
|
||||
|
||||
async def create_session(
|
||||
session: AsyncSession,
|
||||
redis: Redis,
|
||||
username: str,
|
||||
password: str,
|
||||
ttl_seconds: int,
|
||||
) -> AuthSession:
|
||||
"""Проверяет учётные данные и выдаёт новый Bearer-токен."""
|
||||
if (
|
||||
USERNAME_RE.fullmatch(username) is None
|
||||
or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH
|
||||
or not password.strip()
|
||||
):
|
||||
raise unauthorized_basic()
|
||||
|
||||
user = await users_storage.get_user_by_username(session, username)
|
||||
await session.commit()
|
||||
|
||||
if user is None:
|
||||
await security.dummy_verify_password(password)
|
||||
raise unauthorized_basic()
|
||||
|
||||
if not await security.verify_password(password, user.password_hash):
|
||||
raise unauthorized_basic()
|
||||
|
||||
token = security.generate_token()
|
||||
expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
|
||||
await sessions_storage.create_session(
|
||||
redis,
|
||||
token_hash=security.hash_token(token),
|
||||
user_id=user.id,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
auth_session = AuthSession(
|
||||
token=token, expires_at=expires_at, user=to_user_schema(user)
|
||||
)
|
||||
return auth_session
|
||||
|
||||
|
||||
async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User:
|
||||
"""Возвращает владельца действующей сессии или бросает 401."""
|
||||
user_id = await sessions_storage.get_session_user_id(
|
||||
redis, security.hash_token(token)
|
||||
)
|
||||
if user_id is None:
|
||||
raise unauthorized_bearer()
|
||||
|
||||
user = await users_storage.get_user_by_id(session, user_id)
|
||||
if user is None:
|
||||
raise unauthorized_bearer()
|
||||
|
||||
current_user = to_user_schema(user)
|
||||
return current_user
|
||||
|
||||
|
||||
async def revoke_session(redis: Redis, token: str) -> None:
|
||||
"""Отзывает текущую сессию; повторный отзыв даёт 401."""
|
||||
is_deleted = await sessions_storage.delete_session(
|
||||
redis, security.hash_token(token)
|
||||
)
|
||||
if not is_deleted:
|
||||
raise unauthorized_bearer()
|
||||
@@ -0,0 +1,104 @@
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.domain.pagination import (
|
||||
KIND_CHATS,
|
||||
build_next_cursor,
|
||||
decode_cursor,
|
||||
split_page,
|
||||
)
|
||||
from app.errors.exceptions import invalid_request, not_found
|
||||
from app.models.chat import Chat as ChatModel
|
||||
from app.schemas.chat import Chat, ChatPage, CreateChat
|
||||
from app.schemas.user import User
|
||||
from app.storage import chats as chats_storage
|
||||
from app.storage import users as users_storage
|
||||
from app.storage.types import Position
|
||||
|
||||
UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя"
|
||||
|
||||
|
||||
def to_chat_schema(chat: ChatModel) -> Chat:
|
||||
"""Преобразует ORM-модель чата с загруженными участниками в схему."""
|
||||
chat_schema = Chat(
|
||||
id=chat.id,
|
||||
title=chat.title,
|
||||
member_ids=[member.user_id for member in chat.members],
|
||||
created_at=chat.created_at,
|
||||
)
|
||||
return chat_schema
|
||||
|
||||
|
||||
def validate_new_chat(creator: User, new_chat: CreateChat) -> None:
|
||||
"""Проверяет состав участников без обращения к базе."""
|
||||
member_ids = new_chat.member_ids
|
||||
|
||||
if creator.id not in member_ids:
|
||||
raise invalid_request("Создатель чата должен входить в member_ids")
|
||||
|
||||
if len(set(member_ids)) != len(member_ids):
|
||||
raise invalid_request("member_ids содержит дубликаты")
|
||||
|
||||
|
||||
async def create_chat(
|
||||
session: AsyncSession, creator: User, new_chat: CreateChat
|
||||
) -> Chat:
|
||||
"""Создаёт чат с участниками атомарно; неверный состав даёт 400."""
|
||||
validate_new_chat(creator, new_chat)
|
||||
|
||||
existing_ids = await users_storage.get_existing_user_ids(
|
||||
session, new_chat.member_ids
|
||||
)
|
||||
if existing_ids != set(new_chat.member_ids):
|
||||
raise invalid_request(UNKNOWN_MEMBER_MESSAGE)
|
||||
|
||||
try:
|
||||
chat = await chats_storage.create_chat(
|
||||
session, title=new_chat.title, member_ids=new_chat.member_ids
|
||||
)
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None
|
||||
|
||||
created_chat = to_chat_schema(chat)
|
||||
return created_chat
|
||||
|
||||
|
||||
async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat:
|
||||
"""Возвращает чат участнику; чужой и несуществующий чат дают 404."""
|
||||
chat = await chats_storage.get_chat_for_member(session, chat_id, user.id)
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
|
||||
found_chat = to_chat_schema(chat)
|
||||
return found_chat
|
||||
|
||||
|
||||
async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None:
|
||||
"""Бросает 404, если пользователь не состоит в чате или чата нет."""
|
||||
if not await chats_storage.is_chat_member(session, chat_id, user.id):
|
||||
raise not_found()
|
||||
|
||||
|
||||
async def list_chats(
|
||||
session: AsyncSession,
|
||||
user: User,
|
||||
cursor: str | None,
|
||||
limit: int,
|
||||
) -> ChatPage:
|
||||
"""Возвращает страницу чатов пользователя по (created_at, id)."""
|
||||
after: Position | None = None
|
||||
if cursor is not None:
|
||||
after = decode_cursor(cursor, KIND_CHATS, user.id)
|
||||
|
||||
rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1)
|
||||
page_rows, has_next = split_page(rows, limit)
|
||||
items = [to_chat_schema(chat) for chat in page_rows]
|
||||
page = ChatPage(
|
||||
items=items,
|
||||
next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next),
|
||||
)
|
||||
return page
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Условия готовности реплики и ограниченная проверка зависимостей."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from app.core.lifecycle import Lifecycle
|
||||
from app.errors.exceptions import unavailable
|
||||
from app.errors.handlers import operation_timeout
|
||||
from app.schemas.base import Health
|
||||
from app.storage.health import check_database, check_redis
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
def liveness() -> Health:
|
||||
return Health(status='ok')
|
||||
|
||||
|
||||
async def readiness(
|
||||
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
|
||||
lifecycle: Lifecycle, timeout_seconds: float,
|
||||
) -> Health:
|
||||
if lifecycle.draining:
|
||||
raise unavailable()
|
||||
checks = [check_database(engine), check_redis(redis)]
|
||||
if storage is not None:
|
||||
checks.append(storage.check_available())
|
||||
async with operation_timeout(timeout_seconds, unavailable()):
|
||||
dependencies = await asyncio.gather(*checks, return_exceptions=True)
|
||||
if not all(result is True for result in dependencies):
|
||||
raise unavailable()
|
||||
return Health(status='ok')
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Атомарная отправка сообщения с защитой от повторов."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.domain.chats import ensure_chat_member
|
||||
from app.domain.attachments import validate_message_attachments
|
||||
from app.errors.exceptions import conflict, invalid_request, unavailable
|
||||
from app.schemas.message import CreateMessage, Message
|
||||
from app.schemas.user import User
|
||||
from app.storage import idempotency as key_storage
|
||||
from app.storage import messages as message_storage
|
||||
from app.storage import attachments as attachment_storage
|
||||
|
||||
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def validate_idempotency_key(key: str | None) -> str:
|
||||
"""Проверяет обязательный ASCII-ключ запроса."""
|
||||
if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None:
|
||||
raise invalid_request("Требуется корректный Idempotency-Key")
|
||||
return key
|
||||
|
||||
|
||||
def request_fingerprint(new_message: CreateMessage) -> str:
|
||||
"""Хеширует нормализованное содержимое запроса."""
|
||||
body = new_message.model_dump(mode="json")
|
||||
# Сохраняет отпечатки текстовых запросов lab3 и равенство absent / [].
|
||||
if not body["attachment_ids"]:
|
||||
body.pop("attachment_ids")
|
||||
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha256(canonical.encode()).hexdigest()
|
||||
|
||||
|
||||
async def send_message_idempotent(
|
||||
session: AsyncSession,
|
||||
sender: User,
|
||||
chat_id: UUID,
|
||||
new_message: CreateMessage,
|
||||
idempotency_key: str,
|
||||
max_messages: int,
|
||||
ttl_seconds: int,
|
||||
) -> Message:
|
||||
"""Создаёт сообщение или возвращает результат прежнего запроса."""
|
||||
await ensure_chat_member(session, sender, chat_id)
|
||||
scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key)
|
||||
fingerprint = request_fingerprint(new_message)
|
||||
|
||||
await key_storage.lock_key(session, scope)
|
||||
previous = await key_storage.get_active_key(session, scope, ttl_seconds)
|
||||
if previous is not None:
|
||||
if previous.request_hash != fingerprint:
|
||||
raise conflict("Idempotency-Key уже использован для другого сообщения")
|
||||
message = await message_storage.get_message(session, previous.message_id)
|
||||
else:
|
||||
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
|
||||
if not await message_storage.reserve_message_slot(session, max_messages):
|
||||
raise unavailable("Достигнут лимит количества сообщений")
|
||||
message = await message_storage.create_message(
|
||||
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
|
||||
)
|
||||
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
|
||||
await key_storage.save_result(session, scope, fingerprint, message.id)
|
||||
|
||||
links = await attachment_storage.message_attachment_ids(session, [message.id])
|
||||
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
|
||||
await session.commit()
|
||||
return result
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Синхронная проверка изображения; вызывается только из ограниченного процесса."""
|
||||
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app.errors.exceptions import payload_too_large, unsupported_media_type
|
||||
|
||||
|
||||
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(path, formats=["PNG", "JPEG"]) as image:
|
||||
if image.width * image.height > max_pixels:
|
||||
raise payload_too_large("Превышен лимит пикселей изображения")
|
||||
image_format = image.format
|
||||
if image_format is None or Image.MIME.get(image_format) != content_type:
|
||||
raise unsupported_media_type("Содержимое не соответствует Content-Type")
|
||||
image.verify()
|
||||
# Для JPEG одной verify() недостаточно: декодировать весь оригинал.
|
||||
with Image.open(path, formats=["PNG", "JPEG"]) as image:
|
||||
image.load()
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Проверка имени и запуск ограниченного декодера."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
from anyio import CancelScope
|
||||
|
||||
from app.core.s3 import S3Settings
|
||||
from app.errors.exceptions import (
|
||||
image_validation_error,
|
||||
invalid_request,
|
||||
payload_too_large,
|
||||
)
|
||||
from app.errors.handlers import operation_timeout
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255 or any(
|
||||
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
|
||||
for char in filename
|
||||
):
|
||||
raise invalid_request("Недопустимое имя файла")
|
||||
|
||||
|
||||
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"app.core.image_worker",
|
||||
str(path),
|
||||
content_type,
|
||||
str(settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
error = payload_too_large("Превышен бюджет проверки изображения")
|
||||
try:
|
||||
async with operation_timeout(settings.image_timeout_seconds, error):
|
||||
code = await process.wait()
|
||||
finally:
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
with CancelScope(shield=True):
|
||||
await process.wait()
|
||||
if code != 0:
|
||||
raise image_validation_error(code)
|
||||
@@ -0,0 +1,43 @@
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.domain.chats import ensure_chat_member
|
||||
from app.domain.pagination import (
|
||||
KIND_MESSAGES,
|
||||
build_next_cursor,
|
||||
decode_cursor,
|
||||
split_page,
|
||||
)
|
||||
from app.schemas.message import Message, MessagePage
|
||||
from app.schemas.user import User
|
||||
from app.storage import messages as messages_storage
|
||||
from app.storage import attachments as attachments_storage
|
||||
from app.storage.types import Position
|
||||
|
||||
|
||||
async def list_messages(
|
||||
session: AsyncSession,
|
||||
user: User,
|
||||
chat_id: UUID,
|
||||
cursor: str | None,
|
||||
limit: int,
|
||||
) -> MessagePage:
|
||||
"""Возвращает страницу истории чата по (created_at, id)."""
|
||||
after: Position | None = None
|
||||
if cursor is not None:
|
||||
after = decode_cursor(cursor, KIND_MESSAGES, chat_id)
|
||||
|
||||
await ensure_chat_member(session, user, chat_id)
|
||||
|
||||
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
|
||||
page_rows, has_next = split_page(rows, limit)
|
||||
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
|
||||
items = [Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]},
|
||||
) for message in page_rows]
|
||||
page = MessagePage(
|
||||
items=items,
|
||||
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
|
||||
)
|
||||
return page
|
||||
@@ -0,0 +1,83 @@
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from typing import Protocol, TypeVar
|
||||
from uuid import UUID
|
||||
|
||||
from app.errors.exceptions import invalid_request
|
||||
from app.storage.types import Position
|
||||
|
||||
KIND_CHATS = "chats"
|
||||
KIND_MESSAGES = "messages"
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class Positioned(Protocol):
|
||||
"""Объект с позицией в порядке (created_at, id)."""
|
||||
|
||||
@property
|
||||
def id(self) -> UUID: ...
|
||||
|
||||
@property
|
||||
def created_at(self) -> datetime: ...
|
||||
|
||||
|
||||
def encode_cursor(
|
||||
kind: str,
|
||||
resource_id: UUID,
|
||||
created_at: datetime,
|
||||
item_id: UUID,
|
||||
) -> str:
|
||||
"""Кодирует позицию и ресурс, к которому она привязана, в непрозрачный cursor."""
|
||||
payload = {
|
||||
"kind": kind,
|
||||
"resource_id": str(resource_id),
|
||||
"created_at": created_at.isoformat(),
|
||||
"item_id": str(item_id),
|
||||
}
|
||||
raw = json.dumps(payload).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position:
|
||||
"""Возвращает позицию из cursor; битый или чужой cursor даёт 400."""
|
||||
try:
|
||||
payload = json.loads(base64.urlsafe_b64decode(cursor))
|
||||
cursor_kind = payload["kind"]
|
||||
cursor_resource_id = UUID(payload["resource_id"])
|
||||
created_at = datetime.fromisoformat(payload["created_at"])
|
||||
item_id = UUID(payload["item_id"])
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
raise invalid_request("Некорректный cursor") from None
|
||||
|
||||
if (
|
||||
cursor_kind != kind
|
||||
or cursor_resource_id != resource_id
|
||||
or created_at.tzinfo is None
|
||||
):
|
||||
raise invalid_request("Некорректный cursor")
|
||||
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]:
|
||||
"""Отделяет страницу от лишней строки limit + 1, по которой видно продолжение."""
|
||||
has_next = len(rows) > limit
|
||||
items = rows[:limit]
|
||||
return items, has_next
|
||||
|
||||
|
||||
def build_next_cursor(
|
||||
kind: str,
|
||||
resource_id: UUID,
|
||||
items: Sequence[Positioned],
|
||||
has_next: bool,
|
||||
) -> str | None:
|
||||
"""Возвращает cursor на последний элемент страницы или None, если она последняя."""
|
||||
if not has_next:
|
||||
return None
|
||||
|
||||
last_item = items[-1]
|
||||
return encode_cursor(kind, resource_id, last_item.created_at, last_item.id)
|
||||
@@ -0,0 +1,41 @@
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core import security
|
||||
from app.errors.exceptions import conflict
|
||||
from app.models.user import User as UserModel
|
||||
from app.schemas.user import CreateUser, User
|
||||
from app.storage import users as users_storage
|
||||
|
||||
USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято"
|
||||
|
||||
|
||||
def to_user_schema(user: UserModel) -> User:
|
||||
"""Преобразует ORM-модель в публичную схему без password_hash."""
|
||||
user_schema = User.model_validate(user)
|
||||
return user_schema
|
||||
|
||||
|
||||
async def register_user(session: AsyncSession, new_user: CreateUser) -> User:
|
||||
"""Регистрирует пользователя; занятый username даёт 409."""
|
||||
existing_user = await users_storage.get_user_by_username(session, new_user.username)
|
||||
await session.commit()
|
||||
if existing_user is not None:
|
||||
raise conflict(USERNAME_TAKEN_MESSAGE)
|
||||
|
||||
password_hash = await security.hash_password(new_user.password)
|
||||
|
||||
try:
|
||||
user = await users_storage.create_user(
|
||||
session,
|
||||
username=new_user.username,
|
||||
display_name=new_user.display_name,
|
||||
password_hash=password_hash,
|
||||
)
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
raise conflict(USERNAME_TAKEN_MESSAGE) from None
|
||||
|
||||
created_user = to_user_schema(user)
|
||||
return created_user
|
||||
@@ -0,0 +1,52 @@
|
||||
"""HTTP-параметры и упаковка ответов; сценарии находятся в domain."""
|
||||
|
||||
from typing import Annotated
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
from app.domain import attachments
|
||||
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
|
||||
from app.schemas.attachment import Attachment
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=201)
|
||||
async def upload(
|
||||
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
|
||||
storage: S3Storage, capacity: UploadCapacity,
|
||||
filename: Annotated[str, Query(min_length=1, max_length=255)],
|
||||
) -> Attachment:
|
||||
return await attachments.receive_attachment(
|
||||
session, storage, user, chat_id, capacity=capacity, filename=filename,
|
||||
content_type=request.headers.get('content-type', '').lower(),
|
||||
declared_length=request.headers.get('content-length'), chunks=request.stream(),
|
||||
)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}')
|
||||
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
|
||||
return await attachments.get_attachment(session, user, attachment_id)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
async def content(
|
||||
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
|
||||
variant: Annotated[str, Query()] = 'original',
|
||||
) -> StreamingResponse:
|
||||
record, stream = await attachments.download_attachment(
|
||||
session, storage, user, attachment_id, variant=variant,
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream, media_type=record.content_type,
|
||||
headers={
|
||||
'Content-Length': str(record.size_bytes),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
|
||||
},
|
||||
background=BackgroundTask(stream.aclose),
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
from fastapi import APIRouter, Response
|
||||
|
||||
from app.domain.auth import create_session, revoke_session
|
||||
from app.endpoints.dependencies import (
|
||||
AppSettings,
|
||||
BasicCredentials,
|
||||
BearerToken,
|
||||
DbSession,
|
||||
RedisClient,
|
||||
)
|
||||
from app.schemas.session import AuthSession
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
|
||||
|
||||
|
||||
@router.post("/sessions", status_code=201)
|
||||
async def login(
|
||||
credentials: BasicCredentials,
|
||||
session: DbSession,
|
||||
redis: RedisClient,
|
||||
settings: AppSettings,
|
||||
) -> AuthSession:
|
||||
username, password = credentials
|
||||
return await create_session(
|
||||
session, redis, username, password, settings.session_ttl_seconds
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/sessions/current", status_code=204)
|
||||
async def logout(token: BearerToken, redis: RedisClient) -> Response:
|
||||
await revoke_session(redis, token)
|
||||
return Response(status_code=204)
|
||||
@@ -0,0 +1,32 @@
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
|
||||
from app.domain.chats import create_chat, get_chat, list_chats
|
||||
from app.endpoints.dependencies import CurrentUser, DbSession
|
||||
from app.schemas.chat import Chat, ChatPage, CreateChat
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_new_chat(
|
||||
new_chat: CreateChat, creator: CurrentUser, session: DbSession
|
||||
) -> Chat:
|
||||
return await create_chat(session, creator, new_chat)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_user_chats(
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 50,
|
||||
cursor: Annotated[str | None, Query(min_length=1)] = None,
|
||||
) -> ChatPage:
|
||||
return await list_chats(session, user, cursor, limit)
|
||||
|
||||
|
||||
@router.get("/{chat_id}")
|
||||
async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat:
|
||||
return await get_chat(session, user, chat_id)
|
||||
@@ -0,0 +1,112 @@
|
||||
import base64
|
||||
import binascii
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, Request, Security
|
||||
from fastapi.openapi.models import HTTPBase, HTTPBearer
|
||||
from fastapi.security.base import SecurityBase
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.request_capacity import RequestCapacity
|
||||
from app.domain.auth import authenticate_token
|
||||
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
|
||||
from app.schemas.user import User
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
async def get_settings(request: Request) -> Settings:
|
||||
settings: Settings = request.app.state.settings
|
||||
return settings
|
||||
|
||||
|
||||
async def get_db(request: Request) -> AsyncIterator[AsyncSession]:
|
||||
"""Одна AsyncSession на HTTP-запрос; незакоммиченная транзакция откатывается."""
|
||||
session_factory = request.app.state.session_factory
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def get_redis(request: Request) -> Redis:
|
||||
redis: Redis = request.app.state.redis
|
||||
return redis
|
||||
|
||||
|
||||
def get_s3(request: Request) -> MinioStorage | None:
|
||||
return request.app.state.s3
|
||||
|
||||
|
||||
def get_upload_capacity(request: Request) -> RequestCapacity:
|
||||
return request.app.state.upload_capacity
|
||||
|
||||
|
||||
class AuthorizationHeader(SecurityBase):
|
||||
"""Отдаёт сырой заголовок Authorization и описывает схему для OpenAPI."""
|
||||
|
||||
def __init__(self, model: HTTPBase, scheme_name: str) -> None:
|
||||
self.model = model
|
||||
self.scheme_name = scheme_name
|
||||
|
||||
async def __call__(self, request: Request) -> str | None:
|
||||
return request.headers.get("Authorization")
|
||||
|
||||
|
||||
basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth")
|
||||
bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth")
|
||||
|
||||
|
||||
def _split_authorization(authorization: str | None) -> tuple[str, str] | None:
|
||||
if authorization is None:
|
||||
return None
|
||||
scheme, _, value = authorization.partition(" ")
|
||||
if not value.strip():
|
||||
return None
|
||||
return scheme.lower(), value.strip()
|
||||
|
||||
|
||||
async def get_bearer_token(
|
||||
authorization: Annotated[str | None, Security(bearer_auth)],
|
||||
) -> str:
|
||||
parts = _split_authorization(authorization)
|
||||
if parts is None or parts[0] != "bearer":
|
||||
raise unauthorized_bearer()
|
||||
return parts[1]
|
||||
|
||||
|
||||
async def get_basic_credentials(
|
||||
authorization: Annotated[str | None, Security(basic_auth)],
|
||||
) -> tuple[str, str]:
|
||||
"""(username, password) из `Authorization: Basic`; пароль может содержать ':'."""
|
||||
parts = _split_authorization(authorization)
|
||||
if parts is None or parts[0] != "basic":
|
||||
raise unauthorized_basic()
|
||||
|
||||
try:
|
||||
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError, ValueError):
|
||||
raise unauthorized_basic() from None
|
||||
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator or not username:
|
||||
raise unauthorized_basic()
|
||||
return username, password
|
||||
|
||||
|
||||
DbSession = Annotated[AsyncSession, Depends(get_db)]
|
||||
RedisClient = Annotated[Redis, Depends(get_redis)]
|
||||
AppSettings = Annotated[Settings, Depends(get_settings)]
|
||||
S3Storage = Annotated[MinioStorage | None, Depends(get_s3)]
|
||||
UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)]
|
||||
BearerToken = Annotated[str, Depends(get_bearer_token)]
|
||||
BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)]
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
token: BearerToken, session: DbSession, redis: RedisClient
|
||||
) -> User:
|
||||
return await authenticate_token(session, redis, token)
|
||||
|
||||
|
||||
CurrentUser = Annotated[User, Depends(get_current_user)]
|
||||
@@ -0,0 +1,20 @@
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.domain import health
|
||||
from app.schemas.base import Health
|
||||
|
||||
router = APIRouter(prefix='/health', tags=['health'])
|
||||
|
||||
|
||||
@router.get('/live')
|
||||
async def liveness() -> Health:
|
||||
return health.liveness()
|
||||
|
||||
|
||||
@router.get('/ready')
|
||||
async def readiness(request: Request) -> Health:
|
||||
state = request.app.state
|
||||
return await health.readiness(
|
||||
state.engine, state.redis, state.s3, state.lifecycle,
|
||||
state.settings.readiness_timeout_seconds,
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Header, Query
|
||||
|
||||
from app.domain.idempotency import send_message_idempotent, validate_idempotency_key
|
||||
from app.domain.messages import list_messages
|
||||
from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession
|
||||
from app.schemas.message import CreateMessage, Message, MessagePage
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
|
||||
|
||||
|
||||
@router.post("/{chat_id}/messages", status_code=201)
|
||||
async def send_new_message(
|
||||
chat_id: UUID,
|
||||
new_message: CreateMessage,
|
||||
sender: CurrentUser,
|
||||
session: DbSession,
|
||||
settings: AppSettings,
|
||||
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
|
||||
) -> Message:
|
||||
return await send_message_idempotent(
|
||||
session, sender, chat_id, new_message,
|
||||
validate_idempotency_key(idempotency_key),
|
||||
settings.max_messages, settings.idempotency_ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{chat_id}/messages")
|
||||
async def get_history_messages(
|
||||
chat_id: UUID,
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 50,
|
||||
cursor: Annotated[str | None, Query(min_length=1)] = None,
|
||||
) -> MessagePage:
|
||||
return await list_messages(session, user, chat_id, cursor, limit)
|
||||
@@ -0,0 +1,17 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.domain.users import register_user
|
||||
from app.endpoints.dependencies import CurrentUser, DbSession
|
||||
from app.schemas.user import CreateUser, User
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_user(new_user: CreateUser, session: DbSession) -> User:
|
||||
return await register_user(session, new_user)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def get_me(user: CurrentUser) -> User:
|
||||
return user
|
||||
@@ -0,0 +1,109 @@
|
||||
from collections.abc import Mapping
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers
|
||||
|
||||
|
||||
def invalid_request(message: str = "Неверный запрос") -> ApiError:
|
||||
"""Создаёт ошибку некорректного или недопустимого запроса."""
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(
|
||||
message: str = "Нет действительной аутентификации",
|
||||
www_authenticate: str | None = None,
|
||||
) -> ApiError:
|
||||
"""Создаёт ошибку отсутствующей или недействительной аутентификации."""
|
||||
headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None
|
||||
return ApiError(401, "unauthorized", message, headers)
|
||||
|
||||
|
||||
def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError:
|
||||
"""401 для POST /auth/sessions (одинаковый для неверного username и пароля)."""
|
||||
return unauthorized(message, BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError:
|
||||
"""401 для бизнес-endpoint: нет, истёк, отозван или неизвестен токен."""
|
||||
return unauthorized(message, BEARER_CHALLENGE)
|
||||
|
||||
|
||||
def not_found(message: str = "Объект не найден") -> ApiError:
|
||||
"""Создаёт ошибку, указывающую на отсутствие запрошенного ресурса."""
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str = "Конфликт") -> ApiError:
|
||||
"""Создаёт ошибку конфликта с текущим состоянием ресурса."""
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError:
|
||||
"""Создаёт ошибку, указывающую на превышение допустимого размера запроса."""
|
||||
return ApiError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported_media_type(
|
||||
message: str = "Требуется Content-Type: application/json",
|
||||
) -> ApiError:
|
||||
"""Создаёт ошибку неподдерживаемого формата или типа содержимого запроса."""
|
||||
return ApiError(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
def rate_limited(retry_after: int = 1) -> ApiError:
|
||||
"""Создаёт ошибку превышения допустимой частоты запросов."""
|
||||
return ApiError(
|
||||
429,
|
||||
"rate_limited",
|
||||
"Слишком много запросов",
|
||||
{"Retry-After": str(max(1, retry_after))},
|
||||
)
|
||||
|
||||
|
||||
def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
|
||||
"""Создаёт ошибку временной недоступности сервиса."""
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def internal_error() -> ApiError:
|
||||
"""Создаёт ошибку внутренней ошибки сервера."""
|
||||
return ApiError(500, "internal_error", "Внутренняя ошибка сервера")
|
||||
|
||||
|
||||
IMAGE_WORKER_ERRORS = (
|
||||
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
|
||||
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
|
||||
)
|
||||
|
||||
|
||||
def image_worker_error_code(error: Exception) -> int:
|
||||
code = 2
|
||||
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
|
||||
declared_limit = isinstance(error, ApiError) and error.status_code == 413
|
||||
if resource_error or declared_limit:
|
||||
code = 3
|
||||
return code
|
||||
|
||||
|
||||
def image_validation_error(code: int) -> ApiError:
|
||||
error = unsupported_media_type("Повреждённое изображение или несовпадение MIME")
|
||||
if code == 3 or code < 0:
|
||||
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
|
||||
return error
|
||||
@@ -0,0 +1,225 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from collections.abc import Mapping
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from minio.error import MinioException
|
||||
from redis.exceptions import ConnectionError as RedisConnectionError
|
||||
from redis.exceptions import TimeoutError as RedisTimeoutError
|
||||
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from starlette.requests import ClientDisconnect
|
||||
from urllib3.exceptions import HTTPError as StorageHTTPError
|
||||
|
||||
from app.core.instance import INSTANCE_ID_HEADER
|
||||
from app.core.request_id import REQUEST_ID_HEADER
|
||||
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
|
||||
TRANSIENT_SQLSTATES = frozenset({"55P03"})
|
||||
RECOVERABLE_STORAGE_ERRORS = (
|
||||
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def operation_timeout(seconds: float, error: ApiError):
|
||||
"""Единое преобразование таймаутов; владелец операции задаёт ожидаемую ошибку."""
|
||||
try:
|
||||
async with asyncio.timeout(seconds):
|
||||
yield
|
||||
except TimeoutError:
|
||||
raise error from None
|
||||
|
||||
|
||||
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
|
||||
|
||||
|
||||
def log_recoverable_error(operation: str, error: Exception) -> None:
|
||||
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
|
||||
logger.warning('%s deferred: %s', operation, type(error).__name__)
|
||||
|
||||
|
||||
def _get_request_id(request: Request) -> str:
|
||||
request_id = getattr(request.state, "request_id", None)
|
||||
|
||||
if request_id is None:
|
||||
request_id = str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
|
||||
return request_id
|
||||
|
||||
|
||||
def _build_response(
|
||||
request: Request,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
) -> JSONResponse:
|
||||
request_id = _get_request_id(request)
|
||||
|
||||
response_headers = {
|
||||
**(headers or {}),
|
||||
REQUEST_ID_HEADER: request_id,
|
||||
}
|
||||
instance_id = getattr(request.app.state, "instance_id", None)
|
||||
if instance_id is not None:
|
||||
response_headers[INSTANCE_ID_HEADER] = instance_id
|
||||
|
||||
body = {
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
}
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content=body,
|
||||
headers=response_headers,
|
||||
)
|
||||
|
||||
|
||||
async def api_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
assert isinstance(exc, ApiError)
|
||||
|
||||
return _build_response(
|
||||
request,
|
||||
exc.status_code,
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.headers,
|
||||
)
|
||||
|
||||
|
||||
async def routing_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
assert isinstance(exc, StarletteHTTPException)
|
||||
|
||||
error = invalid_request()
|
||||
if exc.status_code == 404:
|
||||
error = not_found()
|
||||
return _build_response(
|
||||
request,
|
||||
exc.status_code,
|
||||
error.code,
|
||||
error.message,
|
||||
exc.headers,
|
||||
)
|
||||
|
||||
|
||||
async def validation_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
assert isinstance(exc, RequestValidationError)
|
||||
|
||||
fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()})
|
||||
message = "Неверный запрос"
|
||||
|
||||
if fields:
|
||||
message = f"{message}: {', '.join(fields)}"
|
||||
|
||||
return _build_response(
|
||||
request,
|
||||
400,
|
||||
"invalid_request",
|
||||
message,
|
||||
)
|
||||
|
||||
|
||||
async def dependency_unavailable_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
"""PostgreSQL/Redis/S3 недоступны либо исчерпан пул: 503 без внутренних деталей."""
|
||||
request_id = _get_request_id(request)
|
||||
logger.error(
|
||||
"Dependency unavailable request_id=%s: %s",
|
||||
request_id,
|
||||
type(exc).__name__,
|
||||
)
|
||||
|
||||
error = unavailable()
|
||||
return _build_response(
|
||||
request,
|
||||
error.status_code,
|
||||
error.code,
|
||||
error.message,
|
||||
{"Retry-After": "1"},
|
||||
)
|
||||
|
||||
|
||||
def _is_transient_database_error(exc: DBAPIError) -> bool:
|
||||
sqlstate = getattr(exc.orig, "sqlstate", None) or ""
|
||||
return (
|
||||
sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES)
|
||||
or sqlstate in TRANSIENT_SQLSTATES
|
||||
)
|
||||
|
||||
|
||||
async def database_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
"""Временные ошибки PostgreSQL (таймаут запроса, блокировки, нехватка ресурсов) — 503."""
|
||||
assert isinstance(exc, DBAPIError)
|
||||
|
||||
handler = unhandled_error_handler
|
||||
if _is_transient_database_error(exc):
|
||||
handler = dependency_unavailable_handler
|
||||
return await handler(request, exc)
|
||||
|
||||
|
||||
async def unhandled_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
request_id = _get_request_id(request)
|
||||
|
||||
logger.error(
|
||||
"Unhandled error request_id=%s",
|
||||
request_id,
|
||||
exc_info=exc,
|
||||
)
|
||||
|
||||
error = internal_error()
|
||||
|
||||
return _build_response(
|
||||
request,
|
||||
error.status_code,
|
||||
error.code,
|
||||
error.message,
|
||||
)
|
||||
|
||||
|
||||
def register_exception_handlers(app: FastAPI) -> None:
|
||||
app.add_exception_handler(ApiError, api_error_handler)
|
||||
app.add_exception_handler(ClientDisconnect, client_disconnect_handler)
|
||||
app.add_exception_handler(MinioException, dependency_unavailable_handler)
|
||||
app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(StarletteHTTPException, routing_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_exception_handler(OperationalError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(InterfaceError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(DBAPIError, database_error_handler)
|
||||
app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(OSError, dependency_unavailable_handler)
|
||||
app.add_exception_handler(Exception, unhandled_error_handler)
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
import logging
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from fastapi import Depends, FastAPI
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.instance import InstanceIdMiddleware, resolve_instance_id
|
||||
from app.core.lifecycle import GracefulServer, Lifecycle
|
||||
from app.core.redis import create_redis_client
|
||||
from app.core.request_capacity import acquire_request_slot, create_request_capacity
|
||||
from app.core.request_id import RequestIdMiddleware
|
||||
from app.endpoints.auth import router as auth_router
|
||||
from app.endpoints.attachments import router as attachments_router
|
||||
from app.endpoints.chats import router as chats_router
|
||||
from app.endpoints.health import router as health_router
|
||||
from app.endpoints.messages import router as messages_router
|
||||
from app.endpoints.users import router as users_router
|
||||
from app.errors.handlers import register_exception_handlers
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
def configure_logging(settings: Settings) -> None:
|
||||
logging.basicConfig(
|
||||
level=settings.log_level,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
settings = settings or Settings() # type: ignore
|
||||
configure_logging(settings)
|
||||
|
||||
engine = create_db_engine(settings)
|
||||
redis = create_redis_client(settings)
|
||||
s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI) -> AsyncGenerator[None]:
|
||||
yield
|
||||
if s3 is not None:
|
||||
await s3.aclose()
|
||||
await redis.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan)
|
||||
|
||||
application.state.settings = settings
|
||||
application.state.engine = engine
|
||||
application.state.session_factory = create_session_factory(engine)
|
||||
application.state.redis = redis
|
||||
application.state.s3 = s3
|
||||
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
|
||||
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
|
||||
application.state.instance_id = resolve_instance_id(settings.instance_id)
|
||||
application.state.lifecycle = Lifecycle()
|
||||
application.add_middleware(RequestIdMiddleware)
|
||||
application.add_middleware(
|
||||
InstanceIdMiddleware, instance_id=application.state.instance_id
|
||||
)
|
||||
register_exception_handlers(application)
|
||||
|
||||
limited = [Depends(acquire_request_slot)]
|
||||
application.include_router(health_router)
|
||||
application.include_router(users_router, dependencies=limited)
|
||||
application.include_router(auth_router, dependencies=limited)
|
||||
application.include_router(chats_router, dependencies=limited)
|
||||
application.include_router(messages_router, dependencies=limited)
|
||||
application.include_router(attachments_router, dependencies=limited)
|
||||
|
||||
return application
|
||||
|
||||
|
||||
def main() -> None:
|
||||
settings = Settings() # type: ignore
|
||||
application = create_app(settings)
|
||||
config = uvicorn.Config(
|
||||
application,
|
||||
host=settings.app_host,
|
||||
port=settings.app_port,
|
||||
log_level=settings.log_level.lower(),
|
||||
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
|
||||
)
|
||||
GracefulServer(
|
||||
config, application.state.lifecycle, settings.shutdown_drain_seconds
|
||||
).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
from app.models.base import Base
|
||||
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
|
||||
from app.models.chat import Chat
|
||||
from app.models.chat_member import ChatMember
|
||||
from app.models.counter import Counter
|
||||
from app.models.idempotency_key import IdempotencyKey
|
||||
from app.models.message import Message
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Метаданные оригиналов и журнал операций до записи в S3."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
__table_args__ = (
|
||||
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
|
||||
CheckConstraint(
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
|
||||
),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
|
||||
CheckConstraint("status = 'ready'", name="status"),
|
||||
Index("ix_attachments_chat_id", "chat_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
|
||||
object_key: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int]
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
status: Mapped[str] = mapped_column(String(16), server_default="ready")
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int]
|
||||
|
||||
|
||||
class UploadIntent(Base):
|
||||
"""Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных."""
|
||||
|
||||
__tablename__ = "upload_intents"
|
||||
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
object_key: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
@@ -0,0 +1,27 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from sqlalchemy import DateTime, MetaData
|
||||
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
from sqlalchemy.types import TypeEngine
|
||||
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""Базовый класс ORM-моделей."""
|
||||
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
|
||||
type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = {
|
||||
uuid.UUID: PG_UUID(as_uuid=True),
|
||||
datetime: DateTime(timezone=True),
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.chat_member import ChatMember
|
||||
from app.models.message import Message
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
"""Чат с фиксированным составом участников."""
|
||||
|
||||
__tablename__ = "chats"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
title: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
members: Mapped[list["ChatMember"]] = relationship(
|
||||
back_populates="chat",
|
||||
cascade="all, delete-orphan",
|
||||
passive_deletes=True,
|
||||
lazy="selectin",
|
||||
)
|
||||
messages: Mapped[list["Message"]] = relationship(
|
||||
back_populates="chat", lazy="raise"
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import ForeignKey, Index
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.chat import Chat
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
"""Членство пользователя в чате."""
|
||||
|
||||
__tablename__ = "chat_members"
|
||||
__table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),)
|
||||
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
|
||||
|
||||
chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise")
|
||||
user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise")
|
||||
@@ -0,0 +1,13 @@
|
||||
from sqlalchemy import BigInteger, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class Counter(Base):
|
||||
"""Именованный счётчик с атомарным обновлением."""
|
||||
|
||||
__tablename__ = "counters"
|
||||
|
||||
name: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
value: Mapped[int] = mapped_column(BigInteger)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Сохранённый результат отправки сообщения."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
"""Ключ запроса и связанное сообщение."""
|
||||
|
||||
__tablename__ = "idempotency_keys"
|
||||
__table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),)
|
||||
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
|
||||
method: Mapped[str] = mapped_column(String(8), primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
request_hash: Mapped[str] = mapped_column(String(64))
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id"))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
@@ -0,0 +1,30 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.chat import Chat
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
class Message(Base):
|
||||
"""Сообщение в чате."""
|
||||
|
||||
__tablename__ = "messages"
|
||||
__table_args__ = (
|
||||
Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise")
|
||||
sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise")
|
||||
@@ -0,0 +1,31 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models.chat_member import ChatMember
|
||||
from app.models.message import Message
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Пользователь и его учётные данные."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100))
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
memberships: Mapped[list["ChatMember"]] = relationship(
|
||||
back_populates="user", lazy="raise"
|
||||
)
|
||||
messages: Mapped[list["Message"]] = relationship(
|
||||
back_populates="sender", lazy="raise"
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
from app.schemas.chat import Chat
|
||||
from app.schemas.message import Message
|
||||
from app.schemas.session import AuthSession
|
||||
from app.schemas.user import User
|
||||
|
||||
__all__ = ["AuthSession", "Chat", "Message", "User"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from app.schemas.base import Base
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str = Field(min_length=1, max_length=255)
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int = Field(ge=1, le=10_485_760)
|
||||
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
status: Literal["ready"] = "ready"
|
||||
variants: list[object] = Field(default_factory=list, max_length=0)
|
||||
error_code: None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -0,0 +1,35 @@
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, field_validator
|
||||
|
||||
|
||||
class Base(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
id: UUID
|
||||
|
||||
|
||||
class Health(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
status: Literal["ok"]
|
||||
|
||||
|
||||
class BaseCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def reject_invalid_strings(cls, value: object) -> object:
|
||||
"""Отклоняет строки из одних пробелов и строки с NUL, который не хранит PostgreSQL."""
|
||||
if isinstance(value, str):
|
||||
if not value.strip():
|
||||
raise ValueError("Строка не может состоять только из пробелов")
|
||||
if "\x00" in value:
|
||||
raise ValueError("Строка не может содержать символ NUL")
|
||||
return value
|
||||
|
||||
|
||||
class BasePage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
@@ -0,0 +1,22 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from app.schemas.base import Base, BaseCreate, BasePage
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CreateChat(BaseCreate):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class ChatPage(BasePage):
|
||||
items: list[Chat]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,41 @@
|
||||
from datetime import datetime
|
||||
from typing import Self
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
|
||||
from app.schemas.base import Base, BaseCreate, BasePage
|
||||
|
||||
|
||||
class Message(Base):
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CreateMessage(BaseCreate):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def reject_invalid_strings(cls, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
if "\x00" in value or (value != "" and not value.strip()):
|
||||
raise ValueError("Некорректная строка сообщения")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_content(self) -> Self:
|
||||
if not self.text and not self.attachment_ids:
|
||||
raise ValueError("Нужен текст или вложение")
|
||||
if len(self.attachment_ids) != len(set(self.attachment_ids)):
|
||||
raise ValueError("attachment_ids содержит дубликаты")
|
||||
return self
|
||||
|
||||
|
||||
class MessagePage(BasePage):
|
||||
items: list[Message]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,17 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.schemas.user import User
|
||||
|
||||
|
||||
class AuthSession(BaseModel):
|
||||
"""Ответ POST /auth/sessions (в OpenAPI — Session)."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
token: str = Field(min_length=32)
|
||||
token_type: Literal["Bearer"] = "Bearer"
|
||||
expires_at: datetime
|
||||
user: User
|
||||
@@ -0,0 +1,25 @@
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from app.schemas.base import Base, BaseCreate
|
||||
|
||||
USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$"
|
||||
PASSWORD_MIN_LENGTH = 12
|
||||
PASSWORD_MAX_LENGTH = 128
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""Публичное представление пользователя. Пароль/хеш сюда не попадают никогда."""
|
||||
|
||||
username: str = Field(pattern=USERNAME_PATTERN)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CreateUser(BaseCreate):
|
||||
username: str = Field(pattern=USERNAME_PATTERN)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(
|
||||
min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""Доступ к данным PostgreSQL и Redis."""
|
||||
@@ -0,0 +1,99 @@
|
||||
"""SQL-запросы метаданных и журнала; commit выполняется в domain."""
|
||||
|
||||
from datetime import timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
|
||||
|
||||
|
||||
async def create_attachment(session: AsyncSession, **values) -> Attachment:
|
||||
record = Attachment(**values)
|
||||
session.add(record)
|
||||
await session.flush()
|
||||
return record
|
||||
|
||||
|
||||
async def get_attachment(
|
||||
session: AsyncSession, attachment_id: UUID
|
||||
) -> Attachment | None:
|
||||
return await session.get(Attachment, attachment_id)
|
||||
|
||||
|
||||
async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
|
||||
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
|
||||
|
||||
|
||||
async def get_for_chat(
|
||||
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
|
||||
) -> list[Attachment]:
|
||||
records = []
|
||||
if attachment_ids:
|
||||
records = list(
|
||||
await session.scalars(
|
||||
select(Attachment).where(
|
||||
Attachment.id.in_(attachment_ids),
|
||||
Attachment.chat_id == chat_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
async def bind_attachments(
|
||||
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
|
||||
) -> None:
|
||||
session.add_all(
|
||||
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
|
||||
for position, id in enumerate(attachment_ids)
|
||||
)
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def message_attachment_ids(
|
||||
session: AsyncSession, message_ids: list[UUID]
|
||||
) -> dict[UUID, list[UUID]]:
|
||||
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
|
||||
if not message_ids:
|
||||
return result
|
||||
rows = await session.execute(
|
||||
select(MessageAttachment)
|
||||
.where(MessageAttachment.message_id.in_(message_ids))
|
||||
.order_by(MessageAttachment.position)
|
||||
)
|
||||
for link in rows.scalars():
|
||||
result[link.message_id].append(link.attachment_id)
|
||||
return result
|
||||
|
||||
|
||||
async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
|
||||
session.add(UploadIntent(id=id, object_key=key))
|
||||
await session.flush()
|
||||
|
||||
|
||||
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
|
||||
return await session.scalar(
|
||||
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
|
||||
)
|
||||
|
||||
|
||||
async def finish_upload(session: AsyncSession, id: UUID) -> None:
|
||||
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
|
||||
|
||||
|
||||
async def stale_uploads(
|
||||
session: AsyncSession, grace_seconds: int, limit: int
|
||||
) -> list[UploadIntent]:
|
||||
result = await session.scalars(
|
||||
select(UploadIntent)
|
||||
.where(
|
||||
UploadIntent.created_at
|
||||
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
|
||||
)
|
||||
.order_by(UploadIntent.created_at)
|
||||
.limit(limit)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
return list(result)
|
||||
@@ -0,0 +1,71 @@
|
||||
from collections.abc import Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import exists, select, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.chat import Chat
|
||||
from app.models.chat_member import ChatMember
|
||||
from app.storage.types import Position
|
||||
|
||||
|
||||
async def create_chat(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
title: str,
|
||||
member_ids: Sequence[UUID],
|
||||
) -> Chat:
|
||||
"""Добавляет чат вместе с участниками."""
|
||||
chat = Chat(
|
||||
title=title,
|
||||
members=[ChatMember(user_id=user_id) for user_id in member_ids],
|
||||
)
|
||||
session.add(chat)
|
||||
await session.flush()
|
||||
return chat
|
||||
|
||||
|
||||
async def get_chat_for_member(
|
||||
session: AsyncSession,
|
||||
chat_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> Chat | None:
|
||||
"""Возвращает чат, если пользователь в нём состоит, иначе None."""
|
||||
stmt = (
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
chat = result.scalar_one_or_none()
|
||||
return chat
|
||||
|
||||
|
||||
async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool:
|
||||
"""Возвращает True, если пользователь состоит в чате; одна проверка по PK."""
|
||||
stmt = select(
|
||||
exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
is_member = bool(result.scalar_one())
|
||||
return is_member
|
||||
|
||||
|
||||
async def list_chats_for_user(
|
||||
session: AsyncSession,
|
||||
user_id: UUID,
|
||||
after: Position | None,
|
||||
limit: int,
|
||||
) -> list[Chat]:
|
||||
"""Возвращает чаты пользователя по (created_at, id) строго после позиции after."""
|
||||
stmt = (
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(ChatMember.user_id == user_id)
|
||||
)
|
||||
if after is not None:
|
||||
stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
|
||||
stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit)
|
||||
result = await session.execute(stmt)
|
||||
chats = list(result.scalars().all())
|
||||
return chats
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Простые проверки соединения с общими хранилищами."""
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
|
||||
async def check_database(engine: AsyncEngine) -> bool:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
return True
|
||||
|
||||
|
||||
async def check_redis(client: Redis) -> bool:
|
||||
return bool(await client.ping())
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Хранение и блокировка ключей в PostgreSQL."""
|
||||
|
||||
from datetime import timedelta
|
||||
import hashlib
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, func, literal, select, tuple_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.idempotency_key import IdempotencyKey
|
||||
|
||||
Scope = tuple[UUID, str, str, str]
|
||||
|
||||
|
||||
async def lock_key(session: AsyncSession, scope: Scope) -> None:
|
||||
"""Сериализует один ключ до конца транзакции."""
|
||||
raw = json.dumps([str(scope[0]), *scope[1:]]).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
||||
await session.execute(
|
||||
select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger)))
|
||||
)
|
||||
|
||||
|
||||
async def get_active_key(
|
||||
session: AsyncSession, scope: Scope, ttl_seconds: int
|
||||
) -> IdempotencyKey | None:
|
||||
"""Читает результат, пока срок ключа не истёк."""
|
||||
stmt = select(IdempotencyKey).where(
|
||||
tuple_(
|
||||
IdempotencyKey.user_id,
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
) == tuple_(*scope),
|
||||
IdempotencyKey.created_at
|
||||
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
|
||||
)
|
||||
return await session.scalar(stmt)
|
||||
|
||||
|
||||
async def save_result(
|
||||
session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID
|
||||
) -> None:
|
||||
"""Сохраняет новый результат или заменяет истёкший."""
|
||||
stmt = insert(IdempotencyKey).values(
|
||||
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
|
||||
request_hash=request_hash, message_id=message_id,
|
||||
)
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=[
|
||||
IdempotencyKey.user_id, IdempotencyKey.method,
|
||||
IdempotencyKey.path, IdempotencyKey.key,
|
||||
],
|
||||
set_={
|
||||
"request_hash": request_hash,
|
||||
"message_id": message_id,
|
||||
"created_at": func.clock_timestamp(),
|
||||
},
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.counter import Counter
|
||||
from app.models.message import Message
|
||||
from app.storage.types import Position
|
||||
|
||||
MESSAGES_COUNTER = "messages"
|
||||
|
||||
|
||||
async def get_message(session: AsyncSession, message_id: UUID) -> Message | None:
|
||||
return await session.get(Message, message_id)
|
||||
|
||||
|
||||
async def create_message(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
) -> Message:
|
||||
"""Добавляет сообщение; id и created_at назначает сервер."""
|
||||
message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
|
||||
session.add(message)
|
||||
await session.flush()
|
||||
return message
|
||||
|
||||
|
||||
async def list_messages(
|
||||
session: AsyncSession,
|
||||
chat_id: UUID,
|
||||
after: Position | None,
|
||||
limit: int,
|
||||
) -> list[Message]:
|
||||
"""Возвращает сообщения чата по (created_at, id) строго после позиции after."""
|
||||
stmt = select(Message).where(Message.chat_id == chat_id)
|
||||
if after is not None:
|
||||
stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
|
||||
stmt = stmt.order_by(Message.created_at, Message.id).limit(limit)
|
||||
result = await session.execute(stmt)
|
||||
messages = list(result.scalars().all())
|
||||
return messages
|
||||
|
||||
|
||||
async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool:
|
||||
"""Атомарно увеличивает счётчик сообщений, если лимит не исчерпан."""
|
||||
stmt = (
|
||||
insert(Counter)
|
||||
.values(name=MESSAGES_COUNTER, value=1)
|
||||
.on_conflict_do_update(
|
||||
index_elements=[Counter.name],
|
||||
set_={"value": Counter.value + 1},
|
||||
where=Counter.value < max_messages,
|
||||
)
|
||||
.returning(Counter.value)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
is_reserved = result.scalar_one_or_none() is not None
|
||||
return is_reserved
|
||||
@@ -0,0 +1,180 @@
|
||||
"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import TypeVar
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from anyio import CancelScope
|
||||
from minio import Minio
|
||||
import urllib3
|
||||
|
||||
from app.core.s3 import S3Settings
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def run_sync(
|
||||
operation: Callable[..., T], *args, discard_result=None, **kwargs
|
||||
) -> T:
|
||||
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
|
||||
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
|
||||
try:
|
||||
result = await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
with CancelScope(shield=True):
|
||||
await settle_cancelled_call(task, discard_result)
|
||||
raise
|
||||
return result
|
||||
|
||||
|
||||
async def settle_cancelled_call(
|
||||
task: asyncio.Task, discard_result: Callable | None
|
||||
) -> None:
|
||||
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
|
||||
while not task.done():
|
||||
with suppress(asyncio.CancelledError):
|
||||
await asyncio.wait([task])
|
||||
completed = not task.cancelled() and task.exception() is None
|
||||
if completed and discard_result is not None:
|
||||
discard_result(task.result())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectInfo:
|
||||
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
|
||||
|
||||
key: str
|
||||
size_bytes: int | None
|
||||
content_type: str | None
|
||||
last_modified: datetime | None
|
||||
|
||||
|
||||
def close_response(response) -> None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
|
||||
class ObjectStream:
|
||||
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
|
||||
|
||||
def __init__(self, storage, response):
|
||||
self.storage = storage
|
||||
self.response = response
|
||||
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
|
||||
self.closed = False
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self.closed:
|
||||
raise StopAsyncIteration
|
||||
try:
|
||||
remaining = self.deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError
|
||||
async with asyncio.timeout(remaining):
|
||||
chunk = await self.storage._call(
|
||||
lambda: self.response.read(self.storage.settings.chunk_bytes)
|
||||
)
|
||||
if not chunk:
|
||||
await self.aclose()
|
||||
raise StopAsyncIteration
|
||||
except BaseException:
|
||||
await self.aclose()
|
||||
raise
|
||||
return chunk
|
||||
|
||||
async def aclose(self):
|
||||
if not self.closed:
|
||||
self.closed = True
|
||||
close_response(self.response)
|
||||
|
||||
|
||||
class MinioStorage:
|
||||
def __init__(self, settings: S3Settings) -> None:
|
||||
self.settings = settings
|
||||
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
|
||||
url = urlsplit(settings.endpoint)
|
||||
self._http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(
|
||||
connect=settings.connect_timeout_seconds,
|
||||
read=settings.read_timeout_seconds,
|
||||
),
|
||||
retries=urllib3.Retry(total=0),
|
||||
maxsize=32,
|
||||
)
|
||||
self._health_http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=0.25, read=0.5),
|
||||
retries=urllib3.Retry(total=0),
|
||||
)
|
||||
self._client = Minio(
|
||||
url.netloc,
|
||||
secure=url.scheme == "https",
|
||||
region=settings.region,
|
||||
access_key=settings.access_key_id_file.read_text().strip(),
|
||||
secret_key=settings.secret_access_key_file.read_text().strip(),
|
||||
http_client=self._http,
|
||||
)
|
||||
|
||||
async def _call(self, operation: Callable, *, discard_result=None):
|
||||
return await run_sync(operation, discard_result=discard_result)
|
||||
|
||||
async def put_file(
|
||||
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
|
||||
) -> None:
|
||||
def put():
|
||||
with path.open("rb") as source:
|
||||
# Один PUT: размер части равен максимальному разрешённому файлу.
|
||||
self._client.put_object(
|
||||
self.settings.bucket,
|
||||
key,
|
||||
source,
|
||||
size_bytes,
|
||||
content_type=content_type,
|
||||
metadata={"sha256": sha256},
|
||||
part_size=10_485_760,
|
||||
num_parallel_uploads=1,
|
||||
)
|
||||
|
||||
await self._call(put)
|
||||
|
||||
async def open_object(self, key: str) -> ObjectStream:
|
||||
response = await self._call(
|
||||
lambda: self._client.get_object(self.settings.bucket, key),
|
||||
discard_result=close_response,
|
||||
)
|
||||
return ObjectStream(self, response)
|
||||
|
||||
async def stat_object(self, key: str) -> ObjectInfo:
|
||||
def stat():
|
||||
info = self._client.stat_object(self.settings.bucket, key)
|
||||
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
|
||||
|
||||
return await self._call(stat)
|
||||
|
||||
async def delete_object(self, key: str) -> None:
|
||||
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
|
||||
|
||||
async def check_available(self) -> bool:
|
||||
status = await self._call(
|
||||
lambda: self._health_http.request(
|
||||
"GET",
|
||||
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
|
||||
).status
|
||||
)
|
||||
return status == 200
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self._http.clear()
|
||||
self._health_http.clear()
|
||||
|
||||
|
||||
def create_s3_storage(settings: S3Settings) -> MinioStorage:
|
||||
return MinioStorage(settings)
|
||||
@@ -0,0 +1,38 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
SESSION_KEY_PREFIX = "session:"
|
||||
|
||||
|
||||
def session_key(token_hash: str) -> str:
|
||||
"""Возвращает ключ Redis для хеша токена."""
|
||||
return f"{SESSION_KEY_PREFIX}{token_hash}"
|
||||
|
||||
|
||||
async def create_session(
|
||||
redis: Redis,
|
||||
*,
|
||||
token_hash: str,
|
||||
user_id: UUID,
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
"""Добавляет сессию с абсолютным временем истечения."""
|
||||
await redis.set(session_key(token_hash), str(user_id), pxat=expires_at)
|
||||
|
||||
|
||||
async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None:
|
||||
"""Возвращает id владельца действующей сессии или None."""
|
||||
raw_user_id = await redis.get(session_key(token_hash))
|
||||
if isinstance(raw_user_id, bytes):
|
||||
raw_user_id = raw_user_id.decode()
|
||||
user_id = UUID(raw_user_id) if raw_user_id else None
|
||||
return user_id
|
||||
|
||||
|
||||
async def delete_session(redis: Redis, token_hash: str) -> bool:
|
||||
"""Удаляет сессию; возвращает True, если она существовала."""
|
||||
deleted_count = await redis.delete(session_key(token_hash))
|
||||
is_deleted = deleted_count > 0
|
||||
return is_deleted
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Общие типы слоя хранения."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
Position = tuple[datetime, UUID]
|
||||
@@ -0,0 +1,47 @@
|
||||
from collections.abc import Collection
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
async def create_user(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
username: str,
|
||||
display_name: str,
|
||||
password_hash: str,
|
||||
) -> User:
|
||||
"""Добавляет пользователя; занятый username даёт IntegrityError при flush."""
|
||||
user = User(
|
||||
username=username, display_name=display_name, password_hash=password_hash
|
||||
)
|
||||
session.add(user)
|
||||
await session.flush()
|
||||
return user
|
||||
|
||||
|
||||
async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
|
||||
"""Возвращает пользователя по id или None."""
|
||||
user = await session.get(User, user_id)
|
||||
return user
|
||||
|
||||
|
||||
async def get_user_by_username(session: AsyncSession, username: str) -> User | None:
|
||||
"""Возвращает пользователя по username или None."""
|
||||
stmt = select(User).where(User.username == username)
|
||||
result = await session.execute(stmt)
|
||||
user = result.scalar_one_or_none()
|
||||
return user
|
||||
|
||||
|
||||
async def get_existing_user_ids(
|
||||
session: AsyncSession, user_ids: Collection[UUID]
|
||||
) -> set[UUID]:
|
||||
"""Возвращает те из user_ids, что существуют, одним запросом."""
|
||||
stmt = select(User.id).where(User.id.in_(user_ids))
|
||||
result = await session.execute(stmt)
|
||||
existing_ids = set(result.scalars().all())
|
||||
return existing_ids
|
||||
@@ -0,0 +1,10 @@
|
||||
# Override для отдельного тестового стенда (make test-stand-up):
|
||||
# публикует PostgreSQL и Redis на 127.0.0.1, чтобы pytest из student-tests/
|
||||
# мог подключаться к ним напрямую. В основном стенде эти порты закрыты.
|
||||
services:
|
||||
postgres:
|
||||
ports:
|
||||
- "127.0.0.1:${DB_HOST_PORT:?Set DB_HOST_PORT in .env}:5432"
|
||||
redis:
|
||||
ports:
|
||||
- "127.0.0.1:${REDIS_HOST_PORT:?Set REDIS_HOST_PORT in .env}:6379"
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
name: mtusi-messenger-lab4
|
||||
|
||||
x-api-image: &api-image
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env}
|
||||
image: ${API_IMAGE:?Set API_IMAGE in .env}
|
||||
|
||||
x-base-env: &base-env
|
||||
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
|
||||
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
|
||||
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
|
||||
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
|
||||
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
|
||||
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
|
||||
DB_HOST: ${DB_HOST:?Set DB_HOST in .env}
|
||||
DB_PORT: ${DB_PORT:?Set DB_PORT in .env}
|
||||
DB_NAME: ${DB_NAME:?Set DB_NAME in .env}
|
||||
DB_USER: ${DB_USER:?Set DB_USER in .env}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env}
|
||||
DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env}
|
||||
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env}
|
||||
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env}
|
||||
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env}
|
||||
SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env}
|
||||
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env}
|
||||
READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env}
|
||||
|
||||
x-s3-env: &s3-env
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
|
||||
S3_REGION: ${S3_REGION:?Set S3_REGION in .env}
|
||||
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env}
|
||||
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env}
|
||||
S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env}
|
||||
S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env}
|
||||
S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env}
|
||||
S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env}
|
||||
S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env}
|
||||
S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env}
|
||||
S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env}
|
||||
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
|
||||
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
|
||||
|
||||
services:
|
||||
minio:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: server
|
||||
image: mtusi-messenger-minio:lab4
|
||||
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
networks: [objects]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
minio-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: bootstrap
|
||||
image: mtusi-messenger-minio-bootstrap:lab4
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
|
||||
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks: [objects]
|
||||
restart: "no"
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
postgres:
|
||||
image: postgres:16.10-bookworm
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: ${DB_USER:?Set DB_USER in .env}
|
||||
POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets:
|
||||
- db_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-bookworm
|
||||
restart: unless-stopped
|
||||
entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"]
|
||||
secrets:
|
||||
- redis_password
|
||||
volumes:
|
||||
- ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro
|
||||
- redisdata:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
migrate:
|
||||
<<: *api-image
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment: *base-env
|
||||
secrets:
|
||||
- db_password
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
api:
|
||||
<<: *api-image
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
<<: [*base-env, *s3-env]
|
||||
REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env}
|
||||
REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env}
|
||||
REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env}
|
||||
REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env}
|
||||
REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env}
|
||||
REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env}
|
||||
REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env}
|
||||
secrets:
|
||||
- db_password
|
||||
- redis_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks: [default, objects]
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:?Set API_REPLICAS in .env}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1536M
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
start_period: 10s
|
||||
retries: 2
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 8s
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
|
||||
cleanup:
|
||||
<<: *api-image
|
||||
command: ["python", "-m", "app.core.cleanup", "--apply"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
<<: [*base-env, *s3-env]
|
||||
secrets:
|
||||
- db_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks: [default, objects]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
minio-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
nginx:
|
||||
image: nginx:1.28.0-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
miniodata:
|
||||
|
||||
networks:
|
||||
default:
|
||||
objects:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
db_password:
|
||||
file: ./secrets/db_password
|
||||
redis_password:
|
||||
file: ./secrets/redis_password
|
||||
minio_root_user:
|
||||
file: ./secrets/minio_root_user
|
||||
minio_root_password:
|
||||
file: ./secrets/minio_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
+113
-3
@@ -1,5 +1,115 @@
|
||||
# Доказательства
|
||||
# Evidence лабораторной 4
|
||||
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
Результаты отдельного полного прогона на чистом проекте
|
||||
`mtusi-messenger-lab4-test`. Дата, базовый commit, версии, лимиты CPU/RAM,
|
||||
идентификаторы образов и точные команды находятся в [run.json](run.json).
|
||||
Проверено незакоммиченное рабочее дерево; его исполняемые файлы зафиксированы
|
||||
в [source-sha256.txt](source-sha256.txt). Манифест снимается до прогона и
|
||||
сравнивается после него. Отчёт, README, RUNBOOK, LAB4_PLAN и сами evidence
|
||||
в манифест не входят; инструкции AGENTS.md/CODE_STYLE.md зафиксированы.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
## Артефакты и скрипты
|
||||
|
||||
Все шесть файлов записывает [scripts/collect_evidence.py](../scripts/collect_evidence.py).
|
||||
Первичные проверки и источники данных указаны ниже.
|
||||
|
||||
| Evidence | Что содержит | Откуда берётся |
|
||||
| --- | --- | --- |
|
||||
| [test-output.txt](test-output.txt) | Общий вывод проверок и сценариев; `188 passed`, строки `PASS`, результаты отказов. | Команды из `run.json`; конкретные скрипты — в таблице требований ниже. |
|
||||
| [nginx-output.txt](nginx-output.txt) | Конфигурация, DNS, балансировка, gzip и ошибки Nginx. | [scripts/check_nginx.py](../scripts/check_nginx.py), команда `make test-nginx`. |
|
||||
| [metrics.json](metrics.json) | Числа pytest, upload, rolling update и отказов API. | `collect_evidence.extract_metrics()` извлекает результаты pytest, `upload_load.py`, `check_grade4.py` и `tests/resilience.py`. |
|
||||
| [run.json](run.json) | Среда, лимиты, образы, команды/exit codes и снимки контейнеров. | `collect_evidence.py`: Docker info/config/inspect и снимки между сценариями. |
|
||||
| [cleanup.txt](cleanup.txt) | Удаление тестовых контейнеров, сетей и volumes. | `collect_evidence.py`: `docker compose down --volumes --remove-orphans`. |
|
||||
| [source-sha256.txt](source-sha256.txt) | Контрольные суммы проверенного кода, конфигураций и тестов. | `collect_evidence.source_manifest()`: SHA-256 до прогона и проверка совпадения после. |
|
||||
|
||||
## Соответствие требованиям на 3 и 4
|
||||
|
||||
Номера соответствуют порядку пунктов в разделах оценок [задания](../README.md).
|
||||
|
||||
| Пункт | Требование | Где доказательство / что смотреть | Проверяющий скрипт |
|
||||
| --- | --- | --- | --- |
|
||||
| 3.1 | Сохранён минимум lab3; метаданные в PostgreSQL, байты в приватном S3, общий стенд. | [run.json](run.json): `services`, `container_snapshots`; [test-output.txt](test-output.txt): smoke, идемпотентность, `PASS infrastructure`. | `tests/smoke.py`, `scripts/check_idempotency.py`, `scripts/check_s3.py`, `scripts/check_grade4.py`; pytest `test_migrations.py` и `test_attachments.py`. |
|
||||
| 3.2 | 201 ready, побайтный оригинал, SHA-256, attachment_ids и доступ участников. | [test-output.txt](test-output.txt): `PASS: lab4 public API scenario`; общий итог pytest. | `tests/smoke.py`, `tests/client.py`; `student-tests/test_attachments.py`. |
|
||||
| 3.3 | PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому, безопасное имя, ранние лимиты. | [test-output.txt](test-output.txt): `188 passed`; в тестах есть приём ровно на лимитах и отклонение превышений. | `student-tests/test_attachments.py`: `test_exact_*`, `test_oversize_*`, `test_pixel_limit_before_decoding`, `test_invalid_*`, `test_unsafe_*`. |
|
||||
| 3.4 | Скачивание другой репликой после отказа загрузившей; сохранность после restart. | [test-output.txt](test-output.txt): `PASS: persistence`, `PASS: failover`; [run.json](run.json): `before_restart` / `after_restart`; [metrics.json](metrics.json): `public_failover`. | `tests/resilience.py`; hooks `scripts/restart.sh` и `scripts/stop-one.sh`. |
|
||||
| 4.1 | Негативные тесты: размер, повреждение, MIME, чужой ID и файл другого чата; предусмотренные 4xx. | [test-output.txt](test-output.txt): `188 passed`; соответствующие тесты включены в полный pytest-прогон. | `student-tests/test_attachments.py`: `test_invalid_*`, `test_oversize_*`, `test_unknown_and_other_chat_attachment_are_404`, `test_upload_download_and_attachment_only_message`. |
|
||||
| 4.2 | Отмена и сбои S3/БД; компенсация, сверка, безопасный grace period. | [test-output.txt](test-output.txt): `PASS: real S3 PUT/database failure...` и итог pytest; [LAB4_PLAN.md](../LAB4_PLAN.md): механизм согласованности. | `scripts/check_s3.py --with-faults` → `scripts/s3_faults.py`; `student-tests/test_attachments.py`, `test_s3_stream.py`. |
|
||||
| 4.3 | Ограничения RAM/таймаутов и эксперимент ≥10 одновременных загрузок. | [metrics.json](metrics.json): `uploads` — 3×10 попыток, 12×201, 18×429, память каждой API; [run.json](run.json): лимиты; [test-output.txt](test-output.txt): тесты timeout/отмены. | `scripts/upload_load.py --concurrency 10`; `student-tests/test_attachments.py`, `test_s3_stream.py`; аудит `scripts/check_grade4.py`. |
|
||||
| 4.4 | Повторяемый bootstrap, отдельные ограниченные credentials, закрытая консоль, private bucket вне API. | [test-output.txt](test-output.txt): `PASS: idempotent bootstrap, anonymous GET/PUT denied...`; [run.json](run.json): порты и назначение секретов. | `scripts/check_s3.py --with-outage --with-faults`, повторно вызывает `infra/minio/bootstrap.py`; `scripts/check_grade4.py` проверяет фактические ресурсы/секреты. |
|
||||
|
||||
Pytest запущен с `-q`: в evidence сохранён общий итог, а названия и проверки
|
||||
отдельных случаев находятся в указанных тестовых файлах. Их точные версии
|
||||
зафиксированы в `source-sha256.txt`.
|
||||
Сверка объектов S3 подтверждается сценарием `s3_faults.py` в `test-output.txt`;
|
||||
`cleanup.txt` фиксирует удаление самого тестового стенда.
|
||||
|
||||
## Воспроизведение
|
||||
|
||||
Из корня проекта, с Python 3.12 и работающим Docker:
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make init
|
||||
.venv/bin/python -m scripts.collect_evidence
|
||||
sha256sum -c evidence/source-sha256.txt
|
||||
```
|
||||
|
||||
Коллектор требует отсутствия прежних ресурсов тестового проекта. Существующие
|
||||
`.env` и секреты сохраняются. MinIO и mc собираются из закреплённых исходников;
|
||||
первая сборка требует доступа к Go-модулям. Основной стенд не запускается.
|
||||
В `finally` выполняется `down --volumes --remove-orphans`; evidence публикуются
|
||||
только после успешного завершения всех проверок и подтверждения очистки.
|
||||
|
||||
## Методика
|
||||
|
||||
1. Проверить контракт и OpenAPI; собрать стенд с двумя API, общей PostgreSQL,
|
||||
Redis, MinIO, bootstrap и cleanup. Pytest использует отдельную БД
|
||||
`messenger_test` и Redis DB 15; публичные проверки — API через Nginx :18080.
|
||||
2. Выполнить все собственные тесты с `DOCKER_TESTS=1`, затем публичный smoke и
|
||||
24 конкурентных повтора одного сообщения через две реплики.
|
||||
3. Снять состояния контейнеров до и после persistence. Hook перезапускает
|
||||
PostgreSQL, Redis, MinIO, обе API, Nginx и cleanup, сохраняя volumes.
|
||||
Публичная проверка повторно читает данные и оригинал с прежним токеном.
|
||||
В снимках совпадают ID контейнеров и изменились PID/StartedAt всех семи
|
||||
работающих контейнеров, включая три хранилища.
|
||||
4. Загрузить файл, остановить обслужившую API-реплику и скачать тот же оригинал
|
||||
через оставшуюся; проверить SHA-256, подтверждённые сообщения и идемпотентность.
|
||||
5. Дважды повторить bootstrap. Проверить прямые анонимные GET/PUT в S3, запрет
|
||||
чтения политики для API credentials, доступ только к своему bucket и
|
||||
недоступность консоли из сети API. `ListBuckets` MinIO может вернуть
|
||||
отфильтрованный список разрешённых bucket; это учитывается проверкой.
|
||||
6. Выполнить реальный PUT и внедрить отказ сохранения метаданных. Проверить
|
||||
компенсацию; затем внедрить отказ удаления, проверить устойчивый журнал,
|
||||
сохранность свежего объекта, dry-run и повторную очистку. Для проверки grace
|
||||
period возраст только тестовой записи изменяется на два дня; cleanup временно
|
||||
остановлен и затем возвращён в работу. Подтверждённый оригинал сохраняется.
|
||||
7. Временно остановить MinIO: download даёт 503; после запуска прежний оригинал
|
||||
снова скачивается побайтно.
|
||||
8. Выполнить три серии по 10 одновременно начатых upload PNG 1500×1500,
|
||||
6 761 054 байта, генератор случайных пикселей с seed=42. Активных обработок
|
||||
не более двух на API; лишние получают 429, повторов upload нет. После опыта
|
||||
скачать все подтверждённые оригиналы и сравнить SHA-256.
|
||||
9. Проверить фактические ресурсы/секреты, SIGTERM, обновление обеих API под
|
||||
записью и последовательный отказ каждой с восстановлением предыдущей.
|
||||
Новая build-метка создаёт новый образ того же бизнес-кода; миграционная
|
||||
команда при rolling update выполняется один раз.
|
||||
10. Выполнить отдельную проверку Nginx и удалить тестовый стенд с volumes.
|
||||
|
||||
Память `memory.current` снимается в cgroup каждой API через 20 мс и включает
|
||||
декодеры, tmpfs и небольшой процесс-монитор. Выборки могут пропустить короткий
|
||||
пик; дополнительно записан kernel `memory.peak` до/после опыта. Последний
|
||||
накопительный с момента создания контейнера и может включать предыдущие проверки.
|
||||
P95 загрузок относится ко всем попыткам, включая 429; скачивание для сверки
|
||||
SHA-256 выполняется после измерения памяти upload.
|
||||
|
||||
RPS в сценариях отказа — успешные HTTP-ответы, включая повторы ключа.
|
||||
`first_recovery_seconds` включает время hook: это первый завершившийся успешный
|
||||
POST, начатый после завершения команды остановки. Ошибки 429 ограничителя и
|
||||
переходные 503 учитываются отдельно; подтверждённые записи проверяются по истории.
|
||||
Один короткий локальный прогон не определяет предельную производительность.
|
||||
|
||||
В артефакты не включаются `.env`, значения credentials, session tokens,
|
||||
Authorization, тела сообщений и дампы хранилищ. Коллектор отбирает несекретные
|
||||
параметры, скрывает известные значения секретов и проверяет их отсутствие
|
||||
перед публикацией файлов.
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
$ docker compose --env-file .env down --volumes --remove-orphans
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Removing
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Removed
|
||||
Container mtusi-messenger-lab4-test-api-3 Stopping
|
||||
Container mtusi-messenger-lab4-test-api-4 Stopping
|
||||
Container mtusi-messenger-lab4-test-api-4 Stopped
|
||||
Container mtusi-messenger-lab4-test-api-4 Removing
|
||||
Container mtusi-messenger-lab4-test-api-3 Stopped
|
||||
Container mtusi-messenger-lab4-test-api-3 Removing
|
||||
Container mtusi-messenger-lab4-test-api-4 Removed
|
||||
Container mtusi-messenger-lab4-test-api-3 Removed
|
||||
Container mtusi-messenger-lab4-test-redis-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-redis-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-redis-1 Removing
|
||||
Container mtusi-messenger-lab4-test-redis-1 Removed
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Removing
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Removed
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Removing
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Removing
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Removed
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Removed
|
||||
Container mtusi-messenger-lab4-test-minio-1 Stopping
|
||||
Container mtusi-messenger-lab4-test-minio-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-minio-1 Removing
|
||||
Container mtusi-messenger-lab4-test-minio-1 Removed
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Stopped
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Removing
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Removed
|
||||
Volume mtusi-messenger-lab4-test_pgdata Removing
|
||||
Volume mtusi-messenger-lab4-test_miniodata Removing
|
||||
Volume mtusi-messenger-lab4-test_redisdata Removing
|
||||
Network mtusi-messenger-lab4-test_objects Removing
|
||||
Network mtusi-messenger-lab4-test_default Removing
|
||||
Volume mtusi-messenger-lab4-test_pgdata Removed
|
||||
Volume mtusi-messenger-lab4-test_miniodata Removed
|
||||
Volume mtusi-messenger-lab4-test_redisdata Removed
|
||||
Network mtusi-messenger-lab4-test_default Removed
|
||||
Network mtusi-messenger-lab4-test_objects Removed
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"pytest": {
|
||||
"passed": 188,
|
||||
"elapsed_seconds": 73.96
|
||||
},
|
||||
"uploads": {
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"attempts": 30,
|
||||
"fixture": {
|
||||
"format": "PNG",
|
||||
"width": 1500,
|
||||
"height": 1500,
|
||||
"size_bytes": 6761054,
|
||||
"sha256": "49cb1a9ec3e54c1f22c649fb787c50186c54e449ade4a6f6758a50b2e374504d"
|
||||
},
|
||||
"statuses": {
|
||||
"201": 12,
|
||||
"429": 18
|
||||
},
|
||||
"elapsed_seconds": 5.138,
|
||||
"successful_uploads": 12,
|
||||
"p95_seconds": 2.143,
|
||||
"memory": {
|
||||
"bf64a4a13acb": {
|
||||
"initial_bytes": 88752128,
|
||||
"sampled_peak_bytes": 162426880,
|
||||
"cgroup_peak_before_bytes": 161976320,
|
||||
"cgroup_peak_after_bytes": 162689024,
|
||||
"interval_ms": 20
|
||||
},
|
||||
"7e4180e529e1": {
|
||||
"initial_bytes": 89436160,
|
||||
"sampled_peak_bytes": 163692544,
|
||||
"cgroup_peak_before_bytes": 157728768,
|
||||
"cgroup_peak_after_bytes": 163692544,
|
||||
"interval_ms": 20
|
||||
}
|
||||
},
|
||||
"api_memory_limit": "1610612736",
|
||||
"notes": "20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries"
|
||||
},
|
||||
"public_failover": [
|
||||
{
|
||||
"observed_instances_before": 2,
|
||||
"successful_writes": 112,
|
||||
"failed_attempts": 4,
|
||||
"first_recovery_seconds": 1.885
|
||||
}
|
||||
],
|
||||
"rolling_update": [
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"replacements": 2,
|
||||
"migration_runs": 1,
|
||||
"successful_writes": 487,
|
||||
"failed_attempts": 13,
|
||||
"errors_by_kind": {
|
||||
"http_503": 13
|
||||
},
|
||||
"elapsed_seconds": 25.341,
|
||||
"successful_rps": 19.218,
|
||||
"p95_seconds": 0.048
|
||||
}
|
||||
],
|
||||
"replica_failures": [
|
||||
{
|
||||
"instance": "429367307e5f",
|
||||
"successful_writes": 777,
|
||||
"failed_attempts": 87,
|
||||
"errors_by_kind": {
|
||||
"http_429": 70,
|
||||
"http_503": 17
|
||||
},
|
||||
"elapsed_seconds": 13.965,
|
||||
"successful_rps": 55.64,
|
||||
"first_recovery_seconds": 1.865
|
||||
},
|
||||
{
|
||||
"instance": "20b4b219e7c8",
|
||||
"successful_writes": 774,
|
||||
"failed_attempts": 96,
|
||||
"errors_by_kind": {
|
||||
"http_429": 79,
|
||||
"http_503": 17
|
||||
},
|
||||
"elapsed_seconds": 14.127,
|
||||
"successful_rps": 54.79,
|
||||
"first_recovery_seconds": 2.042
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
$ make test-nginx
|
||||
.venv/bin/python scripts/check_nginx.py
|
||||
nginx version: nginx/1.28.0
|
||||
PASS: nginx -t и балансировка двух реплик
|
||||
PASS: gzip 2060 → 61 байт, содержимое сохранено
|
||||
PASS: JSON и заголовки ошибки API передаются без подмены
|
||||
PASS: отправленный POST не повторяется на другой реплике
|
||||
PASS: лимит вернул 390 ответов 429; health доступен
|
||||
PASS: третья реплика обнаружена через DNS без reload nginx
|
||||
PASS: отказ реплики, первое успешное чтение через 0.719 с
|
||||
PASS: недоступность всех backend возвращает JSON 503
|
||||
PASS: JSON access log содержит время backend и результат лимита
|
||||
|
||||
@@ -0,0 +1,888 @@
|
||||
{
|
||||
"started_at_utc": "2026-11-04T18:14:58.493240+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:20:25.766354+00:00",
|
||||
"started_at_moscow": "2026-11-04T21:14:58.493240+03:00",
|
||||
"base_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5",
|
||||
"worktree_dirty": true,
|
||||
"source_manifest_sha256": "f3c995d33b82b1996d497b106fb672792f5ad0caf11b5391fd0ef5c89d86ca45",
|
||||
"project": "mtusi-messenger-lab4-test",
|
||||
"runtime": {
|
||||
"python": "3.12.14",
|
||||
"packages": {
|
||||
"fastapi": "0.143.3",
|
||||
"SQLAlchemy": "2.1.4",
|
||||
"pydantic": "2.14.1",
|
||||
"minio": "7.2.20",
|
||||
"Pillow": "13.0.0",
|
||||
"pytest": "9.1.1"
|
||||
},
|
||||
"compose": "Docker Compose version 5.6.1"
|
||||
},
|
||||
"docker_host": {
|
||||
"ServerVersion": "29.9.2",
|
||||
"OperatingSystem": "Arch Linux",
|
||||
"Architecture": "x86_64",
|
||||
"NCPU": 8,
|
||||
"MemTotal": 16400363520,
|
||||
"CgroupVersion": "2"
|
||||
},
|
||||
"services": {
|
||||
"api": {
|
||||
"image": "mtusi-messenger:lab4",
|
||||
"limits": {
|
||||
"cpus": 1,
|
||||
"memory": "1610612736"
|
||||
},
|
||||
"settings": {
|
||||
"APP_PORT": "8080",
|
||||
"DB_MAX_OVERFLOW": "10",
|
||||
"DB_POOL_SIZE": "5",
|
||||
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
|
||||
"IDEMPOTENCY_TTL_SECONDS": "86400",
|
||||
"MAX_IN_FLIGHT": "16",
|
||||
"MAX_MESSAGES": "100000",
|
||||
"READINESS_TIMEOUT_SECONDS": "2",
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"S3_CHUNK_BYTES": "65536",
|
||||
"S3_CLEANUP_GRACE_SECONDS": "86400",
|
||||
"S3_CLEANUP_INTERVAL_SECONDS": "300",
|
||||
"S3_CONNECT_TIMEOUT_SECONDS": "2",
|
||||
"S3_DOWNLOAD_TIMEOUT_SECONDS": "60",
|
||||
"S3_ENDPOINT": "http://minio:9000",
|
||||
"S3_IMAGE_TIMEOUT_SECONDS": "10",
|
||||
"S3_MAX_CONCURRENT_UPLOADS": "2",
|
||||
"S3_MAX_IMAGE_PIXELS": "40000000",
|
||||
"S3_MAX_UPLOAD_BYTES": "10485760",
|
||||
"S3_READ_TIMEOUT_SECONDS": "10",
|
||||
"S3_REGION": "us-east-1",
|
||||
"S3_UPLOAD_TIMEOUT_SECONDS": "30",
|
||||
"SESSION_TTL_SECONDS": "3600",
|
||||
"SHUTDOWN_DRAIN_SECONDS": "1"
|
||||
},
|
||||
"ports": [],
|
||||
"networks": [
|
||||
"default",
|
||||
"objects"
|
||||
],
|
||||
"secret_names": [
|
||||
"db_password",
|
||||
"redis_password",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key"
|
||||
],
|
||||
"tmpfs": [
|
||||
"/tmp:rw,noexec,nosuid,size=64m,mode=1777"
|
||||
]
|
||||
},
|
||||
"cleanup": {
|
||||
"image": "mtusi-messenger:lab4",
|
||||
"limits": {
|
||||
"cpus": 0.25,
|
||||
"memory": "268435456"
|
||||
},
|
||||
"settings": {
|
||||
"APP_PORT": "8080",
|
||||
"DB_MAX_OVERFLOW": "10",
|
||||
"DB_POOL_SIZE": "5",
|
||||
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
|
||||
"MAX_IN_FLIGHT": "16",
|
||||
"MAX_MESSAGES": "100000",
|
||||
"READINESS_TIMEOUT_SECONDS": "2",
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"S3_CHUNK_BYTES": "65536",
|
||||
"S3_CLEANUP_GRACE_SECONDS": "86400",
|
||||
"S3_CLEANUP_INTERVAL_SECONDS": "300",
|
||||
"S3_CONNECT_TIMEOUT_SECONDS": "2",
|
||||
"S3_DOWNLOAD_TIMEOUT_SECONDS": "60",
|
||||
"S3_ENDPOINT": "http://minio:9000",
|
||||
"S3_IMAGE_TIMEOUT_SECONDS": "10",
|
||||
"S3_MAX_CONCURRENT_UPLOADS": "2",
|
||||
"S3_MAX_IMAGE_PIXELS": "40000000",
|
||||
"S3_MAX_UPLOAD_BYTES": "10485760",
|
||||
"S3_READ_TIMEOUT_SECONDS": "10",
|
||||
"S3_REGION": "us-east-1",
|
||||
"S3_UPLOAD_TIMEOUT_SECONDS": "30",
|
||||
"SHUTDOWN_DRAIN_SECONDS": "1"
|
||||
},
|
||||
"ports": [],
|
||||
"networks": [
|
||||
"default",
|
||||
"objects"
|
||||
],
|
||||
"secret_names": [
|
||||
"db_password",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key"
|
||||
],
|
||||
"tmpfs": []
|
||||
},
|
||||
"migrate": {
|
||||
"image": "mtusi-messenger:lab4",
|
||||
"limits": {
|
||||
"cpus": 0.5,
|
||||
"memory": "268435456"
|
||||
},
|
||||
"settings": {
|
||||
"APP_PORT": "8080",
|
||||
"DB_MAX_OVERFLOW": "10",
|
||||
"DB_POOL_SIZE": "5",
|
||||
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
|
||||
"MAX_IN_FLIGHT": "16",
|
||||
"MAX_MESSAGES": "100000",
|
||||
"READINESS_TIMEOUT_SECONDS": "2",
|
||||
"SHUTDOWN_DRAIN_SECONDS": "1"
|
||||
},
|
||||
"ports": [],
|
||||
"networks": [
|
||||
"default"
|
||||
],
|
||||
"secret_names": [
|
||||
"db_password"
|
||||
],
|
||||
"tmpfs": []
|
||||
},
|
||||
"minio": {
|
||||
"image": "mtusi-messenger-minio:lab4",
|
||||
"limits": {
|
||||
"cpus": 1,
|
||||
"memory": "536870912"
|
||||
},
|
||||
"settings": {},
|
||||
"ports": [],
|
||||
"networks": [
|
||||
"objects"
|
||||
],
|
||||
"secret_names": [
|
||||
"minio_root_user",
|
||||
"minio_root_password"
|
||||
],
|
||||
"tmpfs": []
|
||||
},
|
||||
"minio-bootstrap": {
|
||||
"image": "mtusi-messenger-minio-bootstrap:lab4",
|
||||
"limits": {
|
||||
"cpus": 0.25,
|
||||
"memory": "134217728"
|
||||
},
|
||||
"settings": {
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"S3_ENDPOINT": "http://minio:9000"
|
||||
},
|
||||
"ports": [],
|
||||
"networks": [
|
||||
"objects"
|
||||
],
|
||||
"secret_names": [
|
||||
"minio_root_user",
|
||||
"minio_root_password",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key"
|
||||
],
|
||||
"tmpfs": []
|
||||
},
|
||||
"nginx": {
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"limits": {
|
||||
"cpus": 0.5,
|
||||
"memory": "134217728"
|
||||
},
|
||||
"settings": {},
|
||||
"ports": [
|
||||
{
|
||||
"mode": "ingress",
|
||||
"host_ip": "127.0.0.1",
|
||||
"target": 8080,
|
||||
"published": "18080",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
],
|
||||
"networks": [
|
||||
"default"
|
||||
],
|
||||
"secret_names": [],
|
||||
"tmpfs": []
|
||||
},
|
||||
"postgres": {
|
||||
"image": "postgres:16.10-bookworm",
|
||||
"limits": {
|
||||
"cpus": 1,
|
||||
"memory": "536870912"
|
||||
},
|
||||
"settings": {},
|
||||
"ports": [
|
||||
{
|
||||
"mode": "ingress",
|
||||
"host_ip": "127.0.0.1",
|
||||
"target": 5432,
|
||||
"published": "15432",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
],
|
||||
"networks": [
|
||||
"default"
|
||||
],
|
||||
"secret_names": [
|
||||
"db_password"
|
||||
],
|
||||
"tmpfs": []
|
||||
},
|
||||
"redis": {
|
||||
"image": "redis:7.4.2-bookworm",
|
||||
"limits": {
|
||||
"cpus": 0.5,
|
||||
"memory": "536870912"
|
||||
},
|
||||
"settings": {},
|
||||
"ports": [
|
||||
{
|
||||
"mode": "ingress",
|
||||
"host_ip": "127.0.0.1",
|
||||
"target": 6379,
|
||||
"published": "16379",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
],
|
||||
"networks": [
|
||||
"default"
|
||||
],
|
||||
"secret_names": [
|
||||
"redis_password"
|
||||
],
|
||||
"tmpfs": []
|
||||
}
|
||||
},
|
||||
"container_snapshots": {
|
||||
"started": [
|
||||
{
|
||||
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
|
||||
"name": "/mtusi-messenger-lab4-test-nginx-1",
|
||||
"service": "nginx",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:21.942730957Z",
|
||||
"pid": 483246,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
|
||||
"name": "/mtusi-messenger-lab4-test-api-2",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:16.003299195Z",
|
||||
"pid": 482834,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
|
||||
"name": "/mtusi-messenger-lab4-test-api-1",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:16.212298098Z",
|
||||
"pid": 482896,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
|
||||
"name": "/mtusi-messenger-lab4-test-cleanup-1",
|
||||
"service": "cleanup",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:15.695719979Z",
|
||||
"pid": 482774,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
|
||||
"service": "minio-bootstrap",
|
||||
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:15:08.266711675Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
|
||||
"name": "/mtusi-messenger-lab4-test-migrate-1",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:15:11.433538933Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-1",
|
||||
"service": "minio",
|
||||
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:04.59129875Z",
|
||||
"pid": 481985,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
|
||||
"name": "/mtusi-messenger-lab4-test-postgres-1",
|
||||
"service": "postgres",
|
||||
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:04.743334637Z",
|
||||
"pid": 482062,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
|
||||
"name": "/mtusi-messenger-lab4-test-redis-1",
|
||||
"service": "redis",
|
||||
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:04.922389241Z",
|
||||
"pid": 482171,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"before_restart": [
|
||||
{
|
||||
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
|
||||
"name": "/mtusi-messenger-lab4-test-nginx-1",
|
||||
"service": "nginx",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:21.942730957Z",
|
||||
"pid": 483246,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
|
||||
"name": "/mtusi-messenger-lab4-test-api-2",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:16.003299195Z",
|
||||
"pid": 482834,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
|
||||
"name": "/mtusi-messenger-lab4-test-api-1",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:16.212298098Z",
|
||||
"pid": 482896,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
|
||||
"name": "/mtusi-messenger-lab4-test-cleanup-1",
|
||||
"service": "cleanup",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:15.695719979Z",
|
||||
"pid": 482774,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
|
||||
"service": "minio-bootstrap",
|
||||
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:15:08.266711675Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
|
||||
"name": "/mtusi-messenger-lab4-test-migrate-1",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:15:11.433538933Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-1",
|
||||
"service": "minio",
|
||||
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:04.59129875Z",
|
||||
"pid": 481985,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
|
||||
"name": "/mtusi-messenger-lab4-test-postgres-1",
|
||||
"service": "postgres",
|
||||
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:15:04.743334637Z",
|
||||
"pid": 482062,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
|
||||
"name": "/mtusi-messenger-lab4-test-redis-1",
|
||||
"service": "redis",
|
||||
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:35.199893635Z",
|
||||
"pid": 508916,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"after_restart": [
|
||||
{
|
||||
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
|
||||
"name": "/mtusi-messenger-lab4-test-nginx-1",
|
||||
"service": "nginx",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:10.035280757Z",
|
||||
"pid": 513054,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
|
||||
"name": "/mtusi-messenger-lab4-test-api-2",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:03.796170548Z",
|
||||
"pid": 512362,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
|
||||
"name": "/mtusi-messenger-lab4-test-api-1",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:03.782576826Z",
|
||||
"pid": 512342,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
|
||||
"name": "/mtusi-messenger-lab4-test-cleanup-1",
|
||||
"service": "cleanup",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:20.411799591Z",
|
||||
"pid": 514062,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
|
||||
"service": "minio-bootstrap",
|
||||
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:17:16.679335904Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
|
||||
"name": "/mtusi-messenger-lab4-test-migrate-1",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:17:16.521163474Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-1",
|
||||
"service": "minio",
|
||||
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:56.858209997Z",
|
||||
"pid": 511450,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
|
||||
"name": "/mtusi-messenger-lab4-test-postgres-1",
|
||||
"service": "postgres",
|
||||
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:58.236126757Z",
|
||||
"pid": 511695,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
|
||||
"name": "/mtusi-messenger-lab4-test-redis-1",
|
||||
"service": "redis",
|
||||
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:56.95034238Z",
|
||||
"pid": 511521,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"after_failover": [
|
||||
{
|
||||
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
|
||||
"name": "/mtusi-messenger-lab4-test-nginx-1",
|
||||
"service": "nginx",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:10.035280757Z",
|
||||
"pid": 513054,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
|
||||
"name": "/mtusi-messenger-lab4-test-api-2",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:43.290395636Z",
|
||||
"pid": 515682,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
|
||||
"name": "/mtusi-messenger-lab4-test-api-1",
|
||||
"service": "api",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:03.782576826Z",
|
||||
"pid": 512342,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
|
||||
"name": "/mtusi-messenger-lab4-test-cleanup-1",
|
||||
"service": "cleanup",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:20.411799591Z",
|
||||
"pid": 514062,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
|
||||
"service": "minio-bootstrap",
|
||||
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:17:39.599833122Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
|
||||
"name": "/mtusi-messenger-lab4-test-migrate-1",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:17:39.442058694Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-1",
|
||||
"service": "minio",
|
||||
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:56.858209997Z",
|
||||
"pid": 511450,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
|
||||
"name": "/mtusi-messenger-lab4-test-postgres-1",
|
||||
"service": "postgres",
|
||||
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:58.236126757Z",
|
||||
"pid": 511695,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
|
||||
"name": "/mtusi-messenger-lab4-test-redis-1",
|
||||
"service": "redis",
|
||||
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:56.95034238Z",
|
||||
"pid": 511521,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
}
|
||||
],
|
||||
"after_update_and_failures": [
|
||||
{
|
||||
"id": "20b4b219e7c8cea5bae4a8834cc5df68399bc306b005e57509b060aac28cbd89",
|
||||
"name": "/mtusi-messenger-lab4-test-api-4",
|
||||
"service": "api",
|
||||
"image_id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:19:51.393481452Z",
|
||||
"pid": 528706,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "429367307e5f8f1f9e1ef20f378f308849cf51cddfdf2f8b83ef02d915d26455",
|
||||
"name": "/mtusi-messenger-lab4-test-api-3",
|
||||
"service": "api",
|
||||
"image_id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:19:30.856008463Z",
|
||||
"pid": 526806,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
|
||||
"name": "/mtusi-messenger-lab4-test-nginx-1",
|
||||
"service": "nginx",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:17:10.035280757Z",
|
||||
"pid": 513054,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
|
||||
"name": "/mtusi-messenger-lab4-test-cleanup-1",
|
||||
"service": "cleanup",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:18:21.285437286Z",
|
||||
"pid": 518374,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
|
||||
"service": "minio-bootstrap",
|
||||
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:18:17.550878221Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
|
||||
"name": "/mtusi-messenger-lab4-test-migrate-1",
|
||||
"service": "migrate",
|
||||
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"status": "exited",
|
||||
"started_at": "2026-11-04T18:18:17.394179901Z",
|
||||
"pid": 0,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
|
||||
"name": "/mtusi-messenger-lab4-test-minio-1",
|
||||
"service": "minio",
|
||||
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:18:22.354522515Z",
|
||||
"pid": 518578,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
|
||||
"name": "/mtusi-messenger-lab4-test-postgres-1",
|
||||
"service": "postgres",
|
||||
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:58.236126757Z",
|
||||
"pid": 511695,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
},
|
||||
{
|
||||
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
|
||||
"name": "/mtusi-messenger-lab4-test-redis-1",
|
||||
"service": "redis",
|
||||
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-04T18:16:56.95034238Z",
|
||||
"pid": 511521,
|
||||
"exit_code": 0,
|
||||
"oom_killed": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"images": {
|
||||
"sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284": {
|
||||
"id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"repo_digests": [
|
||||
"nginx@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051": {
|
||||
"id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
|
||||
"repo_digests": [
|
||||
"mtusi-messenger@sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9": {
|
||||
"id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
|
||||
"repo_digests": [
|
||||
"mtusi-messenger-minio-bootstrap@sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39": {
|
||||
"id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
|
||||
"repo_digests": [
|
||||
"mtusi-messenger-minio@sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74": {
|
||||
"id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
|
||||
"repo_digests": [
|
||||
"postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9": {
|
||||
"id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
|
||||
"repo_digests": [
|
||||
"redis@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb": {
|
||||
"id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
|
||||
"repo_digests": [
|
||||
"mtusi-messenger@sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb"
|
||||
],
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
}
|
||||
},
|
||||
"commands": [
|
||||
{
|
||||
"command": "make check validate test-stand-up TEST_PROJECT=mtusi-messenger-lab4-test",
|
||||
"started_at_utc": "2026-11-04T18:14:58.640914+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:15:23.342961+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "make test-docker TEST_PROJECT=mtusi-messenger-lab4-test PYTEST_ARGS=--tb=short",
|
||||
"started_at_utc": "2026-11-04T18:15:23.691655+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:16:39.997612+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "make test-stand-smoke test-stand-idempotency TEST_PROJECT=mtusi-messenger-lab4-test",
|
||||
"started_at_utc": "2026-11-04T18:16:39.997664+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:16:44.094964+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "make test-stand-persistence TEST_PROJECT=mtusi-messenger-lab4-test",
|
||||
"started_at_utc": "2026-11-04T18:16:44.152939+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:17:20.879999+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "make test-stand-failover TEST_PROJECT=mtusi-messenger-lab4-test",
|
||||
"started_at_utc": "2026-11-04T18:17:20.936053+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:17:48.976224+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": ".venv/bin/python -m scripts.check_s3 --base-url http://localhost:18080 --with-outage --with-faults",
|
||||
"started_at_utc": "2026-11-04T18:17:49.039051+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:18:28.077697+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": ".venv/bin/python -m scripts.upload_load --base-url http://localhost:18080 --concurrency 10 --output <temporary>/uploads.json",
|
||||
"started_at_utc": "2026-11-04T18:18:28.077774+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:18:35.361777+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": ".venv/bin/python -m scripts.check_grade4 --base-url http://localhost:18080",
|
||||
"started_at_utc": "2026-11-04T18:18:35.361808+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:19:58.301508+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "make test-nginx",
|
||||
"started_at_utc": "2026-11-04T18:19:58.404365+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:20:14.687715+00:00",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"command": "docker compose --env-file .env down --volumes --remove-orphans",
|
||||
"started_at_utc": "2026-11-04T18:20:14.687746+00:00",
|
||||
"finished_at_utc": "2026-11-04T18:20:25.705736+00:00",
|
||||
"exit_code": 0
|
||||
}
|
||||
],
|
||||
"remaining_test_resources": {
|
||||
"container": [],
|
||||
"network": [],
|
||||
"volume": []
|
||||
},
|
||||
"minio_source_release": "RELEASE.2025-10-15T17-29-55Z",
|
||||
"mc_source_release": "RELEASE.2025-08-13T08-35-41Z"
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
62c0534ad74d9f0e2982e97d1fbadb66d80c493ed237df5bcb9fbb797062eb38 .dockerignore
|
||||
0e37adcffea6cf3e485e8df17f64299f45aff55bdd35c8cb600aafe2ea071279 .env.example
|
||||
e10785802b87474192d20e23a31f2ff10ec24eb336642b2743ad341d1afa21be AGENTS.md
|
||||
cf48b9f6dd23f288d3064e22bb755c490e076a2b6cfc034fc6c7b87c1185ade1 CODE_STYLE.md
|
||||
2ea849842e877eca40e5dd853a5299a81be15fa414476fbfe434ac487f59ec1f Dockerfile
|
||||
27cde38208eed865ddd605d902b409850d5e9fb8e2cbd2f15f151533247592dd Makefile
|
||||
274785e9cfb1af978fcdf254f3b9c93847216d9183cf290098f857f174f43b41 alembic.ini
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/__init__.py
|
||||
5839d94c506f0362278a4f85bca8346774f5f323112963dd3830e525a069fb61 app/__main__.py
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/core/__init__.py
|
||||
7eaffea8127600b33d95bb1ec9171cddfca7d9464db7d49cecbc735b61d3f4d8 app/core/cleanup.py
|
||||
2a4a236bc37d2806a072d216adf9d7884a4eb39e7626f681206a04e64f4a3493 app/core/config.py
|
||||
0f37965b0c49f2ae6a2f17262057c19cb0c27c82dbf56b72e2fba6a025ee6d98 app/core/database.py
|
||||
02326fe518487718ae30334a5f4ebd3a96b63b2222cf112ac4fce33d401e613b app/core/image_worker.py
|
||||
da4dcfda3dabeb595aa7b917d321e794074676fd258828c84be4322b917ac43c app/core/instance.py
|
||||
aedcaa71c991f0147603bad32bbd105ebdc689cfc388d274e83ce693f32ba534 app/core/lifecycle.py
|
||||
5b6585c038289868e98c4c0de6d3004a74f0190c19f6f8b78beca116c9a799ca app/core/redis.py
|
||||
290d97e3551e1661feb04debc0272c9c91fe7636d1771de9e78055c64e23933c app/core/request_capacity.py
|
||||
e1f3f08c65cf17ea24574682050e0435d5a5c22b7b2194ea40aad941716871b4 app/core/request_id.py
|
||||
e519cdff5893f9c371f7756d96a5d92697dc01909e385b091652a541eee7b837 app/core/s3.py
|
||||
7717e9ea31415dd9803f14e6290e709bfb63aa38cb1524db9b04881cf2512256 app/core/security.py
|
||||
6acb951afe3a628d9bd9ae25ab8c321b8debb0f481c745881a7af1f48e620c90 app/domain/__init__.py
|
||||
92276fe889ea7e6901462d6acbecb481b33063723746a0c39ac2da56bc01ee4f app/domain/attachments.py
|
||||
d7026b518eb7f3f054a5aaa5307171db031718c0c70555d40c03e7aabc9f539d app/domain/auth.py
|
||||
33f20c63c3fdcb491adb2153a147e822e0a16de4a5f634355a6d2db514aaa668 app/domain/chats.py
|
||||
31138696454b4cb8e8e81249a54c054ed7dfa05f389b6d097fe99cbf0ef5e5d8 app/domain/health.py
|
||||
96e5d43321bf8f518992baf64d5a497ada19733c57ff7c16b36e8e34db0a3e07 app/domain/idempotency.py
|
||||
c93660a0578dcea83fa2fd04cff287833284c1efd555803736273c4298c5b873 app/domain/image_validation.py
|
||||
63409c613751b0fab06d00587fae2209609a4fe85ad1ba6b2a7fe460715aacef app/domain/images.py
|
||||
beca98c31471df3b3fdd0c228c6fc6cb70410a81dd2e0f046c35cfcc0a144e68 app/domain/messages.py
|
||||
df9aa94fab247467fe5f6fa030bdf3e587b5749060f0c3d663f5bd3e57e8dfe9 app/domain/pagination.py
|
||||
31ec7ba4c47ae6c31661eb8b4231efe9f561784c3bc0555c7d9b9bde804fe88f app/domain/users.py
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/endpoints/__init__.py
|
||||
6f7dc5e3aabf9e26d29dba89ad15e573679a665711ffef96b2b30e7a7618222c app/endpoints/attachments.py
|
||||
0cb1dded40038b80eafabcb288f43b1b7fcac1d8d41898e657eb9b62afaa2aab app/endpoints/auth.py
|
||||
2fc7d2b4f155e1cdba6d02ac157b1c2d74932c930d6cba7ac93c0a811af1494a app/endpoints/chats.py
|
||||
86f2c2e9b18004b82e099034d0754b27395381a98f9a9804546ffaf5948742f1 app/endpoints/dependencies.py
|
||||
b26a15a170624acffff0b506031cfffdd9e0e45f42ebb20dad1edd0fc90c73fe app/endpoints/health.py
|
||||
a6424660ce901defac67b5d66a882f8051a531328fc6d4aa998c407ca276c4e3 app/endpoints/messages.py
|
||||
13fe05bec4b4b0a9e7a2f90756065bf97e257601b119a71465b24fc0144f3bf9 app/endpoints/users.py
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/errors/__init__.py
|
||||
70febdabcf684aba98a7106dd312cdb59f9f0ac0b494a91277ea2d979888a1e4 app/errors/exceptions.py
|
||||
40115a09d9e69ddcc8c38afd6f1c60383ce753f46bc5be62e9cd60993271fa71 app/errors/handlers.py
|
||||
a36a2a1b982a5f8ca8033bd6fa0286f0a4fe0ed4b9bc6c217eae01d9d8070953 app/main.py
|
||||
5154a347cb6588477b17935f24b61a5122f1952f4238dfcd762e54f430ac9530 app/models/__init__.py
|
||||
a95f02dbd8f5aa54a9dafb3cb516fa0cbb9b1f5c75ef15ef57a167ad56ea2c57 app/models/attachment.py
|
||||
548f10800483dd91a4d321c3cef845f3f5b126169a50474bf846f78839fdb43e app/models/base.py
|
||||
7076f06111aca1e3845a8515112fa1302da9bd331cf79c699259c91d3d23c9cf app/models/chat.py
|
||||
dfb64d43bd519d06f09e479eab4bdd277f317fdb92e206fdedf0afbed1fe18ad app/models/chat_member.py
|
||||
35eaa453501b48d178d7b17fd22314783386b64951cbb518b450698ba65219f8 app/models/counter.py
|
||||
693039c889d92ebdb5ef3fc6833278f59b4aa88f802b4726b9c3bf441f83fb18 app/models/idempotency_key.py
|
||||
9bbf725b6eae51e23e4fc71aea3dbc137d5a275b069e7e378351773297a7559f app/models/message.py
|
||||
b5f1bbdc4370e2671427c8d7590aba58916b1981f225c4734d8bdeb944b1539a app/models/user.py
|
||||
7c9d4ef216ed8b027ddfbad9746147cf0fed6463baf396b7244bd9b1b286e9ee app/schemas/__init__.py
|
||||
bd00b00ded1c48458b7f3e1ebb51bfb481268de92599d41781cd71e782023b0f app/schemas/attachment.py
|
||||
9a3f14dc23e704d9fdce681ae7187a6b95638f9736711a8edef245fbbaea3eb6 app/schemas/base.py
|
||||
2111aef9c942b34d02afa99da6d23a1d5ca0f6cfd75a1c0ebfb2ba85a29584b1 app/schemas/chat.py
|
||||
1092519716f0c8e5800ccf666c3fe52e8b1e617bee61843d6677fafe83d63d9a app/schemas/message.py
|
||||
617bcba33dcb01e4f66a50b5c425ebcbc1aef1c6148148f0eb862a7743bcf9b6 app/schemas/session.py
|
||||
01fc238693c3686595cc910fadce6200e9753d0c4a0397ff327d8a0c937dc4df app/schemas/user.py
|
||||
33296ca85924a22644fda573be072e5b810a284b939469580445925a85f0b5e9 app/storage/__init__.py
|
||||
59606da77e97e883fd2bdbc4c1ab008ab4165ba545c72c623777e03936fd9805 app/storage/attachments.py
|
||||
e479c8be1f56030fc4faaa74417fb959fb2c1f0c57f72974df8b538ff1209a7b app/storage/chats.py
|
||||
a8cb92ff2fc5de87eb072c9997b1dc33e4c490057c6a6eb8f46d6d1067771345 app/storage/health.py
|
||||
2a9243ad419e7cc48ce0c6da13053df6773bea53a2342eb408cb3ebe2455f921 app/storage/idempotency.py
|
||||
e4b71490655095d4dc9b1fccb04820d7cfaf48295affe05d6c6e1e085c8cbaf0 app/storage/messages.py
|
||||
bec371be921841e7d194f4874c43363a871496299f4f5da61a16c5e0211051fc app/storage/s3.py
|
||||
aeb6851a54d71051912ad588c47de3d5a3d92d518a0a198dbb3e283dd64837c7 app/storage/sessions.py
|
||||
46a0562d557ac96bd49f9975ff25ac8cce7f7f803b8c280d1d0bf2463ebfdaec app/storage/types.py
|
||||
f9dc7df7affda7144991944ff20a1ae5b372921b14fd391afa189d0e613dc45e app/storage/users.py
|
||||
46f25ac85ff2a20a52cddf67b38c69f340bf007a355cd2e18c46e6aab9d22757 compose.test.yaml
|
||||
df8b4d1347466f0aa95a9183e04a8496e29614652229cb3dfd0b78a48be13500 compose.yaml
|
||||
381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37 contracts/examples.json
|
||||
baaeb5a2233096a50d791f521bfc6dd66a23e516f81833c2b17c554cc0bab5d8 contracts/infrastructure.json
|
||||
c5c0ebdcf205018126c5bb91e2512b11f543827e76baa21d776210d05a0b2cca contracts/openapi.json
|
||||
4b492cfedc7d65eb88d8217eefb07db541926ae919770e0d535e767d73fa7947 infra/minio/Dockerfile
|
||||
cfdd8697694458869202d1536edda66c6c376c6d9757720d6b38ac91526ee4bb infra/minio/bootstrap.py
|
||||
8484d85c7d7ad6c7e282300288f614b47a2ff5329289e38dd9d111f0c8e76584 infra/nginx/nginx.conf
|
||||
9d4e7faeab76340dadda9916430048c32c849fb2bf02206653ea827e3304b666 infra/redis/redis.conf
|
||||
0f6fe080f5f7cd1d77da5f8ee9989874e9617bd7f4830f90b91978a5bfb88056 infra/redis/start-redis.sh
|
||||
7d0945d93e302c34c3f26455b152f9d657110ee0a94a5074d85963bc7aa60526 lab.json
|
||||
37dce514d48aa4e021e063f6ebb8fb3e34275f6a76f51926d789802cad8dfa75 migrations/env.py
|
||||
7751187956408f7a64742e0bf746d0684d24013bfe11d1895d936d6757da1766 migrations/script.py.mako
|
||||
4ae6aee859bcbcdb238d4cb2fa35419503d7a1d5a3caaffd0f0d2e9e9dce2937 migrations/versions/49d2c947a3f1_add_idempotency_keys.py
|
||||
0eb1a4e624c0e81a8955245a0071a9e0792722ee0d056df257ddc23062ccc8aa migrations/versions/8f31b94a2d10_add_attachments.py
|
||||
cbe0b9e7584f26c0b859936a2584c147fe3f1ffaa9eb55c85ff753ecb10aa5a4 migrations/versions/abd273076068_add_counters.py
|
||||
dd0e8073817e3a08bae19b17d4d502aa0dca0639f95d0b08caaf0e5dce16f302 migrations/versions/db06fd3126ff_create_initial_schema.py
|
||||
92ce287c8eb0aa508e0efd4553650b4ebd2474d1987587d6467c26e316e6a547 pytest.ini
|
||||
4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525 requirements-dev.txt
|
||||
98fab4585e4ac85841ee93836c9bcacbd09627e278cd54766ba0909a43952191 requirements.txt
|
||||
599958169a2b3c35b72b9b27f54d1c156032b2fadf041cc0e905bd49e48d8c60 scripts/__init__.py
|
||||
7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671 scripts/check_contract.py
|
||||
20fb4ddaa3ab13bc5c350b6762298c6e26dcbf23fcd8e66e496c070d09cdec3b scripts/check_grade4.py
|
||||
1ebdc8a4423305b6b972254e0b856b9381f7a5279aaa537a3c9d3cdb791adccf scripts/check_idempotency.py
|
||||
c95f74ebfec55b047424d65ed201519a79cd3292b9067fcf6913f5ac1bf3ab0b scripts/check_nginx.py
|
||||
4a4b5334ca1a717dfb341dc3ecc4d862b19fcec18d0909b087125800be9ab46e scripts/check_s3.py
|
||||
c46b007560b879cf025a6162a7fe3e405690b23d2716654bb5b4079fa01fbb4a scripts/collect_evidence.py
|
||||
0926fac2e73573a6e8b32c254bfc635132823aed9c4b429e91bf180e047bc32f scripts/deployment.py
|
||||
4cf255992270adfdaa0a3fd304050ed858b7a49d9e3c8856337a310cf59f8596 scripts/expiry_demo.sh
|
||||
21f2e3baa4d9615e0411edbecfbdc394d471b70b4f237d1e45557c9602b0ecad scripts/explain_history.py
|
||||
af3422b5b039583980599765cea8315546a1a3de80671f78c7c205faa2bac65f scripts/failover_all.py
|
||||
e06776d5e44eded34a7fc849ba4cabe955428cf9f924e077a48511656538e7b2 scripts/failure_demo.sh
|
||||
0d132cc0528257b6bf3289607fd796ab2a176748d4921b6ac0c56da28b15154e scripts/init_local.py
|
||||
0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b scripts/restart.example.sh
|
||||
5dcdcb09e56e13e223f5b63e855d853a5a669e6a2b5ff4cb5d4800dd44231352 scripts/restart.sh
|
||||
60d48ba1c8050a79f8a3725a0ea7bba34dadba3bef0b86bd2d2e6a768e6ca7d5 scripts/rolling_update.py
|
||||
563376e1d5bfb235790466d5b51ad0559c8c6a0e27da4109ce82705b73f42670 scripts/s3_faults.py
|
||||
1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811 scripts/stop-one.example.sh
|
||||
c78f3fee788c907764162069ee09e3bd043b3b3eb3823f5bbb8f84dd1ab5bed0 scripts/stop-one.sh
|
||||
b67cf021073a1b0fed6d536f3fc1f41d9bc34834090f02f2db3fdff8b8a6e356 scripts/upload_load.py
|
||||
4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54 scripts/validate_openapi.py
|
||||
0d23d308c846d2db45e14c6890e6b50be5bdd5a17e89a87b5b8e1e75b27aefc9 student-tests/conftest.py
|
||||
8c11a6fbc9003c2d92225bfc676edfffe416e70d8b6972a864cf6c5c0b64c95b student-tests/helpers.py
|
||||
b09d533125981d19774bff19b02ff096c3b367b8bda1dc62fc4647e8816063c7 student-tests/test_attachments.py
|
||||
20cd1c8ea397fe558a9ded78723dd319cabe47f520a631375171c448a01b1de2 student-tests/test_auth.py
|
||||
f57751337441fbc909549d553cd32e20d7f7918dc42f2000c15f9ba2706349a4 student-tests/test_chats.py
|
||||
b9c2ed985bb2cf0261bfb928f0bcc2694c8980a5c55fea4c445d0ca410ca30c6 student-tests/test_concurrency.py
|
||||
b8c9a1f7535318b946adff13b11ac2459622a1a9b23962f0981d0fc7793de0c9 student-tests/test_constraints.py
|
||||
76b4fe648fd257eb603cfc06da76b59b783986f96a9b357bc87238210757ef34 student-tests/test_errors.py
|
||||
045203d83f8ff50e48b8fad4328dba7f12de8adffebef597efadfd40c79ed5c1 student-tests/test_failures.py
|
||||
2594b7bce03d36e6c32c1644e76f4d8f40e7099de6c687e13a74706e329bcefe student-tests/test_health.py
|
||||
edaf79363adb942f9501d933deb86e39aed558f32c1a73188ae2d18d088378ae student-tests/test_history_plan.py
|
||||
640a12067597a7e576d3db0fb6f9bbb14639a14d701f6a9b682ed97ad70cdf8e student-tests/test_idempotency.py
|
||||
722951c437488d0c5d63839d418b2456c37a40c9e8dc55a52c7e8ea3a4066c3c student-tests/test_instance.py
|
||||
c6d2ce0d47b2eae767a9f4d2b63333eb865c505cfaa3dffb5236877187318fdc student-tests/test_lifecycle.py
|
||||
71e48fab2816753b213603bd977c9c1babb2b540596cd3401c1e62a6f538a3b5 student-tests/test_limits.py
|
||||
cc040ae5ee2f196362fab4c7006711bf8d8c7f5e2f56acc8c74646ef2efb801d student-tests/test_messages.py
|
||||
1ef7a28cb478169001a64974ff0632a55771c1f3faa02ea40a919cb722bf9a37 student-tests/test_migrations.py
|
||||
2c573efd6b0a6204a8bac483bb6e2594c565f2d32faa04ebc3540298eb02af2d student-tests/test_pagination.py
|
||||
0cea69086ff8ecc630eccabdb1a7ff79e72bb2df067cb4a59ce8ad31ba5f08a4 student-tests/test_s3_stream.py
|
||||
cb981f15ad186faa2e1f8b6aecb7766eed66378b6db9dfc496dd1d76afc94b1b student-tests/test_sessions.py
|
||||
bf840d8dcb6094412b9e2947df4a0a3c3f32c5d80af9df19693ed5b52b9c8d6d student-tests/test_users.py
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/__init__.py
|
||||
8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d tests/client.py
|
||||
99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 tests/fixtures/sample.png
|
||||
a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13 tests/image_probe.py
|
||||
6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933 tests/load.py
|
||||
1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b tests/resilience.py
|
||||
52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db tests/smoke.py
|
||||
@@ -0,0 +1,396 @@
|
||||
$ make check validate test-stand-up TEST_PROJECT=mtusi-messenger-lab4-test
|
||||
.venv/bin/python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env up -d --build
|
||||
Image mtusi-messenger:lab4 Building
|
||||
Image mtusi-messenger:lab4 Building
|
||||
Image mtusi-messenger:lab4 Building
|
||||
Image mtusi-messenger-minio:lab4 Building
|
||||
Image mtusi-messenger-minio-bootstrap:lab4 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 2.80kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [api internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 797B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [minio-bootstrap internal] load build definition from Dockerfile
|
||||
#3 transferring dockerfile: 1.21kB done
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [minio internal] load metadata for docker.io/library/python:3.12.15-slim-bookworm
|
||||
#4 DONE 1.5s
|
||||
|
||||
#5 [minio-bootstrap internal] load metadata for docker.io/library/golang:1.24.8-bookworm
|
||||
#5 DONE 1.5s
|
||||
|
||||
#6 [minio internal] load .dockerignore
|
||||
#6 transferring context: 196B done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [minio-bootstrap internal] load build context
|
||||
#7 transferring context: 3.50kB done
|
||||
#7 DONE 0.1s
|
||||
|
||||
#8 [migrate internal] load build context
|
||||
#8 transferring context: 318.96kB 0.0s done
|
||||
#8 DONE 0.1s
|
||||
|
||||
#9 [minio compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848
|
||||
#9 resolve docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 0.1s done
|
||||
#9 DONE 0.2s
|
||||
|
||||
#10 [cleanup runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70
|
||||
#10 resolve docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 0.1s done
|
||||
#10 DONE 0.2s
|
||||
|
||||
#11 [minio server 1/1] COPY --from=compiler /out/minio /usr/local/bin/minio
|
||||
#11 CACHED
|
||||
|
||||
#12 [minio-bootstrap bootstrap 1/2] COPY --from=compiler /out/mc /usr/local/bin/mc
|
||||
#12 CACHED
|
||||
|
||||
#13 [minio runtime 2/2] RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
|
||||
#13 CACHED
|
||||
|
||||
#14 [minio compiler 3/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
#14 CACHED
|
||||
|
||||
#15 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
#15 CACHED
|
||||
|
||||
#16 [minio-bootstrap bootstrap 2/2] COPY infra/minio/bootstrap.py /bootstrap.py
|
||||
#16 CACHED
|
||||
|
||||
#17 [migrate 4/8] COPY requirements.txt .
|
||||
#17 CACHED
|
||||
|
||||
#18 [migrate 5/8] RUN pip install -r requirements.txt
|
||||
#18 CACHED
|
||||
|
||||
#19 [migrate 7/8] COPY migrations ./migrations
|
||||
#19 CACHED
|
||||
|
||||
#20 [migrate 6/8] COPY alembic.ini ./
|
||||
#20 CACHED
|
||||
|
||||
#21 [migrate 3/8] RUN useradd --system --uid 10001 --no-create-home appuser
|
||||
#21 CACHED
|
||||
|
||||
#22 [migrate 2/8] WORKDIR /app
|
||||
#22 CACHED
|
||||
|
||||
#23 [migrate 8/8] COPY app ./app
|
||||
#23 CACHED
|
||||
|
||||
#24 [minio-bootstrap] exporting to image
|
||||
#24 exporting layers done
|
||||
#24 exporting manifest sha256:d7d356b436b22b1950d123996ec850fbe44e7b9b1e5c0f6dfee6c5ed8046fc30 0.0s done
|
||||
#24 exporting config sha256:416fb93e9d28043ec257557a451b5b260a8ec2141fd6bffcf5d1fa090bc5aee0 0.0s done
|
||||
#24 exporting attestation manifest sha256:7159f8f74e3f3ab0500179c06398cf1db149d22995073f2793fbcc3e705348c4
|
||||
#24 exporting attestation manifest sha256:7159f8f74e3f3ab0500179c06398cf1db149d22995073f2793fbcc3e705348c4 0.2s done
|
||||
#24 exporting manifest list sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9 0.1s done
|
||||
#24 naming to docker.io/library/mtusi-messenger-minio-bootstrap:lab4
|
||||
#24 naming to docker.io/library/mtusi-messenger-minio-bootstrap:lab4 0.0s done
|
||||
#24 unpacking to docker.io/library/mtusi-messenger-minio-bootstrap:lab4
|
||||
#24 unpacking to docker.io/library/mtusi-messenger-minio-bootstrap:lab4 0.1s done
|
||||
#24 DONE 0.6s
|
||||
|
||||
#25 [minio] exporting to image
|
||||
#25 exporting layers done
|
||||
#25 exporting manifest sha256:91da74b8aa80d8f1967b0b1c34cfd963a7db8186f3f6b502396145eb1fc52c3f 0.0s done
|
||||
#25 exporting config sha256:ae66a6f1ec14052043afc5d2bc4d41de5ad1986ea0ea7c44f9f02c7a9837f7d8 0.0s done
|
||||
#25 exporting attestation manifest sha256:ac35b49ea51ebffe7f862b5fcfb5edb5fa9a1ce064b79aa2732473d216fc717f 0.2s done
|
||||
#25 exporting manifest list sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39 0.1s done
|
||||
#25 naming to docker.io/library/mtusi-messenger-minio:lab4 0.0s done
|
||||
#25 unpacking to docker.io/library/mtusi-messenger-minio:lab4 0.1s done
|
||||
#25 DONE 0.6s
|
||||
|
||||
#26 [migrate] exporting to image
|
||||
#26 exporting layers done
|
||||
#26 exporting manifest sha256:bc6f252b407c7f0f55cef33ac619c87c0af8aa3444bfd48cff70ec5d13a9d3c6 0.1s done
|
||||
#26 exporting config sha256:07e6fed33c1185350eaaf0ca71bef77b85b69b9f702cc9921b9ded39d0a059dc 0.1s done
|
||||
#26 exporting attestation manifest sha256:2b7da561e7004b93f36c46abac5fbf5d7517b9f54885ed119fab6d19c1a153da 0.2s done
|
||||
#26 exporting manifest list sha256:7e3e13a38949ace57409487e70b8510202c0d6ffdeecafcdb400a16c80c7a817 0.1s done
|
||||
#26 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
|
||||
#26 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
|
||||
#26 DONE 0.7s
|
||||
|
||||
#27 [api] exporting to image
|
||||
#27 exporting layers done
|
||||
#27 exporting manifest sha256:5c384a19f40a3937220bb527abc5857f2db10a6b9215ac3440ee1516cc64e779 0.1s done
|
||||
#27 exporting config sha256:4b39ae1766eacf6ef7c55627df3daadcb46ce8ee26f4dd63ada8047fa861ce87 0.1s done
|
||||
#27 exporting attestation manifest sha256:8f6444f4580f96c70c2d6ea0cfd2d3113b93a6c569aea6cde5d714fafa39055a 0.2s done
|
||||
#27 exporting manifest list sha256:01e56602fc2243976c8e491e5dfaebff3cf8aee1cfdf168e2328a442bc47dfca 0.1s done
|
||||
#27 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
|
||||
#27 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
|
||||
#27 DONE 0.7s
|
||||
|
||||
#28 [cleanup] exporting to image
|
||||
#28 exporting layers done
|
||||
#28 exporting manifest sha256:6cc0026405e8326374b7eb6900ce3c4ce8c069dedafb7072793ca7fcba6378d4 0.1s done
|
||||
#28 exporting config sha256:421efb8ea526249eff500a1e0a2936e2334d8244ef3d9558f1a70feffd7afc9b 0.1s done
|
||||
#28 exporting attestation manifest sha256:70bb6141a85943d37fabc6631399a7bce798427ee873e9036b3099b27ae5edd9 0.2s done
|
||||
#28 exporting manifest list sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051 0.1s done
|
||||
#28 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
|
||||
#28 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
|
||||
#28 DONE 0.7s
|
||||
|
||||
#29 [minio-bootstrap] resolving provenance for metadata file
|
||||
#29 DONE 0.2s
|
||||
|
||||
#30 [minio] resolving provenance for metadata file
|
||||
#30 DONE 0.2s
|
||||
|
||||
#31 [api] resolving provenance for metadata file
|
||||
#31 DONE 0.2s
|
||||
|
||||
#32 [migrate] resolving provenance for metadata file
|
||||
#32 DONE 0.1s
|
||||
|
||||
#33 [cleanup] resolving provenance for metadata file
|
||||
#33 DONE 0.0s
|
||||
Image mtusi-messenger-minio-bootstrap:lab4 Built
|
||||
Image mtusi-messenger:lab4 Built
|
||||
Image mtusi-messenger:lab4 Built
|
||||
Image mtusi-messenger:lab4 Built
|
||||
Image mtusi-messenger-minio:lab4 Built
|
||||
Network mtusi-messenger-lab4-test_objects Creating
|
||||
Network mtusi-messenger-lab4-test_default Creating
|
||||
Volume mtusi-messenger-lab4-test_redisdata Creating
|
||||
Network mtusi-messenger-lab4-test_objects Creating
|
||||
Network mtusi-messenger-lab4-test_default Creating
|
||||
Volume mtusi-messenger-lab4-test_redisdata Creating
|
||||
Volume mtusi-messenger-lab4-test_pgdata Creating
|
||||
Volume mtusi-messenger-lab4-test_pgdata Creating
|
||||
Volume mtusi-messenger-lab4-test_miniodata Creating
|
||||
Volume mtusi-messenger-lab4-test_miniodata Creating
|
||||
Volume mtusi-messenger-lab4-test_pgdata Created
|
||||
Volume mtusi-messenger-lab4-test_pgdata Created
|
||||
Volume mtusi-messenger-lab4-test_miniodata Created
|
||||
Volume mtusi-messenger-lab4-test_miniodata Created
|
||||
Volume mtusi-messenger-lab4-test_redisdata Created
|
||||
Volume mtusi-messenger-lab4-test_redisdata Created
|
||||
Network mtusi-messenger-lab4-test_default Created
|
||||
Network mtusi-messenger-lab4-test_default Created
|
||||
Container mtusi-messenger-lab4-test-redis-1 Creating
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Creating
|
||||
Network mtusi-messenger-lab4-test_objects Created
|
||||
Network mtusi-messenger-lab4-test_objects Created
|
||||
Container mtusi-messenger-lab4-test-minio-1 Creating
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Created
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Creating
|
||||
Container mtusi-messenger-lab4-test-minio-1 Created
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Creating
|
||||
Container mtusi-messenger-lab4-test-redis-1 Created
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Created
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Created
|
||||
Container mtusi-messenger-lab4-test-api-1 Creating
|
||||
Container mtusi-messenger-lab4-test-api-2 Creating
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Creating
|
||||
Container mtusi-messenger-lab4-test-api-1 Created
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Created
|
||||
Container mtusi-messenger-lab4-test-api-2 Created
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Creating
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Created
|
||||
Container mtusi-messenger-lab4-test-minio-1 Starting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Starting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Starting
|
||||
Container mtusi-messenger-lab4-test-minio-1 Started
|
||||
Container mtusi-messenger-lab4-test-minio-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Started
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Started
|
||||
Container mtusi-messenger-lab4-test-minio-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Starting
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Started
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Starting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Started
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-minio-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Exited
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Exited
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Starting
|
||||
Container mtusi-messenger-lab4-test-api-2 Starting
|
||||
Container mtusi-messenger-lab4-test-cleanup-1 Started
|
||||
Container mtusi-messenger-lab4-test-api-2 Started
|
||||
Container mtusi-messenger-lab4-test-api-1 Starting
|
||||
Container mtusi-messenger-lab4-test-api-1 Started
|
||||
Container mtusi-messenger-lab4-test-api-2 Waiting
|
||||
Container mtusi-messenger-lab4-test-api-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-api-2 Healthy
|
||||
Container mtusi-messenger-lab4-test-api-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Starting
|
||||
Container mtusi-messenger-lab4-test-nginx-1 Started
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env exec -T postgres sh -c 'createdb -U "$POSTGRES_USER" "$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
|
||||
|
||||
$ make test-docker TEST_PROJECT=mtusi-messenger-lab4-test PYTEST_ARGS=--tb=short
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 DOCKER_TESTS=1 .venv/bin/python -m pytest -q --tb=short
|
||||
........................................................................ [ 38%]
|
||||
........................................................................ [ 76%]
|
||||
............................................ [100%]
|
||||
188 passed in 73.96s (0:01:13)
|
||||
|
||||
$ make test-stand-smoke test-stand-idempotency TEST_PROJECT=mtusi-messenger-lab4-test
|
||||
.venv/bin/python tests/smoke.py --base-url "http://localhost:18080"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
.venv/bin/python -m scripts.check_idempotency --base-url "http://localhost:18080"
|
||||
PASS: 24 concurrent replays, 2 replicas, one message
|
||||
|
||||
$ make test-stand-persistence TEST_PROJECT=mtusi-messenger-lab4-test
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup About a minute ago Up About a minute 8080/tcp
|
||||
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio About a minute ago Up About a minute (healthy)
|
||||
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx About a minute ago Up About a minute (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
|
||||
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres About a minute ago Up About a minute (healthy) 127.0.0.1:15432->5432/tcp
|
||||
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis About a minute ago Up 9 seconds (healthy) 127.0.0.1:16379->6379/tcp
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 .venv/bin/python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "http://localhost:18080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup 2 minutes ago Up Less than a second 8080/tcp
|
||||
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio 2 minutes ago Up 23 seconds (healthy)
|
||||
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx 2 minutes ago Up 10 seconds (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
|
||||
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up 22 seconds (healthy) 127.0.0.1:15432->5432/tcp
|
||||
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis 2 minutes ago Up 23 seconds (healthy) 127.0.0.1:16379->6379/tcp
|
||||
|
||||
$ make test-stand-failover TEST_PROJECT=mtusi-messenger-lab4-test
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup 2 minutes ago Up Less than a second 8080/tcp
|
||||
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio 2 minutes ago Up 24 seconds (healthy)
|
||||
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx 2 minutes ago Up 10 seconds (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
|
||||
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up 22 seconds (healthy) 127.0.0.1:15432->5432/tcp
|
||||
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis 2 minutes ago Up 24 seconds (healthy) 127.0.0.1:16379->6379/tcp
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 .venv/bin/python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "http://localhost:18080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 4, "first_recovery_seconds": 1.885}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps -a api
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 34 seconds (healthy) 8080/tcp
|
||||
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Exited (0) 13 seconds ago
|
||||
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env up -d --wait api
|
||||
Container mtusi-messenger-lab4-test-minio-1 Running
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Running
|
||||
Container mtusi-messenger-lab4-test-redis-1 Running
|
||||
Container mtusi-messenger-lab4-test-api-1 Running
|
||||
Container mtusi-messenger-lab4-test-minio-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Starting
|
||||
Container mtusi-messenger-lab4-test-minio-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Starting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Started
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Started
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-minio-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Exited
|
||||
Container mtusi-messenger-lab4-test-api-2 Starting
|
||||
Container mtusi-messenger-lab4-test-api-2 Started
|
||||
Container mtusi-messenger-lab4-test-api-2 Waiting
|
||||
Container mtusi-messenger-lab4-test-api-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-minio-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-redis-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Waiting
|
||||
Container mtusi-messenger-lab4-test-postgres-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
|
||||
Container mtusi-messenger-lab4-test-migrate-1 Exited
|
||||
Container mtusi-messenger-lab4-test-redis-1 Healthy
|
||||
Container mtusi-messenger-lab4-test-api-2 Healthy
|
||||
Container mtusi-messenger-lab4-test-api-1 Healthy
|
||||
|
||||
$ .venv/bin/python -m scripts.check_s3 --base-url http://localhost:18080 --with-outage --with-faults
|
||||
PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console, MinIO outage/recovery
|
||||
PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup
|
||||
|
||||
$ .venv/bin/python -m scripts.upload_load --base-url http://localhost:18080 --concurrency 10 --output <temporary>/uploads.json
|
||||
{
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"attempts": 30,
|
||||
"fixture": {
|
||||
"format": "PNG",
|
||||
"width": 1500,
|
||||
"height": 1500,
|
||||
"size_bytes": 6761054,
|
||||
"sha256": "49cb1a9ec3e54c1f22c649fb787c50186c54e449ade4a6f6758a50b2e374504d"
|
||||
},
|
||||
"statuses": {
|
||||
"201": 12,
|
||||
"429": 18
|
||||
},
|
||||
"elapsed_seconds": 5.138,
|
||||
"successful_uploads": 12,
|
||||
"p95_seconds": 2.143,
|
||||
"memory": {
|
||||
"bf64a4a13acb": {
|
||||
"initial_bytes": 88752128,
|
||||
"sampled_peak_bytes": 162426880,
|
||||
"cgroup_peak_before_bytes": 161976320,
|
||||
"cgroup_peak_after_bytes": 162689024,
|
||||
"interval_ms": 20
|
||||
},
|
||||
"7e4180e529e1": {
|
||||
"initial_bytes": 89436160,
|
||||
"sampled_peak_bytes": 163692544,
|
||||
"cgroup_peak_before_bytes": 157728768,
|
||||
"cgroup_peak_after_bytes": 163692544,
|
||||
"interval_ms": 20
|
||||
}
|
||||
},
|
||||
"api_memory_limit": "1610612736",
|
||||
"notes": "20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries"
|
||||
}
|
||||
|
||||
$ .venv/bin/python -m scripts.check_grade4 --base-url http://localhost:18080
|
||||
PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords
|
||||
PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in 2.390s
|
||||
Сборка нового образа API
|
||||
Однократное применение совместимых миграций
|
||||
Заменена реплика bf64a4a13acb
|
||||
Заменена реплика 7e4180e529e1
|
||||
Готово: 2 здоровых реплик нового образа
|
||||
{"scenario": "rolling update", "replacements": 2, "migration_runs": 1, "successful_writes": 487, "failed_attempts": 13, "errors_by_kind": {"http_503": 13}, "elapsed_seconds": 25.341, "successful_rps": 19.218, "p95_seconds": 0.048}
|
||||
{"scenario": "replica_failure", "phase": "before", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
|
||||
{"scenario": "replica_failure", "phase": "stopped", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": false, "health": "unhealthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
|
||||
{"scenario": "replica_failure", "phase": "restored", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
|
||||
{"instance": "429367307e5f", "successful_writes": 777, "failed_attempts": 87, "errors_by_kind": {"http_429": 70, "http_503": 17}, "elapsed_seconds": 13.965, "successful_rps": 55.64, "first_recovery_seconds": 1.865}
|
||||
{"scenario": "replica_failure", "phase": "before", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
|
||||
{"scenario": "replica_failure", "phase": "stopped", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": false, "health": "unhealthy"}]}
|
||||
{"scenario": "replica_failure", "phase": "restored", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
|
||||
{"instance": "20b4b219e7c8", "successful_writes": 774, "failed_attempts": 96, "errors_by_kind": {"http_429": 79, "http_503": 17}, "elapsed_seconds": 14.127, "successful_rps": 54.79, "first_recovery_seconds": 2.042}
|
||||
PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Инфраструктура студента
|
||||
|
||||
В этом fork Compose включает Nginx, две API-реплики, PostgreSQL, Redis, MinIO,
|
||||
миграции, bootstrap bucket и cleanup. Актуальные команды запуска и файловые
|
||||
секреты — [RUNBOOK.md](../RUNBOOK.md), транзакции и ограничения MinIO —
|
||||
[LAB4_PLAN.md](../LAB4_PLAN.md). Результаты прогона — [evidence](../evidence/README.md).
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
FROM golang:1.24.8-bookworm AS compiler
|
||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
|
||||
go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
|
||||
go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
FROM python:3.12.15-slim-bookworm AS runtime
|
||||
RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
|
||||
USER minio
|
||||
|
||||
FROM runtime AS server
|
||||
COPY --from=compiler /out/minio /usr/local/bin/minio
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/minio" \
|
||||
org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z"
|
||||
VOLUME ["/data"]
|
||||
ENTRYPOINT ["/usr/local/bin/minio"]
|
||||
|
||||
FROM runtime AS bootstrap
|
||||
COPY --from=compiler /out/mc /usr/local/bin/mc
|
||||
COPY infra/minio/bootstrap.py /bootstrap.py
|
||||
ENTRYPOINT ["python", "/bootstrap.py"]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Идемпотентный private bucket и отдельная ограниченная учётная запись API."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def secret(name):
|
||||
value = (Path("/run/secrets") / name).read_text().strip()
|
||||
if not value:
|
||||
raise ValueError("Пустой файл секрета")
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
endpoint = os.environ["S3_ENDPOINT"]
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
root_user, root_password = secret("minio_root_user"), secret("minio_root_password")
|
||||
access, password = secret("s3_access_key_id"), secret("s3_secret_access_key")
|
||||
if access == root_user:
|
||||
raise ValueError("API не должен использовать root credentials")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
with urlopen(endpoint + "/minio/health/ready", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("MinIO не готов за 60 секунд")
|
||||
time.sleep(1)
|
||||
with TemporaryDirectory(prefix="minio-bootstrap-") as directory:
|
||||
config = Path(directory)
|
||||
# Credentials не передаются mc alias set через аргументы или окружение.
|
||||
config_file = config / "config.json"
|
||||
config_file.write_text(json.dumps({"version": "10", "aliases": {"storage": {
|
||||
"url": endpoint, "accessKey": root_user, "secretKey": root_password,
|
||||
"api": "S3v4", "path": "auto",
|
||||
}}}))
|
||||
config_file.chmod(0o600)
|
||||
|
||||
def mc(*args):
|
||||
result = subprocess.run(["mc", "--config-dir", directory, *args],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
|
||||
if result.returncode:
|
||||
raise RuntimeError("Ошибка bootstrap MinIO; значения credentials скрыты")
|
||||
|
||||
mc("mb", "--ignore-existing", "storage/" + bucket)
|
||||
mc("anonymous", "set", "none", "storage/" + bucket)
|
||||
policy = {
|
||||
"Version": "2012-10-17", "Statement": [
|
||||
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}"]},
|
||||
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/originals/*"]},
|
||||
],
|
||||
}
|
||||
policy_file = config / "policy.json"
|
||||
policy_file.write_text(json.dumps(policy))
|
||||
mc("admin", "policy", "create", "storage", "messenger-attachments", str(policy_file))
|
||||
# mc не поддерживает stdin для user add: вывод и ошибки команды скрыты.
|
||||
mc("admin", "user", "add", "storage", access, password)
|
||||
mc("admin", "policy", "attach", "storage", "messenger-attachments", "--user", access)
|
||||
print("MinIO private bucket and API policy configured", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print("MinIO bootstrap failed: " + type(error).__name__, flush=True)
|
||||
raise SystemExit(1) from None
|
||||
@@ -0,0 +1,140 @@
|
||||
# Единая точка входа: round-robin между API-репликами, без sticky sessions.
|
||||
# Директивы рассчитаны на закреплённый nginx:1.28.0-alpine.
|
||||
|
||||
worker_processes 2;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
# В логе используем request_id ответа API, для собственных ошибок — id nginx.
|
||||
map $upstream_http_x_request_id $response_request_id {
|
||||
"" $request_id;
|
||||
default $upstream_http_x_request_id;
|
||||
}
|
||||
|
||||
log_format requests escape=json
|
||||
'{"time":"$time_iso8601","request_id":"$response_request_id",'
|
||||
'"method":"$request_method","uri":"$uri","status":$status,'
|
||||
'"bytes":$body_bytes_sent,"request_time":$request_time,'
|
||||
'"upstream":"$upstream_addr","upstream_status":"$upstream_status",'
|
||||
'"upstream_time":"$upstream_response_time",'
|
||||
'"instance_id":"$upstream_http_x_instance_id",'
|
||||
'"limit_status":"$limit_req_status"}';
|
||||
|
||||
access_log /dev/stdout requests;
|
||||
error_log /dev/stderr warn;
|
||||
server_tokens off;
|
||||
|
||||
# DNS Docker возвращает адреса всех реплик сервиса api.
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
# Общий для worker-процессов лимит на IP, который видит nginx.
|
||||
# Клиентский X-Forwarded-For не используется в качестве ключа лимита.
|
||||
limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s;
|
||||
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_comp_level 3;
|
||||
gzip_types application/json;
|
||||
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=5s;
|
||||
|
||||
# Кэш свободных соединений на worker, а не лимит запросов к API.
|
||||
keepalive 16;
|
||||
keepalive_timeout 4s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
client_header_timeout 10s;
|
||||
client_body_timeout 10s;
|
||||
# Размер ограничивает API при чтении, включая chunked-передачу.
|
||||
client_max_body_size 0;
|
||||
client_body_buffer_size 64k;
|
||||
send_timeout 10s;
|
||||
keepalive_timeout 30s;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
|
||||
# Пассивное обнаружение отказов; Docker healthcheck сам не управляет LB.
|
||||
# non_idempotent не включён: уже отправленные POST не повторяем.
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
# Окно перехода на другой backend, не общий deadline ответа.
|
||||
proxy_next_upstream_timeout 2s;
|
||||
|
||||
# JSON и заголовки ошибок API передаются без подмены.
|
||||
proxy_intercept_errors off;
|
||||
error_page 429 = @rate_limited;
|
||||
error_page 502 503 504 = @unavailable;
|
||||
|
||||
# Проверка самого LB для healthcheck в compose; не проксируется в API.
|
||||
location = /lb-health {
|
||||
access_log off;
|
||||
default_type text/plain;
|
||||
return 200 "ok\n";
|
||||
}
|
||||
|
||||
# Проверки готовности/жизни не расходуют бюджет бизнес-запросов.
|
||||
location /health/ {
|
||||
proxy_pass http://api_backend;
|
||||
}
|
||||
|
||||
location / {
|
||||
limit_req zone=per_client burst=100 nodelay;
|
||||
limit_req_status 429;
|
||||
limit_req_log_level warn;
|
||||
proxy_pass http://api_backend;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
limit_req zone=per_client burst=100 nodelay;
|
||||
limit_req_status 429;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_send_timeout 15s;
|
||||
# Ошибка одного upload не должна скрыть работающую реплику для чатов.
|
||||
proxy_next_upstream off;
|
||||
proxy_pass http://api_backend;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/attachments/[^/]+/content$ {
|
||||
limit_req zone=per_client burst=100 nodelay;
|
||||
limit_req_status 429;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_pass http://api_backend;
|
||||
}
|
||||
|
||||
# Это ответы балансировщика: X-Instance-Id реплики у них отсутствует.
|
||||
location @rate_limited {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
add_header Retry-After 1 always;
|
||||
return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}';
|
||||
}
|
||||
|
||||
location @unavailable {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
add_header Retry-After 1 always;
|
||||
return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
dir /data
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
maxmemory 256mb
|
||||
maxmemory-policy noeviction
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
password_hash=$(tr -d '\r\n' </run/secrets/redis_password | sha256sum | cut -d ' ' -f 1)
|
||||
mkdir -p /run/redis
|
||||
umask 077
|
||||
printf 'user default reset on #%s ~* &* +@all\n' "$password_hash" >/run/redis/users.acl
|
||||
chown redis:redis /run/redis/users.acl
|
||||
chmod 600 /run/redis/users.acl
|
||||
exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl
|
||||
@@ -0,0 +1,68 @@
|
||||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Base
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name, disable_existing_loggers=False)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
explicit = config.get_main_option("sqlalchemy.url")
|
||||
if explicit:
|
||||
return explicit
|
||||
return Settings().get_database_url() # type: ignore
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=_database_url(),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def do_run_migrations(connection: Connection) -> None:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
section = config.get_section(config.config_ini_section, {})
|
||||
section["sqlalchemy.url"] = _database_url().replace("%", "%%")
|
||||
connectable = async_engine_from_config(
|
||||
section, prefix="sqlalchemy.", poolclass=pool.NullPool
|
||||
)
|
||||
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(run_async_migrations())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Добавляет результаты идемпотентных запросов."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "49d2c947a3f1"
|
||||
down_revision = "abd273076068"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Создаёт таблицу ключей без изменения существующих данных."""
|
||||
op.create_table(
|
||||
"idempotency_keys",
|
||||
sa.Column("user_id", sa.UUID(), nullable=False),
|
||||
sa.Column("method", sa.String(8), nullable=False),
|
||||
sa.Column("path", sa.String(255), nullable=False),
|
||||
sa.Column("key", sa.String(128), nullable=False),
|
||||
sa.Column("request_hash", sa.String(64), nullable=False),
|
||||
sa.Column("message_id", sa.UUID(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["message_id"], ["messages.id"],
|
||||
name=op.f("fk_idempotency_keys_message_id_messages"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
|
||||
),
|
||||
)
|
||||
op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Удаляет таблицу результатов запросов."""
|
||||
op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys")
|
||||
op.drop_table("idempotency_keys")
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Attachment, связи сообщений и устойчивый журнал загрузок."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "8f31b94a2d10"
|
||||
down_revision = "49d2c947a3f1"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"attachments",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("chat_id", sa.UUID(), nullable=False),
|
||||
sa.Column("owner_id", sa.UUID(), nullable=False),
|
||||
sa.Column("object_key", sa.String(128), nullable=False),
|
||||
sa.Column("filename", sa.String(255), nullable=False),
|
||||
sa.Column("content_type", sa.String(32), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(16), server_default="ready", nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")),
|
||||
sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")),
|
||||
sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")),
|
||||
sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
|
||||
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")),
|
||||
sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
|
||||
sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
|
||||
)
|
||||
op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"])
|
||||
op.create_table(
|
||||
"message_attachments",
|
||||
sa.Column("message_id", sa.UUID(), nullable=False),
|
||||
sa.Column("attachment_id", sa.UUID(), nullable=False),
|
||||
sa.Column("position", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")),
|
||||
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")),
|
||||
sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")),
|
||||
)
|
||||
op.create_table(
|
||||
"upload_intents",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("object_key", sa.String(128), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
|
||||
sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
|
||||
)
|
||||
op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_upload_intents_created_at", table_name="upload_intents")
|
||||
op.drop_table("upload_intents")
|
||||
op.drop_table("message_attachments")
|
||||
op.drop_index("ix_attachments_chat_id", table_name="attachments")
|
||||
op.drop_table("attachments")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""add counters
|
||||
|
||||
Revision ID: abd273076068
|
||||
Revises: db06fd3126ff
|
||||
Create Date: 2026-10-26 22:01:01.910396
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = 'abd273076068'
|
||||
down_revision: Union[str, None] = 'db06fd3126ff'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('counters',
|
||||
sa.Column('name', sa.String(length=64), nullable=False),
|
||||
sa.Column('value', sa.BigInteger(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('name', name=op.f('pk_counters'))
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('counters')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,107 @@
|
||||
"""create initial schema
|
||||
|
||||
Revision ID: db06fd3126ff
|
||||
Revises:
|
||||
Create Date: 2026-10-26 17:09:16.675339
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "db06fd3126ff"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"chats",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("title", sa.String(length=100), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")),
|
||||
)
|
||||
op.create_table(
|
||||
"users",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("username", sa.String(length=32), nullable=False),
|
||||
sa.Column("display_name", sa.String(length=100), nullable=False),
|
||||
sa.Column("password_hash", sa.String(length=255), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
|
||||
sa.UniqueConstraint("username", name=op.f("uq_users_username")),
|
||||
)
|
||||
op.create_table(
|
||||
"chat_members",
|
||||
sa.Column("chat_id", sa.UUID(), nullable=False),
|
||||
sa.Column("user_id", sa.UUID(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id"],
|
||||
["chats.id"],
|
||||
name=op.f("fk_chat_members_chat_id_chats"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_chat_members_user_id_chat_id",
|
||||
"chat_members",
|
||||
["user_id", "chat_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"messages",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("chat_id", sa.UUID(), nullable=False),
|
||||
sa.Column("sender_id", sa.UUID(), nullable=False),
|
||||
sa.Column("text", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_messages_chat_id_created_at_id",
|
||||
"messages",
|
||||
["chat_id", "created_at", "id"],
|
||||
unique=False,
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages")
|
||||
op.drop_table("messages")
|
||||
op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members")
|
||||
op.drop_table("chat_members")
|
||||
op.drop_table("users")
|
||||
op.drop_table("chats")
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,6 @@
|
||||
[pytest]
|
||||
testpaths = student-tests
|
||||
python_files = test_*.py
|
||||
asyncio_mode = auto
|
||||
asyncio_default_fixture_loop_scope = session
|
||||
asyncio_default_test_loop_scope = session
|
||||
@@ -0,0 +1,52 @@
|
||||
alembic==1.20.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
attrs==26.1.0
|
||||
certifi==2026.7.22
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
fastapi==0.143.3
|
||||
h11==0.16.0
|
||||
httpcore2==2.13.1
|
||||
httpx2==2.13.1
|
||||
idna==3.20
|
||||
iniconfig==2.3.1
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.12.0
|
||||
Mako==1.4.3
|
||||
MarkupSafe==3.0.4
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
opentelemetry-api==1.45.1
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pluggy==1.7.0
|
||||
pydantic==2.14.1
|
||||
pydantic_core==2.50.1
|
||||
Pygments==2.21.0
|
||||
pytest==9.1.1
|
||||
PyYAML==6.0.3
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
six==1.17.0
|
||||
SQLAlchemy==2.1.4
|
||||
starlette==1.8.0
|
||||
truststore==0.10.5
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.54.0
|
||||
asyncpg==0.32.0
|
||||
argon2-cffi==25.1.0
|
||||
redis==8.1.0
|
||||
greenlet==3.5.6
|
||||
pydantic-settings==2.16.0
|
||||
pytest-asyncio==1.4.1
|
||||
httpx==0.28.1
|
||||
minio==7.2.20
|
||||
Pillow==13.0.0
|
||||
@@ -0,0 +1 @@
|
||||
"""Команды управления учебным стендом."""
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
|
||||
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from scripts.failover_all import exercise, failure_kind, history
|
||||
from scripts.rolling_update import update
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
PROBE = """import json, sys, urllib.request, urllib.error
|
||||
payload = json.load(sys.stdin)
|
||||
results = []
|
||||
for item in payload:
|
||||
data = json.dumps(item['body']).encode() if 'body' in item else None
|
||||
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=7)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
results.append({'status': response.status, 'body': json.loads(response.read())})
|
||||
print(json.dumps(results))
|
||||
"""
|
||||
|
||||
|
||||
def probe(project, runner, requests):
|
||||
"""Обращается к внутреннему API, передавая токен только через stdin."""
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", runner, "python", "-c", PROBE],
|
||||
input=json.dumps(requests),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
check(result.returncode == 0, "internal HTTP probe failed")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def audit(project):
|
||||
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
|
||||
allowed = {
|
||||
"postgres": {"db_password"},
|
||||
"redis": {"redis_password"},
|
||||
"migrate": {"db_password"},
|
||||
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"minio": {"minio_root_user", "minio_root_password"},
|
||||
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"nginx": set(),
|
||||
}
|
||||
ids = project.run("ps", "--all", "--quiet").splitlines()
|
||||
containers = json.loads(project.docker("inspect", *ids))
|
||||
for item in containers:
|
||||
service = item["Config"]["Labels"]["com.docker.compose.service"]
|
||||
check(service in allowed, "unexpected service")
|
||||
host = item["HostConfig"]
|
||||
check(
|
||||
host["Memory"] > 0 and host["NanoCpus"] > 0,
|
||||
service + ": missing CPU/RAM limit",
|
||||
)
|
||||
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
|
||||
check(
|
||||
not env_names
|
||||
& {
|
||||
"DB_PASSWORD",
|
||||
"REDIS_PASSWORD",
|
||||
"POSTGRES_PASSWORD",
|
||||
"REDISCLI_AUTH",
|
||||
"DATABASE_URL",
|
||||
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
|
||||
},
|
||||
service + ": password in environment",
|
||||
)
|
||||
secrets = {
|
||||
Path(mount["Destination"]).name
|
||||
for mount in item["Mounts"]
|
||||
if mount["Destination"].startswith("/run/secrets/")
|
||||
}
|
||||
check(secrets == allowed[service], service + ": incorrect secret scope")
|
||||
if service in {"api", "minio"}:
|
||||
check(not host["PortBindings"], service + " publishes ports")
|
||||
for mount in item["Mounts"]:
|
||||
if mount["Destination"].startswith("/run/secrets/"):
|
||||
check(not mount["RW"], "writable secret mount")
|
||||
check(
|
||||
"container_name" not in project.config["services"][service],
|
||||
"fixed container name",
|
||||
)
|
||||
check(
|
||||
len(containers) == len(allowed) - 1 + len(project.replicas()),
|
||||
"unexpected container count",
|
||||
)
|
||||
print(
|
||||
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
async def graceful(project, base_url):
|
||||
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
|
||||
replicas = project.replicas()
|
||||
target, runner = replicas[:2]
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
raw = json.dumps([actor["id"], "POST", path, key]).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
||||
database = project.config["services"]["postgres"]
|
||||
env = database["environment"]
|
||||
conn = await asyncpg.connect(
|
||||
host="127.0.0.1",
|
||||
port=int(database["ports"][0]["published"]),
|
||||
database=env["POSTGRES_DB"],
|
||||
user=env["POSTGRES_USER"],
|
||||
password=(ROOT / "secrets/db_password").read_text().strip(),
|
||||
timeout=3,
|
||||
)
|
||||
network = project.config['networks']['default']['name']
|
||||
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
|
||||
address = project.docker(
|
||||
"inspect",
|
||||
"--format",
|
||||
address_template,
|
||||
target.id,
|
||||
)
|
||||
check(bool(address), 'API has no address in the application network')
|
||||
port = project.config["services"]["api"]["environment"]["APP_PORT"]
|
||||
direct = f"http://{address}:{port}"
|
||||
started = time.monotonic()
|
||||
task = None
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{
|
||||
"url": direct + path,
|
||||
"method": "POST",
|
||||
"body": {"text": "graceful-" + key},
|
||||
"headers": {
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Idempotency-Key": key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
deadline = time.monotonic() + 3
|
||||
while not await conn.fetchval(
|
||||
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
|
||||
):
|
||||
check(time.monotonic() < deadline, "POST did not reach database lock")
|
||||
await asyncio.sleep(0.02)
|
||||
started = time.monotonic()
|
||||
await asyncio.to_thread(
|
||||
project.docker, "kill", "--signal", "SIGTERM", target.id
|
||||
)
|
||||
results = await asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{"url": direct + "/health/ready"},
|
||||
{"url": direct + "/health/live"},
|
||||
{"url": direct + "/api/v1/users/me"},
|
||||
],
|
||||
)
|
||||
check(
|
||||
[item["status"] for item in results] == [503, 200, 503],
|
||||
"incorrect drain readiness/liveness/new work",
|
||||
)
|
||||
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
|
||||
result = (await task)[0]
|
||||
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
|
||||
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
|
||||
state = json.loads(
|
||||
project.docker("inspect", "--format", "{{json .State}}", target.id)
|
||||
)
|
||||
check(
|
||||
not state["OOMKilled"] and state["ExitCode"] != 137,
|
||||
"forced shutdown instead of graceful completion",
|
||||
)
|
||||
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
|
||||
items = await asyncio.to_thread(history, client, actor, chat)
|
||||
check(
|
||||
result["body"]["id"] in {item["id"] for item in items},
|
||||
"acknowledged in-flight message lost",
|
||||
)
|
||||
print(
|
||||
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
if task:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
if any(item.id == target.id and item.running for item in project.replicas()):
|
||||
await asyncio.to_thread(project.docker, "stop", target.id)
|
||||
await asyncio.to_thread(project.docker, "start", target.id)
|
||||
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
|
||||
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
|
||||
|
||||
|
||||
class ObservedProject(ComposeProject):
|
||||
"""Подсчитывает миграции и проверяет готовность перед заменой."""
|
||||
|
||||
def __init__(self):
|
||||
self.migrations = 0
|
||||
self.replacements = 0
|
||||
self.replacement_started = []
|
||||
super().__init__()
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
|
||||
self.migrations += 1
|
||||
return super().run(*args, **kwargs)
|
||||
|
||||
def docker(self, *args, **kwargs):
|
||||
if args[0] == "stop":
|
||||
healthy = [
|
||||
item
|
||||
for item in self.replicas()
|
||||
if item.id != args[1] and item.running and item.health == "healthy"
|
||||
]
|
||||
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
|
||||
self.replacements += 1
|
||||
self.replacement_started.append(time.monotonic())
|
||||
return super().docker(*args, **kwargs)
|
||||
|
||||
|
||||
def rolling(base_url):
|
||||
"""Заменяет образ под постоянной записью с проверкой сохранности."""
|
||||
project = ObservedProject()
|
||||
before = project.replicas()
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
stop = Event()
|
||||
results, errors = [], []
|
||||
|
||||
def worker():
|
||||
local = Client(base_url, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "rolling-" + key, key=key)
|
||||
results.append(
|
||||
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
|
||||
)
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
errors.append(failure_kind(error))
|
||||
stop.wait(0.08)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(2)]
|
||||
try:
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
update(project, base_url, revision="verification-" + uuid4().hex[:12])
|
||||
finished = time.monotonic()
|
||||
time.sleep(1)
|
||||
finally:
|
||||
stop.set()
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
after = project.replicas()
|
||||
check(
|
||||
project.migrations == 1 and project.replacements == len(before),
|
||||
"migration/replacement count incorrect",
|
||||
)
|
||||
check(
|
||||
{item.image for item in before}.isdisjoint({item.image for item in after}),
|
||||
"new image was not deployed",
|
||||
)
|
||||
check(
|
||||
{item.id for item in before}.isdisjoint({item.id for item in after}),
|
||||
"old containers remain",
|
||||
)
|
||||
writes = [row for row in results if started <= row[0] <= finished]
|
||||
check(bool(writes), "no writes during update")
|
||||
check(
|
||||
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
|
||||
"no writes during container replacement",
|
||||
)
|
||||
items = history(client, actor, chat)
|
||||
check(
|
||||
{row[2] for row in results} <= {item["id"] for item in items},
|
||||
"rolling update lost confirmed messages",
|
||||
)
|
||||
check(
|
||||
len(items) == len({item["text"] for item in items}),
|
||||
"rolling update created duplicate messages",
|
||||
)
|
||||
latencies = sorted(row[3] for row in writes)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"replacements": project.replacements,
|
||||
"migration_runs": project.migrations,
|
||||
"successful_writes": len(writes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(finished - started, 3),
|
||||
"successful_rps": round(len(writes) / (finished - started), 3),
|
||||
"p95_seconds": round(
|
||||
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
|
||||
),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Выполняет проверки только на отдельном тестовом проекте."""
|
||||
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--phase", choices=("all", "update-and-failover"), default="all"
|
||||
)
|
||||
|
||||
args = arguments("Grade 4 integration checks", configure)
|
||||
project = ComposeProject()
|
||||
check(project.name.endswith("-test"), "use isolated test Compose project")
|
||||
with deployment_lock(project):
|
||||
audit(project)
|
||||
if args.phase == "all":
|
||||
asyncio.run(graceful(project, args.base_url))
|
||||
rolling(args.base_url)
|
||||
args.duration, args.concurrency = 10, 2
|
||||
exercise(project, args)
|
||||
audit(project)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Проверяет конкурентные повторы через балансировщик."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Отправляет один ключ параллельно через разные реплики."""
|
||||
args = arguments("Concurrent idempotency through load balancer")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
body = "parallel-" + key
|
||||
|
||||
def send(_):
|
||||
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send, range(24)))
|
||||
ids = {result["json"]["id"] for result in results}
|
||||
replicas = {result["headers"]["X-Instance-Id"] for result in results}
|
||||
check(len(ids) == 1, "parallel retries created duplicate messages")
|
||||
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
|
||||
history = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
check(
|
||||
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
|
||||
)
|
||||
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGE = "nginx:1.28.0-alpine"
|
||||
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
|
||||
|
||||
|
||||
def check(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
class Stand:
|
||||
def __init__(self, directory: str) -> None:
|
||||
self.directory = Path(directory)
|
||||
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
|
||||
self.base_url = ""
|
||||
backend = self.directory / "backend.conf"
|
||||
backend.write_text(
|
||||
"worker_processes 1;\n"
|
||||
"events { worker_connections 1024; }\n"
|
||||
"http {\n"
|
||||
" log_format backend 'BACKEND $request_method $uri';\n"
|
||||
" access_log /dev/stdout backend;\n"
|
||||
" server {\n"
|
||||
" listen 8080;\n"
|
||||
" default_type application/json;\n"
|
||||
" add_header X-Instance-Id $hostname always;\n"
|
||||
" add_header X-Request-Id $request_id always;\n"
|
||||
" add_header Retry-After 7 always;\n"
|
||||
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
|
||||
" location = /api/limited {\n"
|
||||
' return 429 \'{"error":{"code":"rate_limited",'
|
||||
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
|
||||
" }\n"
|
||||
" location = /api/post-failure {\n"
|
||||
' return 503 \'{"error":{"code":"unavailable",'
|
||||
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
|
||||
" }\n"
|
||||
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
)
|
||||
self.compose_file = self.directory / "compose.json"
|
||||
self.compose_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"services": {
|
||||
"api": {
|
||||
"image": IMAGE,
|
||||
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
|
||||
},
|
||||
"nginx": {
|
||||
"image": IMAGE,
|
||||
"volumes": [
|
||||
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
|
||||
],
|
||||
"ports": ["127.0.0.1::8080"],
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def compose(self, *args: str) -> str:
|
||||
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
"/dev/null",
|
||||
"-f",
|
||||
str(self.compose_file),
|
||||
"-p",
|
||||
self.project,
|
||||
*args,
|
||||
],
|
||||
cwd=self.directory,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=45,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stdout + result.stderr)
|
||||
return result.stdout + result.stderr
|
||||
|
||||
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
|
||||
request = Request(
|
||||
self.base_url + path,
|
||||
method=method,
|
||||
headers={"Accept-Encoding": encoding},
|
||||
data=b"{}" if method == "POST" else None,
|
||||
)
|
||||
try:
|
||||
response = urlopen(request, timeout=3)
|
||||
except HTTPError as error:
|
||||
response = error
|
||||
except (URLError, TimeoutError):
|
||||
return 0, {}, b""
|
||||
with response:
|
||||
return (
|
||||
response.status,
|
||||
{key.lower(): value for key, value in response.headers.items()},
|
||||
response.read(),
|
||||
)
|
||||
|
||||
def wait_instances(self, count: int) -> set[str]:
|
||||
seen: set[str] = set()
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
status, headers, _ = self.call("/health/live")
|
||||
if status == 200:
|
||||
seen.add(headers["x-instance-id"])
|
||||
if len(seen) == count:
|
||||
return seen
|
||||
time.sleep(0.05)
|
||||
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
|
||||
|
||||
|
||||
def check_error(response, status: int, code: str) -> None:
|
||||
actual_status, headers, body = response
|
||||
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
|
||||
check(
|
||||
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
|
||||
)
|
||||
error = json.loads(body)["error"]
|
||||
check(error["code"] == code, "Неверный код ошибки")
|
||||
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
|
||||
|
||||
|
||||
def run_checks(stand: Stand) -> None:
|
||||
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
|
||||
address = stand.compose("port", "nginx", "8080").strip()
|
||||
stand.base_url = "http://" + address
|
||||
stand.wait_instances(2)
|
||||
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
|
||||
stand.compose("exec", "-T", "nginx", "nginx", "-t")
|
||||
print("PASS: nginx -t и балансировка двух реплик")
|
||||
|
||||
status, headers, body = stand.call("/api/large", encoding="gzip")
|
||||
check(
|
||||
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
|
||||
)
|
||||
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
|
||||
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
|
||||
check(
|
||||
"accept-encoding" in headers.get("vary", "").lower(),
|
||||
"Нет Vary: Accept-Encoding",
|
||||
)
|
||||
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
|
||||
small = stand.call("/health/live", encoding="gzip")
|
||||
check(
|
||||
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
|
||||
)
|
||||
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
|
||||
|
||||
limited = stand.call("/api/limited")
|
||||
check_error(limited, 429, "rate_limited")
|
||||
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
|
||||
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
|
||||
print("PASS: JSON и заголовки ошибки API передаются без подмены")
|
||||
|
||||
failure = stand.call("/api/post-failure", method="POST")
|
||||
check_error(failure, 503, "unavailable")
|
||||
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
|
||||
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
|
||||
print("PASS: отправленный POST не повторяется на другой реплике")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=32) as pool:
|
||||
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
|
||||
rejected = [response for response in responses if response[0] == 429]
|
||||
check(bool(rejected), "Не удалось вызвать ограничение частоты")
|
||||
check(
|
||||
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
|
||||
)
|
||||
for response in rejected:
|
||||
check_error(response, 429, "rate_limited")
|
||||
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
|
||||
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
|
||||
check(all(response[0] == 200 for response in health), "Лимит затронул health")
|
||||
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
|
||||
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
|
||||
|
||||
stand.compose(
|
||||
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
|
||||
)
|
||||
instances = stand.wait_instances(3)
|
||||
print("PASS: третья реплика обнаружена через DNS без reload nginx")
|
||||
|
||||
target = sorted(instances)[0]
|
||||
containers = stand.compose("ps", "-q", "api").splitlines()
|
||||
matching = [container for container in containers if container.startswith(target)]
|
||||
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
|
||||
# Все id взяты только из уникального временного Compose-проекта.
|
||||
started = time.monotonic()
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
|
||||
)
|
||||
recovery = None
|
||||
for _ in range(20):
|
||||
status, headers, _ = stand.call("/health/live")
|
||||
if status == 200:
|
||||
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
|
||||
if recovery is None:
|
||||
recovery = time.monotonic() - started
|
||||
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
|
||||
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
|
||||
|
||||
stand.compose("stop", "--timeout", "1", "api")
|
||||
unavailable = stand.call("/health/ready")
|
||||
check_error(unavailable, 503, "unavailable")
|
||||
check(
|
||||
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
|
||||
)
|
||||
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
|
||||
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
|
||||
print("PASS: недоступность всех backend возвращает JSON 503")
|
||||
|
||||
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
|
||||
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
|
||||
check(bool(entries), "Нет JSON access log")
|
||||
check(
|
||||
all("request_id" in entry and "upstream_time" in entry for entry in entries),
|
||||
"Неполный лог",
|
||||
)
|
||||
check(
|
||||
any(entry["limit_status"] == "REJECTED" for entry in entries),
|
||||
"Лимит не отражён в логе",
|
||||
)
|
||||
print("PASS: JSON access log содержит время backend и результат лимита")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
|
||||
stand = Stand(directory)
|
||||
try:
|
||||
run_checks(stand)
|
||||
except Exception:
|
||||
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
|
||||
raise
|
||||
finally:
|
||||
stand.compose("down", "--remove-orphans", "--timeout", "1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
|
||||
|
||||
import json
|
||||
import base64
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
PROBE = r'''
|
||||
import asyncio, json, sys
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError, URLError
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.storage.s3 import create_s3_storage
|
||||
from app.models.attachment import Attachment
|
||||
from minio.error import S3Error
|
||||
async def main():
|
||||
settings = Settings()
|
||||
engine = create_db_engine(settings)
|
||||
storage = create_s3_storage(settings.s3)
|
||||
try:
|
||||
async with create_session_factory(engine)() as session:
|
||||
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
|
||||
key = record.object_key
|
||||
results = {}
|
||||
for method in ('GET', 'PUT'):
|
||||
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
|
||||
method=method, data=b'x' if method == 'PUT' else None)
|
||||
try:
|
||||
with urlopen(request, timeout=3) as response:
|
||||
results[method] = response.status
|
||||
except HTTPError as error:
|
||||
results[method] = error.code
|
||||
error.close()
|
||||
try:
|
||||
storage._client.get_bucket_policy(settings.s3.bucket)
|
||||
results['admin_denied'] = False
|
||||
except S3Error as error:
|
||||
results['admin_denied'] = error.code == 'AccessDenied'
|
||||
try:
|
||||
buckets = storage._client.list_buckets()
|
||||
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
|
||||
except S3Error as error:
|
||||
results['other_buckets_denied'] = error.code == 'AccessDenied'
|
||||
try:
|
||||
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
|
||||
results['console_unreachable'] = False
|
||||
except HTTPError as error:
|
||||
results['console_unreachable'] = False
|
||||
error.close()
|
||||
except URLError:
|
||||
results['console_unreachable'] = True
|
||||
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
|
||||
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
|
||||
for name in ('minio_root_user', 'minio_root_password'))
|
||||
print(json.dumps(results))
|
||||
finally:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
|
||||
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
|
||||
args = arguments('MinIO private bucket and failure checks', configure)
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
|
||||
for _ in range(2):
|
||||
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
|
||||
replicas = project.replicas()
|
||||
check(bool(replicas), 'API not running')
|
||||
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
|
||||
input=json.dumps({'attachment_id': attachment['id']}),
|
||||
text=True, capture_output=True, timeout=20)
|
||||
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
|
||||
statuses = json.loads(result.stdout)
|
||||
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
|
||||
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
|
||||
check(statuses['console_unreachable'], 'console reachable from API network')
|
||||
check(statuses['temp_originals_absent'], 'API retained temporary originals')
|
||||
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
|
||||
env = project.config['services']['minio']['environment']
|
||||
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
|
||||
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content'
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
|
||||
if args.with_faults:
|
||||
project.run('stop', 'cleanup')
|
||||
try:
|
||||
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
|
||||
result = subprocess.run(
|
||||
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
|
||||
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
|
||||
'image': base64.b64encode(original).decode()}),
|
||||
text=True, capture_output=True, timeout=60,
|
||||
)
|
||||
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
|
||||
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
|
||||
finally:
|
||||
project.run('start', 'cleanup')
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
|
||||
if args.with_outage:
|
||||
try:
|
||||
project.run('stop', 'minio')
|
||||
client.call('GET', path, expected=503, actor=alice)
|
||||
finally:
|
||||
project.run('start', 'minio')
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
break
|
||||
except Exception:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
|
||||
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
|
||||
+ (', MinIO outage/recovery' if args.with_outage else ''))
|
||||
if args.with_faults:
|
||||
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from importlib.metadata import version
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT = 'mtusi-messenger-lab4-test'
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
ENVIRONMENT = {
|
||||
**os.environ,
|
||||
'COMPOSE_PROJECT_NAME': PROJECT,
|
||||
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
|
||||
'LB_HOST_PORT': '18080',
|
||||
}
|
||||
COMPOSE = ['docker', 'compose', '--env-file', '.env']
|
||||
NONSECRET_SETTINGS = {
|
||||
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
|
||||
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
|
||||
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
|
||||
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
|
||||
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
|
||||
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
|
||||
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
|
||||
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
|
||||
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
|
||||
'S3_CLEANUP_INTERVAL_SECONDS',
|
||||
}
|
||||
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
|
||||
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
|
||||
|
||||
|
||||
def timestamp():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def invoke(command):
|
||||
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
|
||||
def read_command(command):
|
||||
result = invoke(command)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def source_manifest():
|
||||
names = {
|
||||
'.dockerignore', '.env.example', 'AGENTS.md', 'CODE_STYLE.md',
|
||||
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
|
||||
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
|
||||
}
|
||||
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
|
||||
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
|
||||
if path.is_file() and '__pycache__' not in path.parts
|
||||
and path.suffix != '.md' and not path.name.startswith('.'))
|
||||
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
|
||||
|
||||
|
||||
def sanitize(text, secrets, staging):
|
||||
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
|
||||
for secret in secrets:
|
||||
clean = clean.replace(secret, '<redacted>')
|
||||
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
|
||||
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
|
||||
return clean
|
||||
|
||||
|
||||
def container_snapshot(images):
|
||||
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
rows = []
|
||||
if ids:
|
||||
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
|
||||
rows = [json.loads(line) for line in output.splitlines()]
|
||||
for row in rows:
|
||||
image = row['image_id']
|
||||
if image not in images:
|
||||
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
|
||||
return rows
|
||||
|
||||
|
||||
def remaining_resources():
|
||||
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
|
||||
f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
for resource in ('container', 'network', 'volume')}
|
||||
|
||||
|
||||
def extract_metrics(output, upload_file):
|
||||
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
|
||||
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
|
||||
if pytest is None:
|
||||
raise RuntimeError('В выводе нет успешного итогового результата pytest')
|
||||
return {
|
||||
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
|
||||
'uploads': json.loads(upload_file.read_text()),
|
||||
'public_failover': [row for row in rows if 'observed_instances_before' in row],
|
||||
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
|
||||
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if any(remaining_resources().values()):
|
||||
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
|
||||
started = timestamp()
|
||||
manifest = source_manifest()
|
||||
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
|
||||
secrets = [value for value in secrets if value]
|
||||
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
|
||||
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
|
||||
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
|
||||
services = {name: {
|
||||
'image': service.get('image'),
|
||||
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
|
||||
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
|
||||
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
|
||||
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
|
||||
'tmpfs': service.get('tmpfs', []),
|
||||
} for name, service in config['services'].items()}
|
||||
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
|
||||
staging = Path(folder)
|
||||
upload_file = staging / 'uploads.json'
|
||||
|
||||
def execute(command):
|
||||
begin = timestamp()
|
||||
result = invoke(command)
|
||||
output = sanitize(result.stdout, secrets, staging)
|
||||
display = sanitize(shlex.join(command), secrets, staging)
|
||||
commands.append({'command': display, 'started_at_utc': begin,
|
||||
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
|
||||
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
|
||||
if result.returncode:
|
||||
print(output[-5000:], flush=True)
|
||||
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
|
||||
return f'$ {display}\n{output}\n'
|
||||
|
||||
cleanup_output = ''
|
||||
try:
|
||||
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['started'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
|
||||
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['before_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_failover'] = container_snapshot(images)
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:18080', '--with-outage', '--with-faults']))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:18080', '--concurrency', '10', '--output', str(upload_file)]))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:18080']))
|
||||
snapshots['after_update_and_failures'] = container_snapshot(images)
|
||||
nginx_output = execute(['make', 'test-nginx'])
|
||||
finally:
|
||||
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
|
||||
remaining = remaining_resources()
|
||||
if any(remaining.values()):
|
||||
raise RuntimeError('После очистки остались ресурсы тестового проекта')
|
||||
if source_manifest() != manifest:
|
||||
raise RuntimeError('Во время прогона изменились проверяемые исходники')
|
||||
output = ''.join(combined)
|
||||
metrics = extract_metrics(output, upload_file)
|
||||
run = {
|
||||
'started_at_utc': started, 'finished_at_utc': timestamp(),
|
||||
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
|
||||
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
|
||||
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
|
||||
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
|
||||
'project': PROJECT, 'runtime': {'python': platform.python_version(),
|
||||
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
|
||||
'compose': read_command(['docker', 'compose', 'version'])},
|
||||
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
|
||||
'services': services, 'container_snapshots': snapshots, 'images': images,
|
||||
'commands': commands, 'remaining_test_resources': remaining,
|
||||
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
|
||||
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
|
||||
}
|
||||
artifacts = {
|
||||
'test-output.txt': output, 'nginx-output.txt': nginx_output,
|
||||
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
|
||||
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
|
||||
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
|
||||
}
|
||||
for name, content in artifacts.items():
|
||||
if any(secret in content for secret in secrets):
|
||||
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
for name, content in artifacts.items():
|
||||
(EVIDENCE / name).write_text(content)
|
||||
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user