ЛР 4 - Григорян - БПИ2303 - а 4 #57

Closed
gytall wants to merge 4 commits from gytall/mtusi2026lab4-fork:solution/BPI2303-Grigoryan into main
133 changed files with 11044 additions and 40 deletions
+57 -10
View File
@@ -1,20 +1,67 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Скопируйте в .env для локального окружения; .env не коммитится.
# Имена переменных соответствуют полям app/core/config.py.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
LOG_LEVEL=INFO
MAX_MESSAGES=100000
# Лимит одновременных запросов НА ОДНУ реплику
MAX_IN_FLIGHT=16
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
SESSION_STORE=redis
REDIS_HOST=redis
REDIS_PORT=6379
REDIS_DB=0
REDIS_MAX_CONNECTIONS=20
REDIS_POOL_TIMEOUT_SECONDS=1
REDIS_CONNECT_TIMEOUT_SECONDS=1
REDIS_SOCKET_TIMEOUT_SECONDS=0.5
DB_HOST=postgres
DB_PORT=5432
DB_NAME=messenger
DB_USER=messenger
# Пул на одну реплику: суммарно API_REPLICAS * (POOL_SIZE + MAX_OVERFLOW)
# соединений должно помещаться в max_connections PostgreSQL.
DB_POOL_SIZE=5
DB_MAX_OVERFLOW=10
DB_POOL_TIMEOUT_SECONDS=3
DB_CONNECT_TIMEOUT_SECONDS=3
DB_STATEMENT_TIMEOUT_MS=5000
# --- Балансировка и идемпотентность ---
API_REPLICAS=2
API_IMAGE=mtusi-messenger:lab4
API_BUILD_REVISION=development
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
SHUTDOWN_DRAIN_SECONDS=1
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5
READINESS_TIMEOUT_SECONDS=2
# Порт единственной публичной точки входа (nginx)
LB_HOST_PORT=8080
# Используются только тестовым стендом (compose.test.yaml)
DB_HOST_PORT=15432
REDIS_HOST_PORT=16379
# --- Lab4: MinIO, ограниченный upload и очистка журнала ---
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
S3_CONNECT_TIMEOUT_SECONDS=2
S3_READ_TIMEOUT_SECONDS=10
S3_UPLOAD_TIMEOUT_SECONDS=30
S3_DOWNLOAD_TIMEOUT_SECONDS=60
S3_IMAGE_TIMEOUT_SECONDS=10
S3_MAX_UPLOAD_BYTES=10485760
S3_MAX_IMAGE_PIXELS=40000000
S3_CHUNK_BYTES=65536
S3_MAX_CONCURRENT_UPLOADS=2
S3_CLEANUP_GRACE_SECONDS=86400
S3_CLEANUP_INTERVAL_SECONDS=300
+4
View File
@@ -28,5 +28,9 @@ coverage/
.DS_Store
.idea/
.vscode/
venv/
.pytest_cache/
.ruff_cache/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
AGENTS.md
+31
View File
@@ -0,0 +1,31 @@
FROM python:3.12.15-slim-bookworm
LABEL io.mtusi.messenger.secrets="file-v1"
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
RUN useradd --system --uid 10001 --no-create-home appuser
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY alembic.ini ./
COPY migrations ./migrations
COPY app ./app
ARG API_BUILD_REVISION
LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
CMD ["python", "-m", "app"]
+202 -4
View File
@@ -1,13 +1,86 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
-include .env
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard venv/bin/python),venv/bin/python,python3))
PYTEST_ARGS ?=
BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080)
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
COMPOSE = docker compose --env-file .env
# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis
# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh.
TEST_PROJECT ?= mtusi-messenger-lab4-test
TEST_LB_PORT ?= 18080
TEST_DB_PORT ?= $(DB_HOST_PORT)
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
TEST_BASE_URL = http://localhost:$(TEST_LB_PORT)
TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT)
TEST_COMPOSE = $(TEST_ENV) $(COMPOSE)
TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file <secrets/db_password),$(DB_PASSWORD))
TEST_DATABASE_URL ?= postgresql+asyncpg://$(DB_USER):$(TEST_DB_PASSWORD)@localhost:$(TEST_DB_PORT)/$(DB_NAME)_test
TEST_REDIS_HOST ?= localhost
TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/redis_password),$(REDIS_PASSWORD))
TEST_REDIS_DB ?= 15
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
define wait_ready
@for i in $$(seq 1 90); do \
curl -sf "$(1)/health/ready" >/dev/null && exit 0; sleep 1; \
done; echo "API $(1) не готов за 90 секунд" >&2; exit 1
endef
.PHONY: help check validate test load test-persistence test-failover \
init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \
up down restart restore ps logs migrate migration instances \
test-stand-up test-stand-down test-stand-destroy test-stand-ps \
test-stand-smoke test-stand-persistence test-stand-failover \
test-own test-docker test-limits test-concurrency test-migrations test-all \
explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \
rolling-update failover-all test-lab3-grade4
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply"
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
@echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps"
@echo " test-stand-smoke | test-stand-persistence | test-stand-failover"
@echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all"
@echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)"
@echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)"
@echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)"
@echo "База lab3: rolling-update | failover-all"
@echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)"
@echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)"
# --- Проверки API и сценариев отказа (tests/) ---
init:
$(PYTHON) scripts/init_local.py
test-grade4:
$(MAKE) check validate
$(MAKE) test-stand-up
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
$(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults
$(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
test-s3:
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
load-uploads:
$(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10
cleanup-dry-run:
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once
cleanup-apply:
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once --apply
check:
$(PYTHON) scripts/check_contract.py
@@ -15,6 +88,22 @@ check:
validate:
$(PYTHON) scripts/validate_openapi.py
test-nginx:
$(PYTHON) scripts/check_nginx.py
test-lab3-grade3:
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
$(MAKE) check validate; \
$(MAKE) test-stand-up; \
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
test-lab3-grade4:
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
$(MAKE) check validate; \
$(MAKE) test-stand-up; \
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +117,112 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- Основной стенд ---
rolling-update:
$(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS)
failover-all:
$(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS)
up:
$(COMPOSE) up -d --build
$(call wait_ready,$(BASE_URL))
down:
$(COMPOSE) down
restart:
scripts/restart.sh
$(call wait_ready,$(BASE_URL))
# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры).
restore:
$(COMPOSE) up -d --wait api
ps:
$(COMPOSE) ps
logs:
$(COMPOSE) logs -f api nginx
migrate:
$(COMPOSE) run --rm --build migrate
# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id).
instances:
@for i in $$(seq 1 10); do \
curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \
done
# Генерация миграции против БД тестового стенда (основной стенд БД не публикует).
migration:
DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)"
# --- Тестовый стенд ---
test-stand-up:
$(TEST_COMPOSE) up -d --build
$(call wait_ready,$(TEST_BASE_URL))
$(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
test-stand-down:
$(TEST_COMPOSE) down
test-stand-destroy:
$(TEST_COMPOSE) down -v
test-stand-ps:
$(TEST_COMPOSE) ps
test-stand-smoke:
$(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)"
test-stand-idempotency:
$(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)"
test-stand-persistence:
$(TEST_COMPOSE) ps
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)"
$(TEST_COMPOSE) ps
test-stand-failover:
$(TEST_COMPOSE) ps
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)"
$(TEST_COMPOSE) ps -a api
$(TEST_COMPOSE) up -d --wait api
# --- Собственные тесты (student-tests/) ---
test-own:
$(PYTHON) -m pytest -q $(PYTEST_ARGS)
test-docker:
$(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS)
test-limits:
$(PYTHON) -m pytest student-tests/test_limits.py -q
test-concurrency:
$(PYTHON) -m pytest student-tests/test_concurrency.py -q
test-migrations:
$(PYTHON) -m pytest student-tests/test_migrations.py -q
test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence
# --- Демо (на тестовом стенде) ---
explain:
$(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py
expiry-demo: test-stand-up
$(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api
$(call wait_ready,$(TEST_BASE_URL))
BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh
$(TEST_COMPOSE) up -d api
$(call wait_ready,$(TEST_BASE_URL))
failure-demo: test-stand-up
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
+5
View File
@@ -1,5 +1,10 @@
# Лабораторная 4. S3-хранилище и вложения
В этом fork реализованы вложения на MinIO поверх базы lab3. Инструкция
запуска — [RUNBOOK.md](RUNBOOK.md), соответствие требованиям оценки 4 —
[LAB4_PLAN.md](LAB4_PLAN.md). Полный прогон на оценку 4 выполнен;
результаты — в [REPORT.md](REPORT.md) и [evidence](evidence/README.md). Требования ниже сохранены как задание.
Разделить метаданные и бинарное содержимое, добавить изображения в сообщения и сохранить доступ к ним при переключении API-реплики.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
+58 -23
View File
@@ -1,37 +1,72 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Автор: **Григорян Илья Мурадович, БПИ2303**. Целевая оценка: **4**, трек 5 не заявлен.
## Запуск из чистого clone
Проверка: **04.11.2026**, отдельный чистый стенд `mtusi-messenger-lab4-test`.
Базовый commit: `ec540a1a725862a8619fcb758daec4ad197f0dd5`; проверено незакоммиченное рабочее дерево.
Точный снимок — [SHA-256 исходников](evidence/source-sha256.txt), среда и команды —
[run.json](evidence/run.json). Исходники во время прогона не изменялись.
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
## Запуск
## Архитектура и решения
Нужны Python 3.12, Docker и Compose. Из корня проекта:
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
make up
```
## Доказательства критериев
Вход — `http://localhost:8080`. `make init` сохраняет существующие настройки и
секреты. MinIO/mc собираются из закреплённых исходников; первая сборка требует
интернета. Миграции и bootstrap выполняются до API. `make down` сохраняет данные.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Повтор полного прогона с сохранением evidence и удалением тестовых volumes:
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
```sh
.venv/bin/python -m scripts.collect_evidence
```
## Эксперименты и отказы
## Реализация
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Nginx → две FastAPI-реплики → PostgreSQL, Redis и приватный MinIO.
Метаданные и связи вложений находятся в PostgreSQL, оригиналы — в MinIO.
Сохранены общие сессии и идемпотентность lab3; сообщения поддерживают `attachment_ids`.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
PNG/JPEG принимаются потоком во временный tmpfs, проверяются отдельным ограниченным
процессом и сохраняются без перекодирования. `201 ready` возвращается после PUT
и COMMIT. До PUT фиксируется журнал загрузки; при сбое выполняется компенсация,
при её отказе — повторная очистка после защитного интервала 24 часа. Готовые
вложения не удаляются автоматически.
## Ограничения и следующие шаги
## Проверки и результаты
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
**188 pytest прошли за 73,96 с**;
контракт, OpenAPI и все сквозные сценарии завершились успешно.
Исходный вывод — [test-output.txt](evidence/test-output.txt), методика —
[evidence/README.md](evidence/README.md), числа — [metrics.json](evidence/metrics.json).
| Требование | Подтверждение |
| --- | --- |
| 3: сохранён минимум lab3, общий стенд | Две API, общие PostgreSQL/Redis/MinIO; 24 конкурентных повтора через две реплики создали одно сообщение. |
| 3: вложения, SHA-256, авторизованный доступ | Smoke прошёл: 201 ready, метаданные, сообщение с файлом, побайтный оригинал; постороннему 404. |
| 3: PNG/JPEG и ограничения | Проверены MIME, имя, повреждения, превышения и приём ровно 10 MiB / 40 млн пикселей. |
| 3: сохранность и отказ загрузившей API | Persistence и failover прошли с прежней сессией и тем же оригиналом; перезапуск всех работающих сервисов подтверждён PID/StartedAt. |
| 4: негативные сценарии | Слишком большой файл, повреждение, подмена MIME, неизвестный/чужой ID и файл другого чата дают предусмотренные 4xx. |
| 4: согласованность S3/БД | Проверены отмена, неопределённый COMMIT, реальный PUT с отказом метаданных, неудачная компенсация, grace period, dry-run и повтор очистки. |
| 4: ограниченная память | 3 серии по 10 запросов: 12 × 201, 18 × 429; пики выборок API — 154,9 / 156,1 MiB при лимите 1536 MiB. |
| 4: приватность и bootstrap | Повтор bootstrap ×2 сохраняет данные; прямые анонимные GET/PUT запрещены, credentials API ограничены, консоль недоступна из сети API. |
Сохранённые проверки lab3 также прошли: SIGTERM, ресурсы/секреты, rolling update
обеих API и последовательный отказ каждой. Восстановление записи — **1,865 / 2,042 с**.
Nginx отдельно проверен: [nginx-output.txt](evidence/nginx-output.txt).
Тестовые контейнеры, сети и volumes удалены — [cleanup.txt](evidence/cleanup.txt).
## Границы результата
Один хост и один короткий синтетический прогон. PostgreSQL, Redis, MinIO и Nginx
не реплицированы. 429 ограничителя и переходные 503 при отказах/обновлении учтены;
подтверждённые сообщения и файлы сохранились. Память cgroup измерялась через 20 мс,
включая дочерние процессы и tmpfs; накопительный kernel peak — до 156,1 MiB.
Это не измерение предельного RPS. Использована помощь Codex; версии и первичные.
+38
View File
@@ -0,0 +1,38 @@
[alembic]
script_location = migrations
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
View File
+4
View File
@@ -0,0 +1,4 @@
from app.main import main
if __name__ == "__main__":
main()
View File
+60
View File
@@ -0,0 +1,60 @@
"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис."""
import argparse
import asyncio
from dataclasses import asdict
import json
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.s3 import S3Settings
from app.domain.attachments import reconcile_attachments
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
from app.storage.s3 import create_s3_storage
async def cleanup_once(factory, storage, apply: bool) -> bool:
completed = True
try:
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
print(json.dumps(asdict(result)), flush=True)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
return completed
async def run(once: bool, apply: bool) -> None:
settings = Settings() # type: ignore
s3_settings = S3Settings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(s3_settings)
try:
while True:
completed = await cleanup_once(factory, storage, apply)
if once:
if not completed:
raise SystemExit(1)
break
await asyncio.sleep(s3_settings.cleanup_interval_seconds)
finally:
await storage.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
parser.add_argument("--once", action="store_true")
parser.add_argument(
"--apply",
action="store_true",
help="удалять безопасные кандидаты; по умолчанию dry-run",
)
args = parser.parse_args()
asyncio.run(run(args.once, args.apply))
if __name__ == "__main__":
main()
+129
View File
@@ -0,0 +1,129 @@
from enum import StrEnum
from pathlib import Path
from typing import Literal, Self
from pydantic import Field, SecretStr, model_validator
from pydantic_settings import (
BaseSettings,
PydanticBaseSettingsSource,
SettingsConfigDict,
)
from sqlalchemy import URL
from app.core.s3 import S3Settings
ASYNC_DRIVER = "postgresql+asyncpg"
SECRETS_DIR = next(
(
path
for path in (
Path("/run/secrets"),
Path(__file__).resolve().parents[2] / "secrets",
)
if (path / "db_password").is_file()
),
None,
)
class LogLevel(StrEnum):
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
CRITICAL = "CRITICAL"
class Settings(BaseSettings):
s3_enabled: bool = True
s3: S3Settings = Field(default_factory=S3Settings)
app_host: str
app_port: int = Field(ge=1, le=65535)
app_env: str
log_level: LogLevel = LogLevel.INFO
max_messages: int = Field(ge=1)
max_in_flight: int = Field(ge=1)
session_ttl_seconds: int = Field(default=3600, ge=1)
session_store: Literal["redis"] = "redis"
redis_host: str = "localhost"
redis_port: int = Field(default=6379, ge=1, le=65535)
redis_db: int = Field(default=0, ge=0)
redis_password: SecretStr | None = None
redis_max_connections: int = Field(default=20, ge=1)
redis_pool_timeout_seconds: float = Field(default=1.0, gt=0)
redis_connect_timeout_seconds: float = Field(default=1.0, gt=0)
redis_socket_timeout_seconds: float = Field(default=0.5, gt=0)
database_url: SecretStr | None = None
db_host: str | None = None
db_port: int = Field(default=5432, ge=1, le=65535)
db_name: str | None = None
db_user: str | None = None
db_password: SecretStr | None = None
db_pool_size: int = Field(default=5, ge=1)
db_max_overflow: int = Field(default=5, ge=0)
db_pool_timeout_seconds: float = Field(default=3.0, gt=0)
db_connect_timeout_seconds: int = Field(default=3, ge=1)
db_statement_timeout_ms: int = Field(default=5000, ge=1)
instance_id: str | None = None
idempotency_ttl_seconds: int = Field(default=86400, ge=1)
shutdown_drain_seconds: float = Field(default=1, ge=0)
graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1)
readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5)
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=False,
extra="ignore",
secrets_dir=SECRETS_DIR,
)
@classmethod
def settings_customise_sources(
cls,
settings_cls: type[BaseSettings],
init_settings: PydanticBaseSettingsSource,
env_settings: PydanticBaseSettingsSource,
dotenv_settings: PydanticBaseSettingsSource,
file_secret_settings: PydanticBaseSettingsSource,
) -> tuple[PydanticBaseSettingsSource, ...]:
"""Явные параметры и файлы имеют приоритет над окружением."""
return init_settings, file_secret_settings, env_settings, dotenv_settings
@model_validator(mode="after")
def _require_database_source(self) -> Self:
if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
raise ValueError(
"Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
)
if self.database_url is not None:
if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
return self
if not (self.db_host and self.db_name and self.db_user and self.db_password):
raise ValueError(
"Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password"
)
return self
def get_database_url(self) -> str:
if self.database_url is not None:
return self.database_url.get_secret_value()
assert self.db_password is not None
url = URL.create(
ASYNC_DRIVER,
username=self.db_user,
password=self.db_password.get_secret_value(),
host=self.db_host,
port=self.db_port,
database=self.db_name,
)
return url.render_as_string(hide_password=False)
+28
View File
@@ -0,0 +1,28 @@
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from app.core.config import Settings
def create_db_engine(settings: Settings) -> AsyncEngine:
return create_async_engine(
settings.get_database_url(),
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout_seconds,
pool_pre_ping=True,
connect_args={
"timeout": settings.db_connect_timeout_seconds,
"server_settings": {
"statement_timeout": str(settings.db_statement_timeout_ms)
},
},
)
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
+24
View File
@@ -0,0 +1,24 @@
"""Одноразовый Linux-процесс декодирования с пределами памяти и CPU"""
from pathlib import Path
import resource
import sys
from app.domain.image_validation import decode_image
from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code
def main() -> int:
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
path, content_type, maximum = sys.argv[1:]
status = 0
try:
decode_image(Path(path), content_type, int(maximum))
except IMAGE_WORKER_ERRORS as error:
status = image_worker_error_code(error)
return status
if __name__ == "__main__":
raise SystemExit(main())
+40
View File
@@ -0,0 +1,40 @@
"""Идентификатор API-реплики и заголовок X-Instance-Id."""
import os
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
INSTANCE_ID_HEADER = "X-Instance-Id"
def resolve_instance_id(configured: str | None) -> str:
"""Выбирает INSTANCE_ID, HOSTNAME или UUID при старте приложения."""
if configured and configured.strip():
return configured.strip()
hostname = os.getenv("HOSTNAME", "").strip()
if hostname:
return hostname
return str(uuid4())
class InstanceIdMiddleware(BaseHTTPMiddleware):
"""Добавляет сохранённый идентификатор реплики в ответ."""
def __init__(self, app, instance_id: str) -> None:
super().__init__(app)
self.instance_id = instance_id
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
response = await call_next(request)
response.headers[INSTANCE_ID_HEADER] = self.instance_id
return response
+50
View File
@@ -0,0 +1,50 @@
"""Готовность реплики и мягкое завершение сервера."""
import asyncio
from types import FrameType
import uvicorn
class Lifecycle:
"""Хранит состояние завершения реплики."""
def __init__(self) -> None:
self._draining = False
@property
def draining(self) -> bool:
return self._draining
def start_draining(self) -> None:
"""Отключает приём новых бизнес-запросов."""
self._draining = True
class GracefulServer(uvicorn.Server):
"""Оставляет время для исключения завершающейся реплики."""
def __init__(
self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float
) -> None:
super().__init__(config)
self.lifecycle = lifecycle
self.drain_seconds = drain_seconds
self._drain_task: asyncio.Task[None] | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
"""Первый сигнал запускает drain, повторный ускоряет остановку."""
if self._drain_task is not None:
if not self._drain_task.done():
self._drain_task.cancel()
super().handle_exit(sig, frame)
return
self.lifecycle.start_draining()
self._drain_task = asyncio.get_running_loop().create_task(
self._finish_draining(sig, frame)
)
async def _finish_draining(self, sig: int, frame: FrameType | None) -> None:
"""Передаёт остановку Uvicorn после drain-периода."""
await asyncio.sleep(self.drain_seconds)
super().handle_exit(sig, frame)
+25
View File
@@ -0,0 +1,25 @@
from redis.asyncio import BlockingConnectionPool, Redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
)
pool = BlockingConnectionPool(
host=settings.redis_host,
port=settings.redis_port,
db=settings.redis_db,
password=password,
max_connections=settings.redis_max_connections,
timeout=settings.redis_pool_timeout_seconds,
socket_connect_timeout=settings.redis_connect_timeout_seconds,
socket_timeout=settings.redis_socket_timeout_seconds,
retry=Retry(NoBackoff(), retries=1),
decode_responses=True,
)
return Redis.from_pool(pool)
+42
View File
@@ -0,0 +1,42 @@
from collections.abc import AsyncIterator
from fastapi import Request
from app.errors.exceptions import rate_limited, unavailable
class RequestCapacity:
def __init__(self, max_in_flight: int) -> None:
self._max_in_flight = max_in_flight
self._in_flight = 0
@property
def in_flight(self) -> int:
return self._in_flight
def try_acquire(self) -> bool:
if self._in_flight >= self._max_in_flight:
return False
self._in_flight += 1
return True
def release(self) -> None:
self._in_flight -= 1
def create_request_capacity(max_in_flight: int) -> RequestCapacity:
return RequestCapacity(max_in_flight)
async def acquire_request_slot(request: Request) -> AsyncIterator[None]:
if request.app.state.lifecycle.draining:
raise unavailable("Реплика завершает работу")
capacity: RequestCapacity = request.app.state.request_capacity
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
+22
View File
@@ -0,0 +1,22 @@
from uuid import uuid4
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
REQUEST_ID_HEADER = "X-Request-Id"
class RequestIdMiddleware(BaseHTTPMiddleware):
async def dispatch(
self,
request: Request,
call_next: RequestResponseEndpoint,
) -> Response:
request_id = str(uuid4())
request.state.request_id = request_id
response = await call_next(request)
response.headers[REQUEST_ID_HEADER] = request_id
return response
+35
View File
@@ -0,0 +1,35 @@
"""Конфигурация MinIO; значения credentials читаются только из файлов."""
from pathlib import Path
from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
bucket: str = "messenger-attachments"
access_key_id_file: Path = Path("/run/secrets/s3_access_key_id")
secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key")
connect_timeout_seconds: float = Field(default=2, gt=0, le=5)
read_timeout_seconds: float = Field(default=10, gt=0, le=15)
upload_timeout_seconds: float = Field(default=30, gt=0, le=60)
download_timeout_seconds: float = Field(default=60, gt=0, le=120)
image_timeout_seconds: float = Field(default=10, gt=0, le=15)
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144)
max_concurrent_uploads: int = Field(default=2, ge=1, le=2)
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
return self
+63
View File
@@ -0,0 +1,63 @@
"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий."""
import asyncio
import hashlib
import os
import secrets
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
TOKEN_BYTES = 32
_hasher = PasswordHasher()
_executor = ThreadPoolExecutor(
max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)),
thread_name_prefix="argon2",
)
def _hash_password_sync(password: str) -> str:
"""Блокирующий расчёт Argon2id-хеша со случайной солью в формате PHC."""
return _hasher.hash(password)
def _verify_password_sync(password: str, password_hash: str) -> bool:
"""Блокирующая проверка пароля по PHC-строке; False при несовпадении или повреждённом хеше."""
try:
return _hasher.verify(password_hash, password)
except (VerificationError, InvalidHashError):
return False
_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES))
async def hash_password(password: str) -> str:
"""Хеширует пароль в пуле потоков, не блокируя event loop."""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(_executor, _hash_password_sync, password)
async def verify_password(password: str, password_hash: str) -> bool:
"""Проверяет пароль в пуле потоков, не блокируя event loop."""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
_executor, _verify_password_sync, password, password_hash
)
async def dummy_verify_password(password: str) -> None:
"""Проверка против фиктивного хеша: время ответа не выдаёт отсутствие пользователя."""
await verify_password(password, _DUMMY_HASH)
def generate_token() -> str:
"""Непрозрачный токен сессии: 32 случайных байта в base64url."""
return secrets.token_urlsafe(TOKEN_BYTES)
def hash_token(token: str) -> str:
"""SHA-256 токена в hex: детерминированный ключ для хранилища сессий."""
return hashlib.sha256(token.encode()).hexdigest()
+1
View File
@@ -0,0 +1 @@
"""Бизнес-логика: проверки доступа, транзакции и ошибки API."""
+315
View File
@@ -0,0 +1,315 @@
"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
from collections.abc import AsyncIterator, Iterator
from contextlib import contextmanager, suppress
from dataclasses import dataclass
import hashlib
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from anyio import CancelScope
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.request_capacity import RequestCapacity
from app.domain.chats import ensure_chat_member
from app.domain.images import validate_filename, validate_image
from app.errors.exceptions import (
conflict,
invalid_request,
not_found,
payload_too_large,
rate_limited,
unavailable,
unsupported_media_type,
)
from app.errors.handlers import (
RECOVERABLE_STORAGE_ERRORS,
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@dataclass(frozen=True)
class CleanupResult:
examined: int
candidates: int
deleted: int
@dataclass(frozen=True)
class OriginalFile:
path: Path
size_bytes: int
sha256: str
def require_storage(storage: MinioStorage | None) -> MinioStorage:
if storage is None:
raise unavailable("Объектное хранилище не настроено")
return storage
def validate_declared_size(value: str | None, maximum: int) -> None:
if value is not None:
if not value.isascii() or not value.isdecimal() or len(value) > 20:
raise invalid_request("Некорректный Content-Length")
if int(value) > maximum:
raise payload_too_large()
@contextmanager
def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
if not capacity.try_acquire():
raise rate_limited()
try:
yield
finally:
capacity.release()
async def receive_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
chat_id: UUID,
*,
capacity: RequestCapacity,
filename: str,
content_type: str,
declared_length: str | None,
chunks: AsyncIterator[bytes],
) -> Attachment:
storage = require_storage(storage)
validate_declared_size(declared_length, storage.settings.max_upload_bytes)
timeout_error = invalid_request("Истекло время загрузки")
with upload_slot(capacity):
async with operation_timeout(
storage.settings.upload_timeout_seconds, timeout_error
):
result = await upload_attachment(
session,
storage,
user,
chat_id,
filename=filename,
content_type=content_type,
chunks=chunks,
)
return result
async def read_original(
chunks: AsyncIterator[bytes],
path: Path,
maximum: int,
chunk_bytes: int,
) -> OriginalFile:
checksum = hashlib.sha256()
size = 0
with path.open("wb") as target:
async for chunk in chunks:
size += len(chunk)
if size > maximum:
raise payload_too_large()
checksum.update(chunk)
view = memoryview(chunk)
for offset in range(0, len(view), chunk_bytes):
target.write(view[offset : offset + chunk_bytes])
if size == 0:
raise unsupported_media_type("Пустой файл")
return OriginalFile(path, size, checksum.hexdigest())
async def compensate_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Неоднозначный COMMIT сначала проверяется в БД: готовый объект не удаляется."""
await session.rollback()
intent = await repository.lock_upload(session, id)
if intent is not None:
if await repository.get_by_key(session, intent.object_key) is None:
await storage.delete_object(intent.object_key)
await repository.finish_upload(session, id)
await session.commit()
async def recover_upload(
session: AsyncSession, storage: MinioStorage, id: UUID
) -> None:
"""Локальное восстановление после ошибки; журнал остаётся при отказе компенсации."""
try:
await compensate_upload(session, storage, id)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Upload compensation", error)
# Ошибка повторного rollback не должна скрыть исходную ошибку или отмену.
with suppress(*RECOVERABLE_STORAGE_ERRORS):
await session.rollback()
async def publish_original(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
original: OriginalFile,
filename: str,
content_type: str,
) -> Attachment:
id = uuid4()
key = f"originals/{uuid4().hex}"
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
await repository.reserve_upload(session, id, key)
await session.commit()
try:
if await repository.lock_upload(session, id) is None:
raise unavailable("Загрузка отменена до записи оригинала")
await storage.put_file(
key,
original.path,
size_bytes=original.size_bytes,
content_type=content_type,
sha256=original.sha256,
)
record = await repository.create_attachment(
session,
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=key,
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await repository.finish_upload(session, id)
result = Attachment.model_validate(record)
await session.commit()
except BaseException:
# Отмена тоже требует дождаться SDK и компенсировать незавершённый PUT.
with CancelScope(shield=True):
await recover_upload(session, storage, id)
raise
return result
async def upload_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
chat_id: UUID,
*,
filename: str,
content_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
await ensure_chat_member(session, user, chat_id)
validate_filename(filename)
if content_type not in {"image/png", "image/jpeg"}:
raise unsupported_media_type("Принимаются только image/png и image/jpeg")
# Не держим соединение PostgreSQL во время чтения медленного клиента.
await session.rollback()
settings = storage.settings
with TemporaryDirectory(prefix="messenger-upload-") as directory:
original = await read_original(
chunks,
Path(directory) / "original",
settings.max_upload_bytes,
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await publish_original(
session,
storage,
user,
chat_id,
original=original,
filename=filename,
content_type=content_type,
)
return result
async def get_attachment(
session: AsyncSession, user: User, attachment_id: UUID
) -> Attachment:
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
await ensure_chat_member(session, user, record.chat_id)
return Attachment.model_validate(record)
async def download_attachment(
session: AsyncSession,
storage: MinioStorage | None,
user: User,
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[Attachment, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if variant != "original":
raise not_found()
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
key = record.object_key
await session.rollback()
info = await storage.stat_object(key)
if (
info is None
or info.size_bytes != result.size_bytes
or info.content_type != result.content_type
):
raise unavailable("Оригинал временно недоступен")
stream = await storage.open_object(key)
return result, stream
async def validate_message_attachments(
session: AsyncSession,
user: User,
chat_id: UUID,
attachment_ids: list[UUID],
) -> None:
await ensure_chat_member(session, user, chat_id)
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status != "ready" for record in records):
raise conflict("Вложение ещё не готово")
async def reconcile_attachments(
session: AsyncSession,
storage: MinioStorage,
*,
batch_size: int = 20,
dry_run: bool = True,
) -> CleanupResult:
if not 1 <= batch_size <= 100:
raise invalid_request("batch_size должен быть от 1 до 100")
intents = await repository.stale_uploads(
session, storage.settings.cleanup_grace_seconds, batch_size
)
candidates = deleted = 0
for intent in intents:
if await repository.get_by_key(session, intent.object_key) is None:
candidates += 1
if not dry_run:
await storage.delete_object(intent.object_key)
deleted += 1
if not dry_run:
await repository.finish_upload(session, intent.id)
if dry_run:
await session.rollback()
else:
await session.commit()
return CleanupResult(len(intents), candidates, deleted)
+84
View File
@@ -0,0 +1,84 @@
import re
from datetime import UTC, datetime, timedelta
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.domain.users import to_user_schema
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.session import AuthSession
from app.schemas.user import (
PASSWORD_MAX_LENGTH,
PASSWORD_MIN_LENGTH,
USERNAME_PATTERN,
User,
)
from app.storage import sessions as sessions_storage
from app.storage import users as users_storage
USERNAME_RE = re.compile(USERNAME_PATTERN)
async def create_session(
session: AsyncSession,
redis: Redis,
username: str,
password: str,
ttl_seconds: int,
) -> AuthSession:
"""Проверяет учётные данные и выдаёт новый Bearer-токен."""
if (
USERNAME_RE.fullmatch(username) is None
or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH
or not password.strip()
):
raise unauthorized_basic()
user = await users_storage.get_user_by_username(session, username)
await session.commit()
if user is None:
await security.dummy_verify_password(password)
raise unauthorized_basic()
if not await security.verify_password(password, user.password_hash):
raise unauthorized_basic()
token = security.generate_token()
expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
await sessions_storage.create_session(
redis,
token_hash=security.hash_token(token),
user_id=user.id,
expires_at=expires_at,
)
auth_session = AuthSession(
token=token, expires_at=expires_at, user=to_user_schema(user)
)
return auth_session
async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User:
"""Возвращает владельца действующей сессии или бросает 401."""
user_id = await sessions_storage.get_session_user_id(
redis, security.hash_token(token)
)
if user_id is None:
raise unauthorized_bearer()
user = await users_storage.get_user_by_id(session, user_id)
if user is None:
raise unauthorized_bearer()
current_user = to_user_schema(user)
return current_user
async def revoke_session(redis: Redis, token: str) -> None:
"""Отзывает текущую сессию; повторный отзыв даёт 401."""
is_deleted = await sessions_storage.delete_session(
redis, security.hash_token(token)
)
if not is_deleted:
raise unauthorized_bearer()
+104
View File
@@ -0,0 +1,104 @@
from uuid import UUID
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.pagination import (
KIND_CHATS,
build_next_cursor,
decode_cursor,
split_page,
)
from app.errors.exceptions import invalid_request, not_found
from app.models.chat import Chat as ChatModel
from app.schemas.chat import Chat, ChatPage, CreateChat
from app.schemas.user import User
from app.storage import chats as chats_storage
from app.storage import users as users_storage
from app.storage.types import Position
UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя"
def to_chat_schema(chat: ChatModel) -> Chat:
"""Преобразует ORM-модель чата с загруженными участниками в схему."""
chat_schema = Chat(
id=chat.id,
title=chat.title,
member_ids=[member.user_id for member in chat.members],
created_at=chat.created_at,
)
return chat_schema
def validate_new_chat(creator: User, new_chat: CreateChat) -> None:
"""Проверяет состав участников без обращения к базе."""
member_ids = new_chat.member_ids
if creator.id not in member_ids:
raise invalid_request("Создатель чата должен входить в member_ids")
if len(set(member_ids)) != len(member_ids):
raise invalid_request("member_ids содержит дубликаты")
async def create_chat(
session: AsyncSession, creator: User, new_chat: CreateChat
) -> Chat:
"""Создаёт чат с участниками атомарно; неверный состав даёт 400."""
validate_new_chat(creator, new_chat)
existing_ids = await users_storage.get_existing_user_ids(
session, new_chat.member_ids
)
if existing_ids != set(new_chat.member_ids):
raise invalid_request(UNKNOWN_MEMBER_MESSAGE)
try:
chat = await chats_storage.create_chat(
session, title=new_chat.title, member_ids=new_chat.member_ids
)
await session.commit()
except IntegrityError:
await session.rollback()
raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None
created_chat = to_chat_schema(chat)
return created_chat
async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat:
"""Возвращает чат участнику; чужой и несуществующий чат дают 404."""
chat = await chats_storage.get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise not_found()
found_chat = to_chat_schema(chat)
return found_chat
async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None:
"""Бросает 404, если пользователь не состоит в чате или чата нет."""
if not await chats_storage.is_chat_member(session, chat_id, user.id):
raise not_found()
async def list_chats(
session: AsyncSession,
user: User,
cursor: str | None,
limit: int,
) -> ChatPage:
"""Возвращает страницу чатов пользователя по (created_at, id)."""
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_CHATS, user.id)
rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
items = [to_chat_schema(chat) for chat in page_rows]
page = ChatPage(
items=items,
next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next),
)
return page
+33
View File
@@ -0,0 +1,33 @@
"""Условия готовности реплики и ограниченная проверка зависимостей."""
import asyncio
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncEngine
from app.core.lifecycle import Lifecycle
from app.errors.exceptions import unavailable
from app.errors.handlers import operation_timeout
from app.schemas.base import Health
from app.storage.health import check_database, check_redis
from app.storage.s3 import MinioStorage
def liveness() -> Health:
return Health(status='ok')
async def readiness(
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
lifecycle: Lifecycle, timeout_seconds: float,
) -> Health:
if lifecycle.draining:
raise unavailable()
checks = [check_database(engine), check_redis(redis)]
if storage is not None:
checks.append(storage.check_available())
async with operation_timeout(timeout_seconds, unavailable()):
dependencies = await asyncio.gather(*checks, return_exceptions=True)
if not all(result is True for result in dependencies):
raise unavailable()
return Health(status='ok')
+72
View File
@@ -0,0 +1,72 @@
"""Атомарная отправка сообщения с защитой от повторов."""
import hashlib
import json
import re
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.attachments import validate_message_attachments
from app.errors.exceptions import conflict, invalid_request, unavailable
from app.schemas.message import CreateMessage, Message
from app.schemas.user import User
from app.storage import idempotency as key_storage
from app.storage import messages as message_storage
from app.storage import attachments as attachment_storage
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def validate_idempotency_key(key: str | None) -> str:
"""Проверяет обязательный ASCII-ключ запроса."""
if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None:
raise invalid_request("Требуется корректный Idempotency-Key")
return key
def request_fingerprint(new_message: CreateMessage) -> str:
"""Хеширует нормализованное содержимое запроса."""
body = new_message.model_dump(mode="json")
# Сохраняет отпечатки текстовых запросов lab3 и равенство absent / [].
if not body["attachment_ids"]:
body.pop("attachment_ids")
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
return hashlib.sha256(canonical.encode()).hexdigest()
async def send_message_idempotent(
session: AsyncSession,
sender: User,
chat_id: UUID,
new_message: CreateMessage,
idempotency_key: str,
max_messages: int,
ttl_seconds: int,
) -> Message:
"""Создаёт сообщение или возвращает результат прежнего запроса."""
await ensure_chat_member(session, sender, chat_id)
scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key)
fingerprint = request_fingerprint(new_message)
await key_storage.lock_key(session, scope)
previous = await key_storage.get_active_key(session, scope, ttl_seconds)
if previous is not None:
if previous.request_hash != fingerprint:
raise conflict("Idempotency-Key уже использован для другого сообщения")
message = await message_storage.get_message(session, previous.message_id)
else:
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
if not await message_storage.reserve_message_slot(session, max_messages):
raise unavailable("Достигнут лимит количества сообщений")
message = await message_storage.create_message(
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
)
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
await key_storage.save_result(session, scope, fingerprint, message.id)
links = await attachment_storage.message_attachment_ids(session, [message.id])
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
await session.commit()
return result
+23
View File
@@ -0,0 +1,23 @@
"""Синхронная проверка изображения; вызывается только из ограниченного процесса."""
from pathlib import Path
import warnings
from PIL import Image
from app.errors.exceptions import payload_too_large, unsupported_media_type
def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
Image.MAX_IMAGE_PIXELS = max_pixels
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(path, formats=["PNG", "JPEG"]) as image:
if image.width * image.height > max_pixels:
raise payload_too_large("Превышен лимит пикселей изображения")
image_format = image.format
if image_format is None or Image.MIME.get(image_format) != content_type:
raise unsupported_media_type("Содержимое не соответствует Content-Type")
image.verify()
# Для JPEG одной verify() недостаточно: декодировать весь оригинал.
with Image.open(path, formats=["PNG", "JPEG"]) as image:
image.load()
+48
View File
@@ -0,0 +1,48 @@
"""Проверка имени и запуск ограниченного декодера."""
import asyncio
from pathlib import Path
import sys
import unicodedata
from anyio import CancelScope
from app.core.s3 import S3Settings
from app.errors.exceptions import (
image_validation_error,
invalid_request,
payload_too_large,
)
from app.errors.handlers import operation_timeout
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or any(
char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
for char in filename
):
raise invalid_request("Недопустимое имя файла")
async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"app.core.image_worker",
str(path),
content_type,
str(settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
error = payload_too_large("Превышен бюджет проверки изображения")
try:
async with operation_timeout(settings.image_timeout_seconds, error):
code = await process.wait()
finally:
if process.returncode is None:
process.kill()
with CancelScope(shield=True):
await process.wait()
if code != 0:
raise image_validation_error(code)
+43
View File
@@ -0,0 +1,43 @@
from uuid import UUID
from sqlalchemy.ext.asyncio import AsyncSession
from app.domain.chats import ensure_chat_member
from app.domain.pagination import (
KIND_MESSAGES,
build_next_cursor,
decode_cursor,
split_page,
)
from app.schemas.message import Message, MessagePage
from app.schemas.user import User
from app.storage import messages as messages_storage
from app.storage import attachments as attachments_storage
from app.storage.types import Position
async def list_messages(
session: AsyncSession,
user: User,
chat_id: UUID,
cursor: str | None,
limit: int,
) -> MessagePage:
"""Возвращает страницу истории чата по (created_at, id)."""
after: Position | None = None
if cursor is not None:
after = decode_cursor(cursor, KIND_MESSAGES, chat_id)
await ensure_chat_member(session, user, chat_id)
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
items = [Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
) for message in page_rows]
page = MessagePage(
items=items,
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
)
return page
+83
View File
@@ -0,0 +1,83 @@
import base64
import json
from collections.abc import Sequence
from datetime import datetime
from typing import Protocol, TypeVar
from uuid import UUID
from app.errors.exceptions import invalid_request
from app.storage.types import Position
KIND_CHATS = "chats"
KIND_MESSAGES = "messages"
T = TypeVar("T")
class Positioned(Protocol):
"""Объект с позицией в порядке (created_at, id)."""
@property
def id(self) -> UUID: ...
@property
def created_at(self) -> datetime: ...
def encode_cursor(
kind: str,
resource_id: UUID,
created_at: datetime,
item_id: UUID,
) -> str:
"""Кодирует позицию и ресурс, к которому она привязана, в непрозрачный cursor."""
payload = {
"kind": kind,
"resource_id": str(resource_id),
"created_at": created_at.isoformat(),
"item_id": str(item_id),
}
raw = json.dumps(payload).encode()
return base64.urlsafe_b64encode(raw).decode()
def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position:
"""Возвращает позицию из cursor; битый или чужой cursor даёт 400."""
try:
payload = json.loads(base64.urlsafe_b64decode(cursor))
cursor_kind = payload["kind"]
cursor_resource_id = UUID(payload["resource_id"])
created_at = datetime.fromisoformat(payload["created_at"])
item_id = UUID(payload["item_id"])
except (ValueError, KeyError, TypeError, AttributeError):
raise invalid_request("Некорректный cursor") from None
if (
cursor_kind != kind
or cursor_resource_id != resource_id
or created_at.tzinfo is None
):
raise invalid_request("Некорректный cursor")
return created_at, item_id
def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]:
"""Отделяет страницу от лишней строки limit + 1, по которой видно продолжение."""
has_next = len(rows) > limit
items = rows[:limit]
return items, has_next
def build_next_cursor(
kind: str,
resource_id: UUID,
items: Sequence[Positioned],
has_next: bool,
) -> str | None:
"""Возвращает cursor на последний элемент страницы или None, если она последняя."""
if not has_next:
return None
last_item = items[-1]
return encode_cursor(kind, resource_id, last_item.created_at, last_item.id)
+41
View File
@@ -0,0 +1,41 @@
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.core import security
from app.errors.exceptions import conflict
from app.models.user import User as UserModel
from app.schemas.user import CreateUser, User
from app.storage import users as users_storage
USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято"
def to_user_schema(user: UserModel) -> User:
"""Преобразует ORM-модель в публичную схему без password_hash."""
user_schema = User.model_validate(user)
return user_schema
async def register_user(session: AsyncSession, new_user: CreateUser) -> User:
"""Регистрирует пользователя; занятый username даёт 409."""
existing_user = await users_storage.get_user_by_username(session, new_user.username)
await session.commit()
if existing_user is not None:
raise conflict(USERNAME_TAKEN_MESSAGE)
password_hash = await security.hash_password(new_user.password)
try:
user = await users_storage.create_user(
session,
username=new_user.username,
display_name=new_user.display_name,
password_hash=password_hash,
)
await session.commit()
except IntegrityError:
await session.rollback()
raise conflict(USERNAME_TAKEN_MESSAGE) from None
created_user = to_user_schema(user)
return created_user
View File
+52
View File
@@ -0,0 +1,52 @@
"""HTTP-параметры и упаковка ответов; сценарии находятся в domain."""
from typing import Annotated
from urllib.parse import quote
from uuid import UUID
from fastapi import APIRouter, Query, Request
from fastapi.responses import StreamingResponse
from starlette.background import BackgroundTask
from app.domain import attachments
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
from app.schemas.attachment import Attachment
router = APIRouter(prefix='/api/v1', tags=['attachments'])
@router.post('/chats/{chat_id}/attachments', status_code=201)
async def upload(
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
storage: S3Storage, capacity: UploadCapacity,
filename: Annotated[str, Query(min_length=1, max_length=255)],
) -> Attachment:
return await attachments.receive_attachment(
session, storage, user, chat_id, capacity=capacity, filename=filename,
content_type=request.headers.get('content-type', '').lower(),
declared_length=request.headers.get('content-length'), chunks=request.stream(),
)
@router.get('/attachments/{attachment_id}')
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
return await attachments.get_attachment(session, user, attachment_id)
@router.get('/attachments/{attachment_id}/content')
async def content(
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
variant: Annotated[str, Query()] = 'original',
) -> StreamingResponse:
record, stream = await attachments.download_attachment(
session, storage, user, attachment_id, variant=variant,
)
return StreamingResponse(
stream, media_type=record.content_type,
headers={
'Content-Length': str(record.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
},
background=BackgroundTask(stream.aclose),
)
+32
View File
@@ -0,0 +1,32 @@
from fastapi import APIRouter, Response
from app.domain.auth import create_session, revoke_session
from app.endpoints.dependencies import (
AppSettings,
BasicCredentials,
BearerToken,
DbSession,
RedisClient,
)
from app.schemas.session import AuthSession
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
@router.post("/sessions", status_code=201)
async def login(
credentials: BasicCredentials,
session: DbSession,
redis: RedisClient,
settings: AppSettings,
) -> AuthSession:
username, password = credentials
return await create_session(
session, redis, username, password, settings.session_ttl_seconds
)
@router.delete("/sessions/current", status_code=204)
async def logout(token: BearerToken, redis: RedisClient) -> Response:
await revoke_session(redis, token)
return Response(status_code=204)
+32
View File
@@ -0,0 +1,32 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Query
from app.domain.chats import create_chat, get_chat, list_chats
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.chat import Chat, ChatPage, CreateChat
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
@router.post("", status_code=201)
async def create_new_chat(
new_chat: CreateChat, creator: CurrentUser, session: DbSession
) -> Chat:
return await create_chat(session, creator, new_chat)
@router.get("")
async def list_user_chats(
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> ChatPage:
return await list_chats(session, user, cursor, limit)
@router.get("/{chat_id}")
async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat:
return await get_chat(session, user, chat_id)
+112
View File
@@ -0,0 +1,112 @@
import base64
import binascii
from collections.abc import AsyncIterator
from typing import Annotated
from fastapi import Depends, Request, Security
from fastapi.openapi.models import HTTPBase, HTTPBearer
from fastapi.security.base import SecurityBase
from redis.asyncio import Redis
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import Settings
from app.core.request_capacity import RequestCapacity
from app.domain.auth import authenticate_token
from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
from app.schemas.user import User
from app.storage.s3 import MinioStorage
async def get_settings(request: Request) -> Settings:
settings: Settings = request.app.state.settings
return settings
async def get_db(request: Request) -> AsyncIterator[AsyncSession]:
"""Одна AsyncSession на HTTP-запрос; незакоммиченная транзакция откатывается."""
session_factory = request.app.state.session_factory
async with session_factory() as session:
yield session
async def get_redis(request: Request) -> Redis:
redis: Redis = request.app.state.redis
return redis
def get_s3(request: Request) -> MinioStorage | None:
return request.app.state.s3
def get_upload_capacity(request: Request) -> RequestCapacity:
return request.app.state.upload_capacity
class AuthorizationHeader(SecurityBase):
"""Отдаёт сырой заголовок Authorization и описывает схему для OpenAPI."""
def __init__(self, model: HTTPBase, scheme_name: str) -> None:
self.model = model
self.scheme_name = scheme_name
async def __call__(self, request: Request) -> str | None:
return request.headers.get("Authorization")
basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth")
bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth")
def _split_authorization(authorization: str | None) -> tuple[str, str] | None:
if authorization is None:
return None
scheme, _, value = authorization.partition(" ")
if not value.strip():
return None
return scheme.lower(), value.strip()
async def get_bearer_token(
authorization: Annotated[str | None, Security(bearer_auth)],
) -> str:
parts = _split_authorization(authorization)
if parts is None or parts[0] != "bearer":
raise unauthorized_bearer()
return parts[1]
async def get_basic_credentials(
authorization: Annotated[str | None, Security(basic_auth)],
) -> tuple[str, str]:
"""(username, password) из `Authorization: Basic`; пароль может содержать ':'."""
parts = _split_authorization(authorization)
if parts is None or parts[0] != "basic":
raise unauthorized_basic()
try:
decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
except (binascii.Error, UnicodeDecodeError, ValueError):
raise unauthorized_basic() from None
username, separator, password = decoded.partition(":")
if not separator or not username:
raise unauthorized_basic()
return username, password
DbSession = Annotated[AsyncSession, Depends(get_db)]
RedisClient = Annotated[Redis, Depends(get_redis)]
AppSettings = Annotated[Settings, Depends(get_settings)]
S3Storage = Annotated[MinioStorage | None, Depends(get_s3)]
UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)]
BearerToken = Annotated[str, Depends(get_bearer_token)]
BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)]
async def get_current_user(
token: BearerToken, session: DbSession, redis: RedisClient
) -> User:
return await authenticate_token(session, redis, token)
CurrentUser = Annotated[User, Depends(get_current_user)]
+20
View File
@@ -0,0 +1,20 @@
from fastapi import APIRouter, Request
from app.domain import health
from app.schemas.base import Health
router = APIRouter(prefix='/health', tags=['health'])
@router.get('/live')
async def liveness() -> Health:
return health.liveness()
@router.get('/ready')
async def readiness(request: Request) -> Health:
state = request.app.state
return await health.readiness(
state.engine, state.redis, state.s3, state.lifecycle,
state.settings.readiness_timeout_seconds,
)
+38
View File
@@ -0,0 +1,38 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Header, Query
from app.domain.idempotency import send_message_idempotent, validate_idempotency_key
from app.domain.messages import list_messages
from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession
from app.schemas.message import CreateMessage, Message, MessagePage
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
@router.post("/{chat_id}/messages", status_code=201)
async def send_new_message(
chat_id: UUID,
new_message: CreateMessage,
sender: CurrentUser,
session: DbSession,
settings: AppSettings,
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
) -> Message:
return await send_message_idempotent(
session, sender, chat_id, new_message,
validate_idempotency_key(idempotency_key),
settings.max_messages, settings.idempotency_ttl_seconds,
)
@router.get("/{chat_id}/messages")
async def get_history_messages(
chat_id: UUID,
user: CurrentUser,
session: DbSession,
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
) -> MessagePage:
return await list_messages(session, user, chat_id, cursor, limit)
+17
View File
@@ -0,0 +1,17 @@
from fastapi import APIRouter
from app.domain.users import register_user
from app.endpoints.dependencies import CurrentUser, DbSession
from app.schemas.user import CreateUser, User
router = APIRouter(prefix="/api/v1/users", tags=["users"])
@router.post("", status_code=201)
async def create_user(new_user: CreateUser, session: DbSession) -> User:
return await register_user(session, new_user)
@router.get("/me")
async def get_me(user: CurrentUser) -> User:
return user
View File
+109
View File
@@ -0,0 +1,109 @@
from collections.abc import Mapping
from PIL import Image, UnidentifiedImageError
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(
self,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.headers = headers
def invalid_request(message: str = "Неверный запрос") -> ApiError:
"""Создаёт ошибку некорректного или недопустимого запроса."""
return ApiError(400, "invalid_request", message)
def unauthorized(
message: str = "Нет действительной аутентификации",
www_authenticate: str | None = None,
) -> ApiError:
"""Создаёт ошибку отсутствующей или недействительной аутентификации."""
headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None
return ApiError(401, "unauthorized", message, headers)
def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError:
"""401 для POST /auth/sessions (одинаковый для неверного username и пароля)."""
return unauthorized(message, BASIC_CHALLENGE)
def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError:
"""401 для бизнес-endpoint: нет, истёк, отозван или неизвестен токен."""
return unauthorized(message, BEARER_CHALLENGE)
def not_found(message: str = "Объект не найден") -> ApiError:
"""Создаёт ошибку, указывающую на отсутствие запрошенного ресурса."""
return ApiError(404, "not_found", message)
def conflict(message: str = "Конфликт") -> ApiError:
"""Создаёт ошибку конфликта с текущим состоянием ресурса."""
return ApiError(409, "conflict", message)
def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError:
"""Создаёт ошибку, указывающую на превышение допустимого размера запроса."""
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(
message: str = "Требуется Content-Type: application/json",
) -> ApiError:
"""Создаёт ошибку неподдерживаемого формата или типа содержимого запроса."""
return ApiError(415, "unsupported_media_type", message)
def rate_limited(retry_after: int = 1) -> ApiError:
"""Создаёт ошибку превышения допустимой частоты запросов."""
return ApiError(
429,
"rate_limited",
"Слишком много запросов",
{"Retry-After": str(max(1, retry_after))},
)
def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
"""Создаёт ошибку временной недоступности сервиса."""
return ApiError(503, "unavailable", message)
def internal_error() -> ApiError:
"""Создаёт ошибку внутренней ошибки сервера."""
return ApiError(500, "internal_error", "Внутренняя ошибка сервера")
IMAGE_WORKER_ERRORS = (
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
)
def image_worker_error_code(error: Exception) -> int:
code = 2
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
declared_limit = isinstance(error, ApiError) and error.status_code == 413
if resource_error or declared_limit:
code = 3
return code
def image_validation_error(code: int) -> ApiError:
error = unsupported_media_type("Повреждённое изображение или несовпадение MIME")
if code == 3 or code < 0:
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
return error
+225
View File
@@ -0,0 +1,225 @@
import asyncio
import logging
from contextlib import asynccontextmanager
from collections.abc import Mapping
from uuid import uuid4
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from minio.error import MinioException
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import TimeoutError as RedisTimeoutError
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.requests import ClientDisconnect
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.instance import INSTANCE_ID_HEADER
from app.core.request_id import REQUEST_ID_HEADER
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
logger = logging.getLogger(__name__)
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
TRANSIENT_SQLSTATES = frozenset({"55P03"})
RECOVERABLE_STORAGE_ERRORS = (
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
)
@asynccontextmanager
async def operation_timeout(seconds: float, error: ApiError):
"""Единое преобразование таймаутов; владелец операции задаёт ожидаемую ошибку."""
try:
async with asyncio.timeout(seconds):
yield
except TimeoutError:
raise error from None
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
def log_recoverable_error(operation: str, error: Exception) -> None:
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
logger.warning('%s deferred: %s', operation, type(error).__name__)
def _get_request_id(request: Request) -> str:
request_id = getattr(request.state, "request_id", None)
if request_id is None:
request_id = str(uuid4())
request.state.request_id = request_id
return request_id
def _build_response(
request: Request,
status_code: int,
code: str,
message: str,
headers: Mapping[str, str] | None = None,
) -> JSONResponse:
request_id = _get_request_id(request)
response_headers = {
**(headers or {}),
REQUEST_ID_HEADER: request_id,
}
instance_id = getattr(request.app.state, "instance_id", None)
if instance_id is not None:
response_headers[INSTANCE_ID_HEADER] = instance_id
body = {
"error": {
"code": code,
"message": message,
"request_id": request_id,
}
}
return JSONResponse(
status_code=status_code,
content=body,
headers=response_headers,
)
async def api_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, ApiError)
return _build_response(
request,
exc.status_code,
exc.code,
exc.message,
exc.headers,
)
async def routing_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, StarletteHTTPException)
error = invalid_request()
if exc.status_code == 404:
error = not_found()
return _build_response(
request,
exc.status_code,
error.code,
error.message,
exc.headers,
)
async def validation_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
assert isinstance(exc, RequestValidationError)
fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()})
message = "Неверный запрос"
if fields:
message = f"{message}: {', '.join(fields)}"
return _build_response(
request,
400,
"invalid_request",
message,
)
async def dependency_unavailable_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
"""PostgreSQL/Redis/S3 недоступны либо исчерпан пул: 503 без внутренних деталей."""
request_id = _get_request_id(request)
logger.error(
"Dependency unavailable request_id=%s: %s",
request_id,
type(exc).__name__,
)
error = unavailable()
return _build_response(
request,
error.status_code,
error.code,
error.message,
{"Retry-After": "1"},
)
def _is_transient_database_error(exc: DBAPIError) -> bool:
sqlstate = getattr(exc.orig, "sqlstate", None) or ""
return (
sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES)
or sqlstate in TRANSIENT_SQLSTATES
)
async def database_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
"""Временные ошибки PostgreSQL (таймаут запроса, блокировки, нехватка ресурсов) — 503."""
assert isinstance(exc, DBAPIError)
handler = unhandled_error_handler
if _is_transient_database_error(exc):
handler = dependency_unavailable_handler
return await handler(request, exc)
async def unhandled_error_handler(
request: Request,
exc: Exception,
) -> JSONResponse:
request_id = _get_request_id(request)
logger.error(
"Unhandled error request_id=%s",
request_id,
exc_info=exc,
)
error = internal_error()
return _build_response(
request,
error.status_code,
error.code,
error.message,
)
def register_exception_handlers(app: FastAPI) -> None:
app.add_exception_handler(ApiError, api_error_handler)
app.add_exception_handler(ClientDisconnect, client_disconnect_handler)
app.add_exception_handler(MinioException, dependency_unavailable_handler)
app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler)
app.add_exception_handler(StarletteHTTPException, routing_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(OperationalError, dependency_unavailable_handler)
app.add_exception_handler(InterfaceError, dependency_unavailable_handler)
app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(DBAPIError, database_error_handler)
app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler)
app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler)
app.add_exception_handler(OSError, dependency_unavailable_handler)
app.add_exception_handler(Exception, unhandled_error_handler)
+92
View File
@@ -0,0 +1,92 @@
import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
import uvicorn
from fastapi import Depends, FastAPI
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.instance import InstanceIdMiddleware, resolve_instance_id
from app.core.lifecycle import GracefulServer, Lifecycle
from app.core.redis import create_redis_client
from app.core.request_capacity import acquire_request_slot, create_request_capacity
from app.core.request_id import RequestIdMiddleware
from app.endpoints.auth import router as auth_router
from app.endpoints.attachments import router as attachments_router
from app.endpoints.chats import router as chats_router
from app.endpoints.health import router as health_router
from app.endpoints.messages import router as messages_router
from app.endpoints.users import router as users_router
from app.errors.handlers import register_exception_handlers
from app.storage.s3 import create_s3_storage
def configure_logging(settings: Settings) -> None:
logging.basicConfig(
level=settings.log_level,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings() # type: ignore
configure_logging(settings)
engine = create_db_engine(settings)
redis = create_redis_client(settings)
s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None
@asynccontextmanager
async def lifespan(_: FastAPI) -> AsyncGenerator[None]:
yield
if s3 is not None:
await s3.aclose()
await redis.aclose()
await engine.dispose()
application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan)
application.state.settings = settings
application.state.engine = engine
application.state.session_factory = create_session_factory(engine)
application.state.redis = redis
application.state.s3 = s3
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
application.state.lifecycle = Lifecycle()
application.add_middleware(RequestIdMiddleware)
application.add_middleware(
InstanceIdMiddleware, instance_id=application.state.instance_id
)
register_exception_handlers(application)
limited = [Depends(acquire_request_slot)]
application.include_router(health_router)
application.include_router(users_router, dependencies=limited)
application.include_router(auth_router, dependencies=limited)
application.include_router(chats_router, dependencies=limited)
application.include_router(messages_router, dependencies=limited)
application.include_router(attachments_router, dependencies=limited)
return application
def main() -> None:
settings = Settings() # type: ignore
application = create_app(settings)
config = uvicorn.Config(
application,
host=settings.app_host,
port=settings.app_port,
log_level=settings.log_level.lower(),
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
)
GracefulServer(
config, application.state.lifecycle, settings.shutdown_drain_seconds
).run()
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
from app.models.base import Base
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.models.counter import Counter
from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
+57
View File
@@ -0,0 +1,57 @@
"""Метаданные оригиналов и журнал операций до записи в S3."""
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Attachment(Base):
__tablename__ = "attachments"
__table_args__ = (
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status = 'ready'", name="status"),
Index("ix_attachments_chat_id", "chat_id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
object_key: Mapped[str] = mapped_column(String(128), unique=True)
filename: Mapped[str] = mapped_column(String(255))
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int]
sha256: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16), server_default="ready")
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
)
attachment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("attachments.id"), primary_key=True
)
position: Mapped[int]
class UploadIntent(Base):
"""Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных."""
__tablename__ = "upload_intents"
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
object_key: Mapped[str] = mapped_column(String(128), unique=True)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+27
View File
@@ -0,0 +1,27 @@
import uuid
from datetime import datetime
from typing import Any, ClassVar
from sqlalchemy import DateTime, MetaData
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
from sqlalchemy.orm import DeclarativeBase
from sqlalchemy.types import TypeEngine
NAMING_CONVENTION = {
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
class Base(DeclarativeBase):
"""Базовый класс ORM-моделей."""
metadata = MetaData(naming_convention=NAMING_CONVENTION)
type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = {
uuid.UUID: PG_UUID(as_uuid=True),
datetime: DateTime(timezone=True),
}
+32
View File
@@ -0,0 +1,32 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class Chat(Base):
"""Чат с фиксированным составом участников."""
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
title: Mapped[str] = mapped_column(String(100))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
members: Mapped[list["ChatMember"]] = relationship(
back_populates="chat",
cascade="all, delete-orphan",
passive_deletes=True,
lazy="selectin",
)
messages: Mapped[list["Message"]] = relationship(
back_populates="chat", lazy="raise"
)
+26
View File
@@ -0,0 +1,26 @@
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class ChatMember(Base):
"""Членство пользователя в чате."""
__tablename__ = "chat_members"
__table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),)
chat_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise")
user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise")
+13
View File
@@ -0,0 +1,13 @@
from sqlalchemy import BigInteger, String
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class Counter(Base):
"""Именованный счётчик с атомарным обновлением."""
__tablename__ = "counters"
name: Mapped[str] = mapped_column(String(64), primary_key=True)
value: Mapped[int] = mapped_column(BigInteger)
+24
View File
@@ -0,0 +1,24 @@
"""Сохранённый результат отправки сообщения."""
import uuid
from datetime import datetime
from sqlalchemy import ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class IdempotencyKey(Base):
"""Ключ запроса и связанное сообщение."""
__tablename__ = "idempotency_keys"
__table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),)
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
method: Mapped[str] = mapped_column(String(8), primary_key=True)
path: Mapped[str] = mapped_column(String(255), primary_key=True)
key: Mapped[str] = mapped_column(String(128), primary_key=True)
request_hash: Mapped[str] = mapped_column(String(64))
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id"))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+30
View File
@@ -0,0 +1,30 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import ForeignKey, Index, Text, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat import Chat
from app.models.user import User
class Message(Base):
"""Сообщение в чате."""
__tablename__ = "messages"
__table_args__ = (
Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"),
)
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
text: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise")
sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise")
+31
View File
@@ -0,0 +1,31 @@
import uuid
from datetime import datetime
from typing import TYPE_CHECKING
from sqlalchemy import String, func
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
if TYPE_CHECKING:
from app.models.chat_member import ChatMember
from app.models.message import Message
class User(Base):
"""Пользователь и его учётные данные."""
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
username: Mapped[str] = mapped_column(String(32), unique=True)
display_name: Mapped[str] = mapped_column(String(100))
password_hash: Mapped[str] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
memberships: Mapped[list["ChatMember"]] = relationship(
back_populates="user", lazy="raise"
)
messages: Mapped[list["Message"]] = relationship(
back_populates="sender", lazy="raise"
)
+6
View File
@@ -0,0 +1,6 @@
from app.schemas.chat import Chat
from app.schemas.message import Message
from app.schemas.session import AuthSession
from app.schemas.user import User
__all__ = ["AuthSession", "Chat", "Message", "User"]
+23
View File
@@ -0,0 +1,23 @@
"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту."""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import Field
from app.schemas.base import Base
class Attachment(Base):
chat_id: UUID
owner_id: UUID
filename: str = Field(min_length=1, max_length=255)
content_type: Literal["image/png", "image/jpeg"]
size_bytes: int = Field(ge=1, le=10_485_760)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
status: Literal["ready"] = "ready"
variants: list[object] = Field(default_factory=list, max_length=0)
error_code: None = None
created_at: datetime
updated_at: datetime
+35
View File
@@ -0,0 +1,35 @@
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, field_validator
class Base(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
id: UUID
class Health(BaseModel):
model_config = ConfigDict(extra="forbid")
status: Literal["ok"]
class BaseCreate(BaseModel):
model_config = ConfigDict(extra="forbid")
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
"""Отклоняет строки из одних пробелов и строки с NUL, который не хранит PostgreSQL."""
if isinstance(value, str):
if not value.strip():
raise ValueError("Строка не может состоять только из пробелов")
if "\x00" in value:
raise ValueError("Строка не может содержать символ NUL")
return value
class BasePage(BaseModel):
model_config = ConfigDict(extra="forbid")
+22
View File
@@ -0,0 +1,22 @@
from datetime import datetime
from uuid import UUID
from pydantic import Field
from app.schemas.base import Base, BaseCreate, BasePage
class Chat(Base):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
created_at: datetime
class CreateChat(BaseCreate):
title: str = Field(min_length=1, max_length=100)
member_ids: list[UUID] = Field(min_length=1, max_length=100)
class ChatPage(BasePage):
items: list[Chat]
next_cursor: str | None
+41
View File
@@ -0,0 +1,41 @@
from datetime import datetime
from typing import Self
from uuid import UUID
from pydantic import Field, field_validator, model_validator
from app.schemas.base import Base, BaseCreate, BasePage
class Message(Base):
chat_id: UUID
sender_id: UUID
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
created_at: datetime
class CreateMessage(BaseCreate):
text: str = Field(max_length=4000)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator("*", mode="before")
@classmethod
def reject_invalid_strings(cls, value: object) -> object:
if isinstance(value, str):
if "\x00" in value or (value != "" and not value.strip()):
raise ValueError("Некорректная строка сообщения")
return value
@model_validator(mode="after")
def require_content(self) -> Self:
if not self.text and not self.attachment_ids:
raise ValueError("Нужен текст или вложение")
if len(self.attachment_ids) != len(set(self.attachment_ids)):
raise ValueError("attachment_ids содержит дубликаты")
return self
class MessagePage(BasePage):
items: list[Message]
next_cursor: str | None
+17
View File
@@ -0,0 +1,17 @@
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from app.schemas.user import User
class AuthSession(BaseModel):
"""Ответ POST /auth/sessions (в OpenAPI — Session)."""
model_config = ConfigDict(extra="forbid")
token: str = Field(min_length=32)
token_type: Literal["Bearer"] = "Bearer"
expires_at: datetime
user: User
+25
View File
@@ -0,0 +1,25 @@
from datetime import datetime
from pydantic import Field
from app.schemas.base import Base, BaseCreate
USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$"
PASSWORD_MIN_LENGTH = 12
PASSWORD_MAX_LENGTH = 128
class User(Base):
"""Публичное представление пользователя. Пароль/хеш сюда не попадают никогда."""
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
created_at: datetime
class CreateUser(BaseCreate):
username: str = Field(pattern=USERNAME_PATTERN)
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(
min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False
)
+1
View File
@@ -0,0 +1 @@
"""Доступ к данным PostgreSQL и Redis."""
+99
View File
@@ -0,0 +1,99 @@
"""SQL-запросы метаданных и журнала; commit выполняется в domain."""
from datetime import timedelta
from uuid import UUID
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
async def create_attachment(session: AsyncSession, **values) -> Attachment:
record = Attachment(**values)
session.add(record)
await session.flush()
return record
async def get_attachment(
session: AsyncSession, attachment_id: UUID
) -> Attachment | None:
return await session.get(Attachment, attachment_id)
async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
return await session.scalar(select(Attachment).where(Attachment.object_key == key))
async def get_for_chat(
session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
) -> list[Attachment]:
records = []
if attachment_ids:
records = list(
await session.scalars(
select(Attachment).where(
Attachment.id.in_(attachment_ids),
Attachment.chat_id == chat_id,
)
)
)
return records
async def bind_attachments(
session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
) -> None:
session.add_all(
MessageAttachment(message_id=message_id, attachment_id=id, position=position)
for position, id in enumerate(attachment_ids)
)
await session.flush()
async def message_attachment_ids(
session: AsyncSession, message_ids: list[UUID]
) -> dict[UUID, list[UUID]]:
result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
if not message_ids:
return result
rows = await session.execute(
select(MessageAttachment)
.where(MessageAttachment.message_id.in_(message_ids))
.order_by(MessageAttachment.position)
)
for link in rows.scalars():
result[link.message_id].append(link.attachment_id)
return result
async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
session.add(UploadIntent(id=id, object_key=key))
await session.flush()
async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
return await session.scalar(
select(UploadIntent).where(UploadIntent.id == id).with_for_update()
)
async def finish_upload(session: AsyncSession, id: UUID) -> None:
await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
async def stale_uploads(
session: AsyncSession, grace_seconds: int, limit: int
) -> list[UploadIntent]:
result = await session.scalars(
select(UploadIntent)
.where(
UploadIntent.created_at
< func.clock_timestamp() - timedelta(seconds=grace_seconds),
)
.order_by(UploadIntent.created_at)
.limit(limit)
.with_for_update(skip_locked=True)
)
return list(result)
+71
View File
@@ -0,0 +1,71 @@
from collections.abc import Sequence
from uuid import UUID
from sqlalchemy import exists, select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.chat import Chat
from app.models.chat_member import ChatMember
from app.storage.types import Position
async def create_chat(
session: AsyncSession,
*,
title: str,
member_ids: Sequence[UUID],
) -> Chat:
"""Добавляет чат вместе с участниками."""
chat = Chat(
title=title,
members=[ChatMember(user_id=user_id) for user_id in member_ids],
)
session.add(chat)
await session.flush()
return chat
async def get_chat_for_member(
session: AsyncSession,
chat_id: UUID,
user_id: UUID,
) -> Chat | None:
"""Возвращает чат, если пользователь в нём состоит, иначе None."""
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
chat = result.scalar_one_or_none()
return chat
async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool:
"""Возвращает True, если пользователь состоит в чате; одна проверка по PK."""
stmt = select(
exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id)
)
result = await session.execute(stmt)
is_member = bool(result.scalar_one())
return is_member
async def list_chats_for_user(
session: AsyncSession,
user_id: UUID,
after: Position | None,
limit: int,
) -> list[Chat]:
"""Возвращает чаты пользователя по (created_at, id) строго после позиции after."""
stmt = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(ChatMember.user_id == user_id)
)
if after is not None:
stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit)
result = await session.execute(stmt)
chats = list(result.scalars().all())
return chats
+15
View File
@@ -0,0 +1,15 @@
"""Простые проверки соединения с общими хранилищами."""
from redis.asyncio import Redis
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
return True
async def check_redis(client: Redis) -> bool:
return bool(await client.ping())
+63
View File
@@ -0,0 +1,63 @@
"""Хранение и блокировка ключей в PostgreSQL."""
from datetime import timedelta
import hashlib
import json
from uuid import UUID
from sqlalchemy import BigInteger, func, literal, select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.idempotency_key import IdempotencyKey
Scope = tuple[UUID, str, str, str]
async def lock_key(session: AsyncSession, scope: Scope) -> None:
"""Сериализует один ключ до конца транзакции."""
raw = json.dumps([str(scope[0]), *scope[1:]]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
await session.execute(
select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger)))
)
async def get_active_key(
session: AsyncSession, scope: Scope, ttl_seconds: int
) -> IdempotencyKey | None:
"""Читает результат, пока срок ключа не истёк."""
stmt = select(IdempotencyKey).where(
tuple_(
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
) == tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
return await session.scalar(stmt)
async def save_result(
session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID
) -> None:
"""Сохраняет новый результат или заменяет истёкший."""
stmt = insert(IdempotencyKey).values(
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
request_hash=request_hash, message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id, IdempotencyKey.method,
IdempotencyKey.path, IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
"message_id": message_id,
"created_at": func.clock_timestamp(),
},
)
)
+62
View File
@@ -0,0 +1,62 @@
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.counter import Counter
from app.models.message import Message
from app.storage.types import Position
MESSAGES_COUNTER = "messages"
async def get_message(session: AsyncSession, message_id: UUID) -> Message | None:
return await session.get(Message, message_id)
async def create_message(
session: AsyncSession,
*,
chat_id: UUID,
sender_id: UUID,
text: str,
) -> Message:
"""Добавляет сообщение; id и created_at назначает сервер."""
message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
session.add(message)
await session.flush()
return message
async def list_messages(
session: AsyncSession,
chat_id: UUID,
after: Position | None,
limit: int,
) -> list[Message]:
"""Возвращает сообщения чата по (created_at, id) строго после позиции after."""
stmt = select(Message).where(Message.chat_id == chat_id)
if after is not None:
stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
stmt = stmt.order_by(Message.created_at, Message.id).limit(limit)
result = await session.execute(stmt)
messages = list(result.scalars().all())
return messages
async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool:
"""Атомарно увеличивает счётчик сообщений, если лимит не исчерпан."""
stmt = (
insert(Counter)
.values(name=MESSAGES_COUNTER, value=1)
.on_conflict_do_update(
index_elements=[Counter.name],
set_={"value": Counter.value + 1},
where=Counter.value < max_messages,
)
.returning(Counter.value)
)
result = await session.execute(stmt)
is_reserved = result.scalar_one_or_none() is not None
return is_reserved
+180
View File
@@ -0,0 +1,180 @@
"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
import asyncio
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import TypeVar
import time
from urllib.parse import urlsplit
from anyio import CancelScope
from minio import Minio
import urllib3
from app.core.s3 import S3Settings
T = TypeVar("T")
async def run_sync(
operation: Callable[..., T], *args, discard_result=None, **kwargs
) -> T:
"""При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
try:
result = await asyncio.shield(task)
except asyncio.CancelledError:
with CancelScope(shield=True):
await settle_cancelled_call(task, discard_result)
raise
return result
async def settle_cancelled_call(
task: asyncio.Task, discard_result: Callable | None
) -> None:
"""wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
while not task.done():
with suppress(asyncio.CancelledError):
await asyncio.wait([task])
completed = not task.cancelled() and task.exception() is None
if completed and discard_result is not None:
discard_result(task.result())
@dataclass(frozen=True)
class ObjectInfo:
"""Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
key: str
size_bytes: int | None
content_type: str | None
last_modified: datetime | None
def close_response(response) -> None:
response.close()
response.release_conn()
class ObjectStream:
"""Закрывается даже если StreamingResponse не успел начать итерацию."""
def __init__(self, storage, response):
self.storage = storage
self.response = response
self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
self.closed = False
def __aiter__(self):
return self
async def __anext__(self):
if self.closed:
raise StopAsyncIteration
try:
remaining = self.deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError
async with asyncio.timeout(remaining):
chunk = await self.storage._call(
lambda: self.response.read(self.storage.settings.chunk_bytes)
)
if not chunk:
await self.aclose()
raise StopAsyncIteration
except BaseException:
await self.aclose()
raise
return chunk
async def aclose(self):
if not self.closed:
self.closed = True
close_response(self.response)
class MinioStorage:
def __init__(self, settings: S3Settings) -> None:
self.settings = settings
# Клиент создаётся в event loop один раз, не конкурентно в worker threads.
url = urlsplit(settings.endpoint)
self._http = urllib3.PoolManager(
timeout=urllib3.Timeout(
connect=settings.connect_timeout_seconds,
read=settings.read_timeout_seconds,
),
retries=urllib3.Retry(total=0),
maxsize=32,
)
self._health_http = urllib3.PoolManager(
timeout=urllib3.Timeout(connect=0.25, read=0.5),
retries=urllib3.Retry(total=0),
)
self._client = Minio(
url.netloc,
secure=url.scheme == "https",
region=settings.region,
access_key=settings.access_key_id_file.read_text().strip(),
secret_key=settings.secret_access_key_file.read_text().strip(),
http_client=self._http,
)
async def _call(self, operation: Callable, *, discard_result=None):
return await run_sync(operation, discard_result=discard_result)
async def put_file(
self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
) -> None:
def put():
with path.open("rb") as source:
# Один PUT: размер части равен максимальному разрешённому файлу.
self._client.put_object(
self.settings.bucket,
key,
source,
size_bytes,
content_type=content_type,
metadata={"sha256": sha256},
part_size=10_485_760,
num_parallel_uploads=1,
)
await self._call(put)
async def open_object(self, key: str) -> ObjectStream:
response = await self._call(
lambda: self._client.get_object(self.settings.bucket, key),
discard_result=close_response,
)
return ObjectStream(self, response)
async def stat_object(self, key: str) -> ObjectInfo:
def stat():
info = self._client.stat_object(self.settings.bucket, key)
return ObjectInfo(key, info.size, info.content_type, info.last_modified)
return await self._call(stat)
async def delete_object(self, key: str) -> None:
await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
async def check_available(self) -> bool:
status = await self._call(
lambda: self._health_http.request(
"GET",
self.settings.endpoint.rstrip("/") + "/minio/health/ready",
).status
)
return status == 200
async def aclose(self) -> None:
self._http.clear()
self._health_http.clear()
def create_s3_storage(settings: S3Settings) -> MinioStorage:
return MinioStorage(settings)
+38
View File
@@ -0,0 +1,38 @@
from datetime import datetime
from uuid import UUID
from redis.asyncio import Redis
SESSION_KEY_PREFIX = "session:"
def session_key(token_hash: str) -> str:
"""Возвращает ключ Redis для хеша токена."""
return f"{SESSION_KEY_PREFIX}{token_hash}"
async def create_session(
redis: Redis,
*,
token_hash: str,
user_id: UUID,
expires_at: datetime,
) -> None:
"""Добавляет сессию с абсолютным временем истечения."""
await redis.set(session_key(token_hash), str(user_id), pxat=expires_at)
async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None:
"""Возвращает id владельца действующей сессии или None."""
raw_user_id = await redis.get(session_key(token_hash))
if isinstance(raw_user_id, bytes):
raw_user_id = raw_user_id.decode()
user_id = UUID(raw_user_id) if raw_user_id else None
return user_id
async def delete_session(redis: Redis, token_hash: str) -> bool:
"""Удаляет сессию; возвращает True, если она существовала."""
deleted_count = await redis.delete(session_key(token_hash))
is_deleted = deleted_count > 0
return is_deleted
+6
View File
@@ -0,0 +1,6 @@
"""Общие типы слоя хранения."""
from datetime import datetime
from uuid import UUID
Position = tuple[datetime, UUID]
+47
View File
@@ -0,0 +1,47 @@
from collections.abc import Collection
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.user import User
async def create_user(
session: AsyncSession,
*,
username: str,
display_name: str,
password_hash: str,
) -> User:
"""Добавляет пользователя; занятый username даёт IntegrityError при flush."""
user = User(
username=username, display_name=display_name, password_hash=password_hash
)
session.add(user)
await session.flush()
return user
async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
"""Возвращает пользователя по id или None."""
user = await session.get(User, user_id)
return user
async def get_user_by_username(session: AsyncSession, username: str) -> User | None:
"""Возвращает пользователя по username или None."""
stmt = select(User).where(User.username == username)
result = await session.execute(stmt)
user = result.scalar_one_or_none()
return user
async def get_existing_user_ids(
session: AsyncSession, user_ids: Collection[UUID]
) -> set[UUID]:
"""Возвращает те из user_ids, что существуют, одним запросом."""
stmt = select(User.id).where(User.id.in_(user_ids))
result = await session.execute(stmt)
existing_ids = set(result.scalars().all())
return existing_ids
+10
View File
@@ -0,0 +1,10 @@
# Override для отдельного тестового стенда (make test-stand-up):
# публикует PostgreSQL и Redis на 127.0.0.1, чтобы pytest из student-tests/
# мог подключаться к ним напрямую. В основном стенде эти порты закрыты.
services:
postgres:
ports:
- "127.0.0.1:${DB_HOST_PORT:?Set DB_HOST_PORT in .env}:5432"
redis:
ports:
- "127.0.0.1:${REDIS_HOST_PORT:?Set REDIS_HOST_PORT in .env}:6379"
+279
View File
@@ -0,0 +1,279 @@
name: mtusi-messenger-lab4
x-api-image: &api-image
build:
context: .
args:
API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env}
image: ${API_IMAGE:?Set API_IMAGE in .env}
x-base-env: &base-env
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
DB_HOST: ${DB_HOST:?Set DB_HOST in .env}
DB_PORT: ${DB_PORT:?Set DB_PORT in .env}
DB_NAME: ${DB_NAME:?Set DB_NAME in .env}
DB_USER: ${DB_USER:?Set DB_USER in .env}
DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env}
DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env}
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env}
SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env}
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env}
READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env}
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_REGION: ${S3_REGION:?Set S3_REGION in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env}
S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env}
S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env}
S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env}
S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env}
S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env}
S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env}
S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env}
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
services:
minio:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: server
image: mtusi-messenger-minio:lab4
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
restart: unless-stopped
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
MINIO_BROWSER: "off"
secrets:
- minio_root_user
- minio_root_password
volumes:
- miniodata:/data
networks: [objects]
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
minio-bootstrap:
build:
context: .
dockerfile: infra/minio/Dockerfile
target: bootstrap
image: mtusi-messenger-minio-bootstrap:lab4
environment:
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
networks: [objects]
restart: "no"
depends_on:
minio:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
postgres:
image: postgres:16.10-bookworm
restart: unless-stopped
environment:
POSTGRES_USER: ${DB_USER:?Set DB_USER in .env}
POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env}
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets:
- db_password
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
redis:
image: redis:7.4.2-bookworm
restart: unless-stopped
entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"]
secrets:
- redis_password
volumes:
- ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
- ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro
- redisdata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"]
interval: 3s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
migrate:
<<: *api-image
command: ["alembic", "upgrade", "head"]
environment: *base-env
secrets:
- db_password
depends_on:
postgres:
condition: service_healthy
restart: "no"
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
api:
<<: *api-image
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env}
REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env}
REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env}
REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env}
REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env}
REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env}
REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env}
secrets:
- db_password
- redis_password
- s3_access_key_id
- s3_secret_access_key
networks: [default, objects]
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
deploy:
replicas: ${API_REPLICAS:?Set API_REPLICAS in .env}
resources:
limits:
cpus: "1.0"
memory: 1536M
healthcheck:
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
interval: 3s
timeout: 3s
start_period: 10s
retries: 2
stop_signal: SIGTERM
stop_grace_period: 8s
depends_on:
minio:
condition: service_healthy
minio-bootstrap:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
cleanup:
<<: *api-image
command: ["python", "-m", "app.core.cleanup", "--apply"]
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
secrets:
- db_password
- s3_access_key_id
- s3_secret_access_key
networks: [default, objects]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
minio-bootstrap:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
nginx:
image: nginx:1.28.0-alpine
restart: unless-stopped
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"]
interval: 5s
timeout: 3s
retries: 5
depends_on:
api:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
volumes:
pgdata:
redisdata:
miniodata:
networks:
default:
objects:
internal: true
secrets:
db_password:
file: ./secrets/db_password
redis_password:
file: ./secrets/redis_password
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+113 -3
View File
@@ -1,5 +1,115 @@
# Доказательства
# Evidence лабораторной 4
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Результаты отдельного полного прогона на чистом проекте
`mtusi-messenger-lab4-test`. Дата, базовый commit, версии, лимиты CPU/RAM,
идентификаторы образов и точные команды находятся в [run.json](run.json).
Проверено незакоммиченное рабочее дерево; его исполняемые файлы зафиксированы
в [source-sha256.txt](source-sha256.txt). Манифест снимается до прогона и
сравнивается после него. Отчёт, README, RUNBOOK, LAB4_PLAN и сами evidence
в манифест не входят; инструкции AGENTS.md/CODE_STYLE.md зафиксированы.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
## Артефакты и скрипты
Все шесть файлов записывает [scripts/collect_evidence.py](../scripts/collect_evidence.py).
Первичные проверки и источники данных указаны ниже.
| Evidence | Что содержит | Откуда берётся |
| --- | --- | --- |
| [test-output.txt](test-output.txt) | Общий вывод проверок и сценариев; `188 passed`, строки `PASS`, результаты отказов. | Команды из `run.json`; конкретные скрипты — в таблице требований ниже. |
| [nginx-output.txt](nginx-output.txt) | Конфигурация, DNS, балансировка, gzip и ошибки Nginx. | [scripts/check_nginx.py](../scripts/check_nginx.py), команда `make test-nginx`. |
| [metrics.json](metrics.json) | Числа pytest, upload, rolling update и отказов API. | `collect_evidence.extract_metrics()` извлекает результаты pytest, `upload_load.py`, `check_grade4.py` и `tests/resilience.py`. |
| [run.json](run.json) | Среда, лимиты, образы, команды/exit codes и снимки контейнеров. | `collect_evidence.py`: Docker info/config/inspect и снимки между сценариями. |
| [cleanup.txt](cleanup.txt) | Удаление тестовых контейнеров, сетей и volumes. | `collect_evidence.py`: `docker compose down --volumes --remove-orphans`. |
| [source-sha256.txt](source-sha256.txt) | Контрольные суммы проверенного кода, конфигураций и тестов. | `collect_evidence.source_manifest()`: SHA-256 до прогона и проверка совпадения после. |
## Соответствие требованиям на 3 и 4
Номера соответствуют порядку пунктов в разделах оценок [задания](../README.md).
| Пункт | Требование | Где доказательство / что смотреть | Проверяющий скрипт |
| --- | --- | --- | --- |
| 3.1 | Сохранён минимум lab3; метаданные в PostgreSQL, байты в приватном S3, общий стенд. | [run.json](run.json): `services`, `container_snapshots`; [test-output.txt](test-output.txt): smoke, идемпотентность, `PASS infrastructure`. | `tests/smoke.py`, `scripts/check_idempotency.py`, `scripts/check_s3.py`, `scripts/check_grade4.py`; pytest `test_migrations.py` и `test_attachments.py`. |
| 3.2 | 201 ready, побайтный оригинал, SHA-256, attachment_ids и доступ участников. | [test-output.txt](test-output.txt): `PASS: lab4 public API scenario`; общий итог pytest. | `tests/smoke.py`, `tests/client.py`; `student-tests/test_attachments.py`. |
| 3.3 | PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому, безопасное имя, ранние лимиты. | [test-output.txt](test-output.txt): `188 passed`; в тестах есть приём ровно на лимитах и отклонение превышений. | `student-tests/test_attachments.py`: `test_exact_*`, `test_oversize_*`, `test_pixel_limit_before_decoding`, `test_invalid_*`, `test_unsafe_*`. |
| 3.4 | Скачивание другой репликой после отказа загрузившей; сохранность после restart. | [test-output.txt](test-output.txt): `PASS: persistence`, `PASS: failover`; [run.json](run.json): `before_restart` / `after_restart`; [metrics.json](metrics.json): `public_failover`. | `tests/resilience.py`; hooks `scripts/restart.sh` и `scripts/stop-one.sh`. |
| 4.1 | Негативные тесты: размер, повреждение, MIME, чужой ID и файл другого чата; предусмотренные 4xx. | [test-output.txt](test-output.txt): `188 passed`; соответствующие тесты включены в полный pytest-прогон. | `student-tests/test_attachments.py`: `test_invalid_*`, `test_oversize_*`, `test_unknown_and_other_chat_attachment_are_404`, `test_upload_download_and_attachment_only_message`. |
| 4.2 | Отмена и сбои S3/БД; компенсация, сверка, безопасный grace period. | [test-output.txt](test-output.txt): `PASS: real S3 PUT/database failure...` и итог pytest; [LAB4_PLAN.md](../LAB4_PLAN.md): механизм согласованности. | `scripts/check_s3.py --with-faults` → `scripts/s3_faults.py`; `student-tests/test_attachments.py`, `test_s3_stream.py`. |
| 4.3 | Ограничения RAM/таймаутов и эксперимент ≥10 одновременных загрузок. | [metrics.json](metrics.json): `uploads` — 3×10 попыток, 12×201, 18×429, память каждой API; [run.json](run.json): лимиты; [test-output.txt](test-output.txt): тесты timeout/отмены. | `scripts/upload_load.py --concurrency 10`; `student-tests/test_attachments.py`, `test_s3_stream.py`; аудит `scripts/check_grade4.py`. |
| 4.4 | Повторяемый bootstrap, отдельные ограниченные credentials, закрытая консоль, private bucket вне API. | [test-output.txt](test-output.txt): `PASS: idempotent bootstrap, anonymous GET/PUT denied...`; [run.json](run.json): порты и назначение секретов. | `scripts/check_s3.py --with-outage --with-faults`, повторно вызывает `infra/minio/bootstrap.py`; `scripts/check_grade4.py` проверяет фактические ресурсы/секреты. |
Pytest запущен с `-q`: в evidence сохранён общий итог, а названия и проверки
отдельных случаев находятся в указанных тестовых файлах. Их точные версии
зафиксированы в `source-sha256.txt`.
Сверка объектов S3 подтверждается сценарием `s3_faults.py` в `test-output.txt`;
`cleanup.txt` фиксирует удаление самого тестового стенда.
## Воспроизведение
Из корня проекта, с Python 3.12 и работающим Docker:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
.venv/bin/python -m scripts.collect_evidence
sha256sum -c evidence/source-sha256.txt
```
Коллектор требует отсутствия прежних ресурсов тестового проекта. Существующие
`.env` и секреты сохраняются. MinIO и mc собираются из закреплённых исходников;
первая сборка требует доступа к Go-модулям. Основной стенд не запускается.
В `finally` выполняется `down --volumes --remove-orphans`; evidence публикуются
только после успешного завершения всех проверок и подтверждения очистки.
## Методика
1. Проверить контракт и OpenAPI; собрать стенд с двумя API, общей PostgreSQL,
Redis, MinIO, bootstrap и cleanup. Pytest использует отдельную БД
`messenger_test` и Redis DB 15; публичные проверки — API через Nginx :18080.
2. Выполнить все собственные тесты с `DOCKER_TESTS=1`, затем публичный smoke и
24 конкурентных повтора одного сообщения через две реплики.
3. Снять состояния контейнеров до и после persistence. Hook перезапускает
PostgreSQL, Redis, MinIO, обе API, Nginx и cleanup, сохраняя volumes.
Публичная проверка повторно читает данные и оригинал с прежним токеном.
В снимках совпадают ID контейнеров и изменились PID/StartedAt всех семи
работающих контейнеров, включая три хранилища.
4. Загрузить файл, остановить обслужившую API-реплику и скачать тот же оригинал
через оставшуюся; проверить SHA-256, подтверждённые сообщения и идемпотентность.
5. Дважды повторить bootstrap. Проверить прямые анонимные GET/PUT в S3, запрет
чтения политики для API credentials, доступ только к своему bucket и
недоступность консоли из сети API. `ListBuckets` MinIO может вернуть
отфильтрованный список разрешённых bucket; это учитывается проверкой.
6. Выполнить реальный PUT и внедрить отказ сохранения метаданных. Проверить
компенсацию; затем внедрить отказ удаления, проверить устойчивый журнал,
сохранность свежего объекта, dry-run и повторную очистку. Для проверки grace
period возраст только тестовой записи изменяется на два дня; cleanup временно
остановлен и затем возвращён в работу. Подтверждённый оригинал сохраняется.
7. Временно остановить MinIO: download даёт 503; после запуска прежний оригинал
снова скачивается побайтно.
8. Выполнить три серии по 10 одновременно начатых upload PNG 1500×1500,
6 761 054 байта, генератор случайных пикселей с seed=42. Активных обработок
не более двух на API; лишние получают 429, повторов upload нет. После опыта
скачать все подтверждённые оригиналы и сравнить SHA-256.
9. Проверить фактические ресурсы/секреты, SIGTERM, обновление обеих API под
записью и последовательный отказ каждой с восстановлением предыдущей.
Новая build-метка создаёт новый образ того же бизнес-кода; миграционная
команда при rolling update выполняется один раз.
10. Выполнить отдельную проверку Nginx и удалить тестовый стенд с volumes.
Память `memory.current` снимается в cgroup каждой API через 20 мс и включает
декодеры, tmpfs и небольшой процесс-монитор. Выборки могут пропустить короткий
пик; дополнительно записан kernel `memory.peak` до/после опыта. Последний
накопительный с момента создания контейнера и может включать предыдущие проверки.
P95 загрузок относится ко всем попыткам, включая 429; скачивание для сверки
SHA-256 выполняется после измерения памяти upload.
RPS в сценариях отказа — успешные HTTP-ответы, включая повторы ключа.
`first_recovery_seconds` включает время hook: это первый завершившийся успешный
POST, начатый после завершения команды остановки. Ошибки 429 ограничителя и
переходные 503 учитываются отдельно; подтверждённые записи проверяются по истории.
Один короткий локальный прогон не определяет предельную производительность.
В артефакты не включаются `.env`, значения credentials, session tokens,
Authorization, тела сообщений и дампы хранилищ. Коллектор отбирает несекретные
параметры, скрывает известные значения секретов и проверяет их отсутствие
перед публикацией файлов.
+48
View File
@@ -0,0 +1,48 @@
$ docker compose --env-file .env down --volumes --remove-orphans
Container mtusi-messenger-lab4-test-nginx-1 Stopping
Container mtusi-messenger-lab4-test-cleanup-1 Stopping
Container mtusi-messenger-lab4-test-nginx-1 Stopped
Container mtusi-messenger-lab4-test-nginx-1 Removing
Container mtusi-messenger-lab4-test-nginx-1 Removed
Container mtusi-messenger-lab4-test-api-3 Stopping
Container mtusi-messenger-lab4-test-api-4 Stopping
Container mtusi-messenger-lab4-test-api-4 Stopped
Container mtusi-messenger-lab4-test-api-4 Removing
Container mtusi-messenger-lab4-test-api-3 Stopped
Container mtusi-messenger-lab4-test-api-3 Removing
Container mtusi-messenger-lab4-test-api-4 Removed
Container mtusi-messenger-lab4-test-api-3 Removed
Container mtusi-messenger-lab4-test-redis-1 Stopping
Container mtusi-messenger-lab4-test-redis-1 Stopped
Container mtusi-messenger-lab4-test-redis-1 Removing
Container mtusi-messenger-lab4-test-redis-1 Removed
Container mtusi-messenger-lab4-test-cleanup-1 Stopped
Container mtusi-messenger-lab4-test-cleanup-1 Removing
Container mtusi-messenger-lab4-test-cleanup-1 Removed
Container mtusi-messenger-lab4-test-migrate-1 Stopping
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Stopping
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Stopped
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Removing
Container mtusi-messenger-lab4-test-migrate-1 Stopped
Container mtusi-messenger-lab4-test-migrate-1 Removing
Container mtusi-messenger-lab4-test-migrate-1 Removed
Container mtusi-messenger-lab4-test-postgres-1 Stopping
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Removed
Container mtusi-messenger-lab4-test-minio-1 Stopping
Container mtusi-messenger-lab4-test-minio-1 Stopped
Container mtusi-messenger-lab4-test-minio-1 Removing
Container mtusi-messenger-lab4-test-minio-1 Removed
Container mtusi-messenger-lab4-test-postgres-1 Stopped
Container mtusi-messenger-lab4-test-postgres-1 Removing
Container mtusi-messenger-lab4-test-postgres-1 Removed
Volume mtusi-messenger-lab4-test_pgdata Removing
Volume mtusi-messenger-lab4-test_miniodata Removing
Volume mtusi-messenger-lab4-test_redisdata Removing
Network mtusi-messenger-lab4-test_objects Removing
Network mtusi-messenger-lab4-test_default Removing
Volume mtusi-messenger-lab4-test_pgdata Removed
Volume mtusi-messenger-lab4-test_miniodata Removed
Volume mtusi-messenger-lab4-test_redisdata Removed
Network mtusi-messenger-lab4-test_default Removed
Network mtusi-messenger-lab4-test_objects Removed
+92
View File
@@ -0,0 +1,92 @@
{
"pytest": {
"passed": 188,
"elapsed_seconds": 73.96
},
"uploads": {
"concurrency": 10,
"rounds": 3,
"attempts": 30,
"fixture": {
"format": "PNG",
"width": 1500,
"height": 1500,
"size_bytes": 6761054,
"sha256": "49cb1a9ec3e54c1f22c649fb787c50186c54e449ade4a6f6758a50b2e374504d"
},
"statuses": {
"201": 12,
"429": 18
},
"elapsed_seconds": 5.138,
"successful_uploads": 12,
"p95_seconds": 2.143,
"memory": {
"bf64a4a13acb": {
"initial_bytes": 88752128,
"sampled_peak_bytes": 162426880,
"cgroup_peak_before_bytes": 161976320,
"cgroup_peak_after_bytes": 162689024,
"interval_ms": 20
},
"7e4180e529e1": {
"initial_bytes": 89436160,
"sampled_peak_bytes": 163692544,
"cgroup_peak_before_bytes": 157728768,
"cgroup_peak_after_bytes": 163692544,
"interval_ms": 20
}
},
"api_memory_limit": "1610612736",
"notes": "20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries"
},
"public_failover": [
{
"observed_instances_before": 2,
"successful_writes": 112,
"failed_attempts": 4,
"first_recovery_seconds": 1.885
}
],
"rolling_update": [
{
"scenario": "rolling update",
"replacements": 2,
"migration_runs": 1,
"successful_writes": 487,
"failed_attempts": 13,
"errors_by_kind": {
"http_503": 13
},
"elapsed_seconds": 25.341,
"successful_rps": 19.218,
"p95_seconds": 0.048
}
],
"replica_failures": [
{
"instance": "429367307e5f",
"successful_writes": 777,
"failed_attempts": 87,
"errors_by_kind": {
"http_429": 70,
"http_503": 17
},
"elapsed_seconds": 13.965,
"successful_rps": 55.64,
"first_recovery_seconds": 1.865
},
{
"instance": "20b4b219e7c8",
"successful_writes": 774,
"failed_attempts": 96,
"errors_by_kind": {
"http_429": 79,
"http_503": 17
},
"elapsed_seconds": 14.127,
"successful_rps": 54.79,
"first_recovery_seconds": 2.042
}
]
}
+13
View File
@@ -0,0 +1,13 @@
$ make test-nginx
.venv/bin/python scripts/check_nginx.py
nginx version: nginx/1.28.0
PASS: nginx -t и балансировка двух реплик
PASS: gzip 2060 → 61 байт, содержимое сохранено
PASS: JSON и заголовки ошибки API передаются без подмены
PASS: отправленный POST не повторяется на другой реплике
PASS: лимит вернул 390 ответов 429; health доступен
PASS: третья реплика обнаружена через DNS без reload nginx
PASS: отказ реплики, первое успешное чтение через 0.719 с
PASS: недоступность всех backend возвращает JSON 503
PASS: JSON access log содержит время backend и результат лимита
+888
View File
@@ -0,0 +1,888 @@
{
"started_at_utc": "2026-11-04T18:14:58.493240+00:00",
"finished_at_utc": "2026-11-04T18:20:25.766354+00:00",
"started_at_moscow": "2026-11-04T21:14:58.493240+03:00",
"base_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5",
"worktree_dirty": true,
"source_manifest_sha256": "f3c995d33b82b1996d497b106fb672792f5ad0caf11b5391fd0ef5c89d86ca45",
"project": "mtusi-messenger-lab4-test",
"runtime": {
"python": "3.12.14",
"packages": {
"fastapi": "0.143.3",
"SQLAlchemy": "2.1.4",
"pydantic": "2.14.1",
"minio": "7.2.20",
"Pillow": "13.0.0",
"pytest": "9.1.1"
},
"compose": "Docker Compose version 5.6.1"
},
"docker_host": {
"ServerVersion": "29.9.2",
"OperatingSystem": "Arch Linux",
"Architecture": "x86_64",
"NCPU": 8,
"MemTotal": 16400363520,
"CgroupVersion": "2"
},
"services": {
"api": {
"image": "mtusi-messenger:lab4",
"limits": {
"cpus": 1,
"memory": "1610612736"
},
"settings": {
"APP_PORT": "8080",
"DB_MAX_OVERFLOW": "10",
"DB_POOL_SIZE": "5",
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
"IDEMPOTENCY_TTL_SECONDS": "86400",
"MAX_IN_FLIGHT": "16",
"MAX_MESSAGES": "100000",
"READINESS_TIMEOUT_SECONDS": "2",
"S3_BUCKET": "messenger-attachments",
"S3_CHUNK_BYTES": "65536",
"S3_CLEANUP_GRACE_SECONDS": "86400",
"S3_CLEANUP_INTERVAL_SECONDS": "300",
"S3_CONNECT_TIMEOUT_SECONDS": "2",
"S3_DOWNLOAD_TIMEOUT_SECONDS": "60",
"S3_ENDPOINT": "http://minio:9000",
"S3_IMAGE_TIMEOUT_SECONDS": "10",
"S3_MAX_CONCURRENT_UPLOADS": "2",
"S3_MAX_IMAGE_PIXELS": "40000000",
"S3_MAX_UPLOAD_BYTES": "10485760",
"S3_READ_TIMEOUT_SECONDS": "10",
"S3_REGION": "us-east-1",
"S3_UPLOAD_TIMEOUT_SECONDS": "30",
"SESSION_TTL_SECONDS": "3600",
"SHUTDOWN_DRAIN_SECONDS": "1"
},
"ports": [],
"networks": [
"default",
"objects"
],
"secret_names": [
"db_password",
"redis_password",
"s3_access_key_id",
"s3_secret_access_key"
],
"tmpfs": [
"/tmp:rw,noexec,nosuid,size=64m,mode=1777"
]
},
"cleanup": {
"image": "mtusi-messenger:lab4",
"limits": {
"cpus": 0.25,
"memory": "268435456"
},
"settings": {
"APP_PORT": "8080",
"DB_MAX_OVERFLOW": "10",
"DB_POOL_SIZE": "5",
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
"MAX_IN_FLIGHT": "16",
"MAX_MESSAGES": "100000",
"READINESS_TIMEOUT_SECONDS": "2",
"S3_BUCKET": "messenger-attachments",
"S3_CHUNK_BYTES": "65536",
"S3_CLEANUP_GRACE_SECONDS": "86400",
"S3_CLEANUP_INTERVAL_SECONDS": "300",
"S3_CONNECT_TIMEOUT_SECONDS": "2",
"S3_DOWNLOAD_TIMEOUT_SECONDS": "60",
"S3_ENDPOINT": "http://minio:9000",
"S3_IMAGE_TIMEOUT_SECONDS": "10",
"S3_MAX_CONCURRENT_UPLOADS": "2",
"S3_MAX_IMAGE_PIXELS": "40000000",
"S3_MAX_UPLOAD_BYTES": "10485760",
"S3_READ_TIMEOUT_SECONDS": "10",
"S3_REGION": "us-east-1",
"S3_UPLOAD_TIMEOUT_SECONDS": "30",
"SHUTDOWN_DRAIN_SECONDS": "1"
},
"ports": [],
"networks": [
"default",
"objects"
],
"secret_names": [
"db_password",
"s3_access_key_id",
"s3_secret_access_key"
],
"tmpfs": []
},
"migrate": {
"image": "mtusi-messenger:lab4",
"limits": {
"cpus": 0.5,
"memory": "268435456"
},
"settings": {
"APP_PORT": "8080",
"DB_MAX_OVERFLOW": "10",
"DB_POOL_SIZE": "5",
"GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS": "5",
"MAX_IN_FLIGHT": "16",
"MAX_MESSAGES": "100000",
"READINESS_TIMEOUT_SECONDS": "2",
"SHUTDOWN_DRAIN_SECONDS": "1"
},
"ports": [],
"networks": [
"default"
],
"secret_names": [
"db_password"
],
"tmpfs": []
},
"minio": {
"image": "mtusi-messenger-minio:lab4",
"limits": {
"cpus": 1,
"memory": "536870912"
},
"settings": {},
"ports": [],
"networks": [
"objects"
],
"secret_names": [
"minio_root_user",
"minio_root_password"
],
"tmpfs": []
},
"minio-bootstrap": {
"image": "mtusi-messenger-minio-bootstrap:lab4",
"limits": {
"cpus": 0.25,
"memory": "134217728"
},
"settings": {
"S3_BUCKET": "messenger-attachments",
"S3_ENDPOINT": "http://minio:9000"
},
"ports": [],
"networks": [
"objects"
],
"secret_names": [
"minio_root_user",
"minio_root_password",
"s3_access_key_id",
"s3_secret_access_key"
],
"tmpfs": []
},
"nginx": {
"image": "nginx:1.28.0-alpine",
"limits": {
"cpus": 0.5,
"memory": "134217728"
},
"settings": {},
"ports": [
{
"mode": "ingress",
"host_ip": "127.0.0.1",
"target": 8080,
"published": "18080",
"protocol": "tcp"
}
],
"networks": [
"default"
],
"secret_names": [],
"tmpfs": []
},
"postgres": {
"image": "postgres:16.10-bookworm",
"limits": {
"cpus": 1,
"memory": "536870912"
},
"settings": {},
"ports": [
{
"mode": "ingress",
"host_ip": "127.0.0.1",
"target": 5432,
"published": "15432",
"protocol": "tcp"
}
],
"networks": [
"default"
],
"secret_names": [
"db_password"
],
"tmpfs": []
},
"redis": {
"image": "redis:7.4.2-bookworm",
"limits": {
"cpus": 0.5,
"memory": "536870912"
},
"settings": {},
"ports": [
{
"mode": "ingress",
"host_ip": "127.0.0.1",
"target": 6379,
"published": "16379",
"protocol": "tcp"
}
],
"networks": [
"default"
],
"secret_names": [
"redis_password"
],
"tmpfs": []
}
},
"container_snapshots": {
"started": [
{
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
"name": "/mtusi-messenger-lab4-test-nginx-1",
"service": "nginx",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"status": "running",
"started_at": "2026-11-04T18:15:21.942730957Z",
"pid": 483246,
"exit_code": 0,
"oom_killed": false
},
{
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
"name": "/mtusi-messenger-lab4-test-api-2",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:16.003299195Z",
"pid": 482834,
"exit_code": 0,
"oom_killed": false
},
{
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
"name": "/mtusi-messenger-lab4-test-api-1",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:16.212298098Z",
"pid": 482896,
"exit_code": 0,
"oom_killed": false
},
{
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
"name": "/mtusi-messenger-lab4-test-cleanup-1",
"service": "cleanup",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:15.695719979Z",
"pid": 482774,
"exit_code": 0,
"oom_killed": false
},
{
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
"service": "minio-bootstrap",
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"status": "exited",
"started_at": "2026-11-04T18:15:08.266711675Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
"name": "/mtusi-messenger-lab4-test-migrate-1",
"service": "migrate",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "exited",
"started_at": "2026-11-04T18:15:11.433538933Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
"name": "/mtusi-messenger-lab4-test-minio-1",
"service": "minio",
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"status": "running",
"started_at": "2026-11-04T18:15:04.59129875Z",
"pid": 481985,
"exit_code": 0,
"oom_killed": false
},
{
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
"name": "/mtusi-messenger-lab4-test-postgres-1",
"service": "postgres",
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"status": "running",
"started_at": "2026-11-04T18:15:04.743334637Z",
"pid": 482062,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
"name": "/mtusi-messenger-lab4-test-redis-1",
"service": "redis",
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"status": "running",
"started_at": "2026-11-04T18:15:04.922389241Z",
"pid": 482171,
"exit_code": 0,
"oom_killed": false
}
],
"before_restart": [
{
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
"name": "/mtusi-messenger-lab4-test-nginx-1",
"service": "nginx",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"status": "running",
"started_at": "2026-11-04T18:15:21.942730957Z",
"pid": 483246,
"exit_code": 0,
"oom_killed": false
},
{
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
"name": "/mtusi-messenger-lab4-test-api-2",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:16.003299195Z",
"pid": 482834,
"exit_code": 0,
"oom_killed": false
},
{
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
"name": "/mtusi-messenger-lab4-test-api-1",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:16.212298098Z",
"pid": 482896,
"exit_code": 0,
"oom_killed": false
},
{
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
"name": "/mtusi-messenger-lab4-test-cleanup-1",
"service": "cleanup",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:15:15.695719979Z",
"pid": 482774,
"exit_code": 0,
"oom_killed": false
},
{
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
"service": "minio-bootstrap",
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"status": "exited",
"started_at": "2026-11-04T18:15:08.266711675Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
"name": "/mtusi-messenger-lab4-test-migrate-1",
"service": "migrate",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "exited",
"started_at": "2026-11-04T18:15:11.433538933Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
"name": "/mtusi-messenger-lab4-test-minio-1",
"service": "minio",
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"status": "running",
"started_at": "2026-11-04T18:15:04.59129875Z",
"pid": 481985,
"exit_code": 0,
"oom_killed": false
},
{
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
"name": "/mtusi-messenger-lab4-test-postgres-1",
"service": "postgres",
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"status": "running",
"started_at": "2026-11-04T18:15:04.743334637Z",
"pid": 482062,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
"name": "/mtusi-messenger-lab4-test-redis-1",
"service": "redis",
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"status": "running",
"started_at": "2026-11-04T18:16:35.199893635Z",
"pid": 508916,
"exit_code": 0,
"oom_killed": false
}
],
"after_restart": [
{
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
"name": "/mtusi-messenger-lab4-test-nginx-1",
"service": "nginx",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"status": "running",
"started_at": "2026-11-04T18:17:10.035280757Z",
"pid": 513054,
"exit_code": 0,
"oom_killed": false
},
{
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
"name": "/mtusi-messenger-lab4-test-api-2",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:03.796170548Z",
"pid": 512362,
"exit_code": 0,
"oom_killed": false
},
{
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
"name": "/mtusi-messenger-lab4-test-api-1",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:03.782576826Z",
"pid": 512342,
"exit_code": 0,
"oom_killed": false
},
{
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
"name": "/mtusi-messenger-lab4-test-cleanup-1",
"service": "cleanup",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:20.411799591Z",
"pid": 514062,
"exit_code": 0,
"oom_killed": false
},
{
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
"service": "minio-bootstrap",
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"status": "exited",
"started_at": "2026-11-04T18:17:16.679335904Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
"name": "/mtusi-messenger-lab4-test-migrate-1",
"service": "migrate",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "exited",
"started_at": "2026-11-04T18:17:16.521163474Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
"name": "/mtusi-messenger-lab4-test-minio-1",
"service": "minio",
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"status": "running",
"started_at": "2026-11-04T18:16:56.858209997Z",
"pid": 511450,
"exit_code": 0,
"oom_killed": false
},
{
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
"name": "/mtusi-messenger-lab4-test-postgres-1",
"service": "postgres",
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"status": "running",
"started_at": "2026-11-04T18:16:58.236126757Z",
"pid": 511695,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
"name": "/mtusi-messenger-lab4-test-redis-1",
"service": "redis",
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"status": "running",
"started_at": "2026-11-04T18:16:56.95034238Z",
"pid": 511521,
"exit_code": 0,
"oom_killed": false
}
],
"after_failover": [
{
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
"name": "/mtusi-messenger-lab4-test-nginx-1",
"service": "nginx",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"status": "running",
"started_at": "2026-11-04T18:17:10.035280757Z",
"pid": 513054,
"exit_code": 0,
"oom_killed": false
},
{
"id": "7e4180e529e1af96093cf30b826bb3686d669b469a2e7da5e0995597033cf77a",
"name": "/mtusi-messenger-lab4-test-api-2",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:43.290395636Z",
"pid": 515682,
"exit_code": 0,
"oom_killed": false
},
{
"id": "bf64a4a13acbc3a3282a858fc6e91182d5c5d970813700e09a38fed17ca5035c",
"name": "/mtusi-messenger-lab4-test-api-1",
"service": "api",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:03.782576826Z",
"pid": 512342,
"exit_code": 0,
"oom_killed": false
},
{
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
"name": "/mtusi-messenger-lab4-test-cleanup-1",
"service": "cleanup",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:17:20.411799591Z",
"pid": 514062,
"exit_code": 0,
"oom_killed": false
},
{
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
"service": "minio-bootstrap",
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"status": "exited",
"started_at": "2026-11-04T18:17:39.599833122Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
"name": "/mtusi-messenger-lab4-test-migrate-1",
"service": "migrate",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "exited",
"started_at": "2026-11-04T18:17:39.442058694Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
"name": "/mtusi-messenger-lab4-test-minio-1",
"service": "minio",
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"status": "running",
"started_at": "2026-11-04T18:16:56.858209997Z",
"pid": 511450,
"exit_code": 0,
"oom_killed": false
},
{
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
"name": "/mtusi-messenger-lab4-test-postgres-1",
"service": "postgres",
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"status": "running",
"started_at": "2026-11-04T18:16:58.236126757Z",
"pid": 511695,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
"name": "/mtusi-messenger-lab4-test-redis-1",
"service": "redis",
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"status": "running",
"started_at": "2026-11-04T18:16:56.95034238Z",
"pid": 511521,
"exit_code": 0,
"oom_killed": false
}
],
"after_update_and_failures": [
{
"id": "20b4b219e7c8cea5bae4a8834cc5df68399bc306b005e57509b060aac28cbd89",
"name": "/mtusi-messenger-lab4-test-api-4",
"service": "api",
"image_id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
"status": "running",
"started_at": "2026-11-04T18:19:51.393481452Z",
"pid": 528706,
"exit_code": 0,
"oom_killed": false
},
{
"id": "429367307e5f8f1f9e1ef20f378f308849cf51cddfdf2f8b83ef02d915d26455",
"name": "/mtusi-messenger-lab4-test-api-3",
"service": "api",
"image_id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
"status": "running",
"started_at": "2026-11-04T18:19:30.856008463Z",
"pid": 526806,
"exit_code": 0,
"oom_killed": false
},
{
"id": "1529d6364c6b6b1723ec39e3ff19b62546ce3ee1ad6c5c5a7291b201ca9e28c5",
"name": "/mtusi-messenger-lab4-test-nginx-1",
"service": "nginx",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"status": "running",
"started_at": "2026-11-04T18:17:10.035280757Z",
"pid": 513054,
"exit_code": 0,
"oom_killed": false
},
{
"id": "3f22d53254f2de954440861c58a8e3c2df8a8e94a5d295cb0d1d1eae9231cd66",
"name": "/mtusi-messenger-lab4-test-cleanup-1",
"service": "cleanup",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "running",
"started_at": "2026-11-04T18:18:21.285437286Z",
"pid": 518374,
"exit_code": 0,
"oom_killed": false
},
{
"id": "9eabadbeba78bb9996ab265dc658abedb53d69768b8dc20f0135eb05eee8b00d",
"name": "/mtusi-messenger-lab4-test-minio-bootstrap-1",
"service": "minio-bootstrap",
"image_id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"status": "exited",
"started_at": "2026-11-04T18:18:17.550878221Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "95c90f5a182c8efe761f73597e71a0ceb2cc8e6b27be965479614cf7eb1117f0",
"name": "/mtusi-messenger-lab4-test-migrate-1",
"service": "migrate",
"image_id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"status": "exited",
"started_at": "2026-11-04T18:18:17.394179901Z",
"pid": 0,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0b3cbe2a85dd2f318c47dfc1f6d810c689eadac53bdf1e4db6de7c09d2d25d30",
"name": "/mtusi-messenger-lab4-test-minio-1",
"service": "minio",
"image_id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"status": "running",
"started_at": "2026-11-04T18:18:22.354522515Z",
"pid": 518578,
"exit_code": 0,
"oom_killed": false
},
{
"id": "a66592d1e4e53bcdcfb68b7bf1bd95ece69aa35c8d6d947033d094e130814f9e",
"name": "/mtusi-messenger-lab4-test-postgres-1",
"service": "postgres",
"image_id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"status": "running",
"started_at": "2026-11-04T18:16:58.236126757Z",
"pid": 511695,
"exit_code": 0,
"oom_killed": false
},
{
"id": "0c11e544554910b289df958c872ba838b22b3628bf0cc54c57c81df9411bbf48",
"name": "/mtusi-messenger-lab4-test-redis-1",
"service": "redis",
"image_id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"status": "running",
"started_at": "2026-11-04T18:16:56.95034238Z",
"pid": 511521,
"exit_code": 0,
"oom_killed": false
}
]
},
"images": {
"sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284": {
"id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"repo_digests": [
"nginx@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051": {
"id": "sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051",
"repo_digests": [
"mtusi-messenger@sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9": {
"id": "sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9",
"repo_digests": [
"mtusi-messenger-minio-bootstrap@sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39": {
"id": "sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39",
"repo_digests": [
"mtusi-messenger-minio@sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74": {
"id": "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74",
"repo_digests": [
"postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9": {
"id": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9",
"repo_digests": [
"redis@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9"
],
"architecture": "amd64",
"os": "linux"
},
"sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb": {
"id": "sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb",
"repo_digests": [
"mtusi-messenger@sha256:5a1c0517ac1e3d283b140ada91994c3b058ec00b1671ee55f7cda13af8c9f6eb"
],
"architecture": "amd64",
"os": "linux"
}
},
"commands": [
{
"command": "make check validate test-stand-up TEST_PROJECT=mtusi-messenger-lab4-test",
"started_at_utc": "2026-11-04T18:14:58.640914+00:00",
"finished_at_utc": "2026-11-04T18:15:23.342961+00:00",
"exit_code": 0
},
{
"command": "make test-docker TEST_PROJECT=mtusi-messenger-lab4-test PYTEST_ARGS=--tb=short",
"started_at_utc": "2026-11-04T18:15:23.691655+00:00",
"finished_at_utc": "2026-11-04T18:16:39.997612+00:00",
"exit_code": 0
},
{
"command": "make test-stand-smoke test-stand-idempotency TEST_PROJECT=mtusi-messenger-lab4-test",
"started_at_utc": "2026-11-04T18:16:39.997664+00:00",
"finished_at_utc": "2026-11-04T18:16:44.094964+00:00",
"exit_code": 0
},
{
"command": "make test-stand-persistence TEST_PROJECT=mtusi-messenger-lab4-test",
"started_at_utc": "2026-11-04T18:16:44.152939+00:00",
"finished_at_utc": "2026-11-04T18:17:20.879999+00:00",
"exit_code": 0
},
{
"command": "make test-stand-failover TEST_PROJECT=mtusi-messenger-lab4-test",
"started_at_utc": "2026-11-04T18:17:20.936053+00:00",
"finished_at_utc": "2026-11-04T18:17:48.976224+00:00",
"exit_code": 0
},
{
"command": ".venv/bin/python -m scripts.check_s3 --base-url http://localhost:18080 --with-outage --with-faults",
"started_at_utc": "2026-11-04T18:17:49.039051+00:00",
"finished_at_utc": "2026-11-04T18:18:28.077697+00:00",
"exit_code": 0
},
{
"command": ".venv/bin/python -m scripts.upload_load --base-url http://localhost:18080 --concurrency 10 --output <temporary>/uploads.json",
"started_at_utc": "2026-11-04T18:18:28.077774+00:00",
"finished_at_utc": "2026-11-04T18:18:35.361777+00:00",
"exit_code": 0
},
{
"command": ".venv/bin/python -m scripts.check_grade4 --base-url http://localhost:18080",
"started_at_utc": "2026-11-04T18:18:35.361808+00:00",
"finished_at_utc": "2026-11-04T18:19:58.301508+00:00",
"exit_code": 0
},
{
"command": "make test-nginx",
"started_at_utc": "2026-11-04T18:19:58.404365+00:00",
"finished_at_utc": "2026-11-04T18:20:14.687715+00:00",
"exit_code": 0
},
{
"command": "docker compose --env-file .env down --volumes --remove-orphans",
"started_at_utc": "2026-11-04T18:20:14.687746+00:00",
"finished_at_utc": "2026-11-04T18:20:25.705736+00:00",
"exit_code": 0
}
],
"remaining_test_resources": {
"container": [],
"network": [],
"volume": []
},
"minio_source_release": "RELEASE.2025-10-15T17-29-55Z",
"mc_source_release": "RELEASE.2025-08-13T08-35-41Z"
}
+139
View File
@@ -0,0 +1,139 @@
62c0534ad74d9f0e2982e97d1fbadb66d80c493ed237df5bcb9fbb797062eb38 .dockerignore
0e37adcffea6cf3e485e8df17f64299f45aff55bdd35c8cb600aafe2ea071279 .env.example
e10785802b87474192d20e23a31f2ff10ec24eb336642b2743ad341d1afa21be AGENTS.md
cf48b9f6dd23f288d3064e22bb755c490e076a2b6cfc034fc6c7b87c1185ade1 CODE_STYLE.md
2ea849842e877eca40e5dd853a5299a81be15fa414476fbfe434ac487f59ec1f Dockerfile
27cde38208eed865ddd605d902b409850d5e9fb8e2cbd2f15f151533247592dd Makefile
274785e9cfb1af978fcdf254f3b9c93847216d9183cf290098f857f174f43b41 alembic.ini
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/__init__.py
5839d94c506f0362278a4f85bca8346774f5f323112963dd3830e525a069fb61 app/__main__.py
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/core/__init__.py
7eaffea8127600b33d95bb1ec9171cddfca7d9464db7d49cecbc735b61d3f4d8 app/core/cleanup.py
2a4a236bc37d2806a072d216adf9d7884a4eb39e7626f681206a04e64f4a3493 app/core/config.py
0f37965b0c49f2ae6a2f17262057c19cb0c27c82dbf56b72e2fba6a025ee6d98 app/core/database.py
02326fe518487718ae30334a5f4ebd3a96b63b2222cf112ac4fce33d401e613b app/core/image_worker.py
da4dcfda3dabeb595aa7b917d321e794074676fd258828c84be4322b917ac43c app/core/instance.py
aedcaa71c991f0147603bad32bbd105ebdc689cfc388d274e83ce693f32ba534 app/core/lifecycle.py
5b6585c038289868e98c4c0de6d3004a74f0190c19f6f8b78beca116c9a799ca app/core/redis.py
290d97e3551e1661feb04debc0272c9c91fe7636d1771de9e78055c64e23933c app/core/request_capacity.py
e1f3f08c65cf17ea24574682050e0435d5a5c22b7b2194ea40aad941716871b4 app/core/request_id.py
e519cdff5893f9c371f7756d96a5d92697dc01909e385b091652a541eee7b837 app/core/s3.py
7717e9ea31415dd9803f14e6290e709bfb63aa38cb1524db9b04881cf2512256 app/core/security.py
6acb951afe3a628d9bd9ae25ab8c321b8debb0f481c745881a7af1f48e620c90 app/domain/__init__.py
92276fe889ea7e6901462d6acbecb481b33063723746a0c39ac2da56bc01ee4f app/domain/attachments.py
d7026b518eb7f3f054a5aaa5307171db031718c0c70555d40c03e7aabc9f539d app/domain/auth.py
33f20c63c3fdcb491adb2153a147e822e0a16de4a5f634355a6d2db514aaa668 app/domain/chats.py
31138696454b4cb8e8e81249a54c054ed7dfa05f389b6d097fe99cbf0ef5e5d8 app/domain/health.py
96e5d43321bf8f518992baf64d5a497ada19733c57ff7c16b36e8e34db0a3e07 app/domain/idempotency.py
c93660a0578dcea83fa2fd04cff287833284c1efd555803736273c4298c5b873 app/domain/image_validation.py
63409c613751b0fab06d00587fae2209609a4fe85ad1ba6b2a7fe460715aacef app/domain/images.py
beca98c31471df3b3fdd0c228c6fc6cb70410a81dd2e0f046c35cfcc0a144e68 app/domain/messages.py
df9aa94fab247467fe5f6fa030bdf3e587b5749060f0c3d663f5bd3e57e8dfe9 app/domain/pagination.py
31ec7ba4c47ae6c31661eb8b4231efe9f561784c3bc0555c7d9b9bde804fe88f app/domain/users.py
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/endpoints/__init__.py
6f7dc5e3aabf9e26d29dba89ad15e573679a665711ffef96b2b30e7a7618222c app/endpoints/attachments.py
0cb1dded40038b80eafabcb288f43b1b7fcac1d8d41898e657eb9b62afaa2aab app/endpoints/auth.py
2fc7d2b4f155e1cdba6d02ac157b1c2d74932c930d6cba7ac93c0a811af1494a app/endpoints/chats.py
86f2c2e9b18004b82e099034d0754b27395381a98f9a9804546ffaf5948742f1 app/endpoints/dependencies.py
b26a15a170624acffff0b506031cfffdd9e0e45f42ebb20dad1edd0fc90c73fe app/endpoints/health.py
a6424660ce901defac67b5d66a882f8051a531328fc6d4aa998c407ca276c4e3 app/endpoints/messages.py
13fe05bec4b4b0a9e7a2f90756065bf97e257601b119a71465b24fc0144f3bf9 app/endpoints/users.py
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 app/errors/__init__.py
70febdabcf684aba98a7106dd312cdb59f9f0ac0b494a91277ea2d979888a1e4 app/errors/exceptions.py
40115a09d9e69ddcc8c38afd6f1c60383ce753f46bc5be62e9cd60993271fa71 app/errors/handlers.py
a36a2a1b982a5f8ca8033bd6fa0286f0a4fe0ed4b9bc6c217eae01d9d8070953 app/main.py
5154a347cb6588477b17935f24b61a5122f1952f4238dfcd762e54f430ac9530 app/models/__init__.py
a95f02dbd8f5aa54a9dafb3cb516fa0cbb9b1f5c75ef15ef57a167ad56ea2c57 app/models/attachment.py
548f10800483dd91a4d321c3cef845f3f5b126169a50474bf846f78839fdb43e app/models/base.py
7076f06111aca1e3845a8515112fa1302da9bd331cf79c699259c91d3d23c9cf app/models/chat.py
dfb64d43bd519d06f09e479eab4bdd277f317fdb92e206fdedf0afbed1fe18ad app/models/chat_member.py
35eaa453501b48d178d7b17fd22314783386b64951cbb518b450698ba65219f8 app/models/counter.py
693039c889d92ebdb5ef3fc6833278f59b4aa88f802b4726b9c3bf441f83fb18 app/models/idempotency_key.py
9bbf725b6eae51e23e4fc71aea3dbc137d5a275b069e7e378351773297a7559f app/models/message.py
b5f1bbdc4370e2671427c8d7590aba58916b1981f225c4734d8bdeb944b1539a app/models/user.py
7c9d4ef216ed8b027ddfbad9746147cf0fed6463baf396b7244bd9b1b286e9ee app/schemas/__init__.py
bd00b00ded1c48458b7f3e1ebb51bfb481268de92599d41781cd71e782023b0f app/schemas/attachment.py
9a3f14dc23e704d9fdce681ae7187a6b95638f9736711a8edef245fbbaea3eb6 app/schemas/base.py
2111aef9c942b34d02afa99da6d23a1d5ca0f6cfd75a1c0ebfb2ba85a29584b1 app/schemas/chat.py
1092519716f0c8e5800ccf666c3fe52e8b1e617bee61843d6677fafe83d63d9a app/schemas/message.py
617bcba33dcb01e4f66a50b5c425ebcbc1aef1c6148148f0eb862a7743bcf9b6 app/schemas/session.py
01fc238693c3686595cc910fadce6200e9753d0c4a0397ff327d8a0c937dc4df app/schemas/user.py
33296ca85924a22644fda573be072e5b810a284b939469580445925a85f0b5e9 app/storage/__init__.py
59606da77e97e883fd2bdbc4c1ab008ab4165ba545c72c623777e03936fd9805 app/storage/attachments.py
e479c8be1f56030fc4faaa74417fb959fb2c1f0c57f72974df8b538ff1209a7b app/storage/chats.py
a8cb92ff2fc5de87eb072c9997b1dc33e4c490057c6a6eb8f46d6d1067771345 app/storage/health.py
2a9243ad419e7cc48ce0c6da13053df6773bea53a2342eb408cb3ebe2455f921 app/storage/idempotency.py
e4b71490655095d4dc9b1fccb04820d7cfaf48295affe05d6c6e1e085c8cbaf0 app/storage/messages.py
bec371be921841e7d194f4874c43363a871496299f4f5da61a16c5e0211051fc app/storage/s3.py
aeb6851a54d71051912ad588c47de3d5a3d92d518a0a198dbb3e283dd64837c7 app/storage/sessions.py
46a0562d557ac96bd49f9975ff25ac8cce7f7f803b8c280d1d0bf2463ebfdaec app/storage/types.py
f9dc7df7affda7144991944ff20a1ae5b372921b14fd391afa189d0e613dc45e app/storage/users.py
46f25ac85ff2a20a52cddf67b38c69f340bf007a355cd2e18c46e6aab9d22757 compose.test.yaml
df8b4d1347466f0aa95a9183e04a8496e29614652229cb3dfd0b78a48be13500 compose.yaml
381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37 contracts/examples.json
baaeb5a2233096a50d791f521bfc6dd66a23e516f81833c2b17c554cc0bab5d8 contracts/infrastructure.json
c5c0ebdcf205018126c5bb91e2512b11f543827e76baa21d776210d05a0b2cca contracts/openapi.json
4b492cfedc7d65eb88d8217eefb07db541926ae919770e0d535e767d73fa7947 infra/minio/Dockerfile
cfdd8697694458869202d1536edda66c6c376c6d9757720d6b38ac91526ee4bb infra/minio/bootstrap.py
8484d85c7d7ad6c7e282300288f614b47a2ff5329289e38dd9d111f0c8e76584 infra/nginx/nginx.conf
9d4e7faeab76340dadda9916430048c32c849fb2bf02206653ea827e3304b666 infra/redis/redis.conf
0f6fe080f5f7cd1d77da5f8ee9989874e9617bd7f4830f90b91978a5bfb88056 infra/redis/start-redis.sh
7d0945d93e302c34c3f26455b152f9d657110ee0a94a5074d85963bc7aa60526 lab.json
37dce514d48aa4e021e063f6ebb8fb3e34275f6a76f51926d789802cad8dfa75 migrations/env.py
7751187956408f7a64742e0bf746d0684d24013bfe11d1895d936d6757da1766 migrations/script.py.mako
4ae6aee859bcbcdb238d4cb2fa35419503d7a1d5a3caaffd0f0d2e9e9dce2937 migrations/versions/49d2c947a3f1_add_idempotency_keys.py
0eb1a4e624c0e81a8955245a0071a9e0792722ee0d056df257ddc23062ccc8aa migrations/versions/8f31b94a2d10_add_attachments.py
cbe0b9e7584f26c0b859936a2584c147fe3f1ffaa9eb55c85ff753ecb10aa5a4 migrations/versions/abd273076068_add_counters.py
dd0e8073817e3a08bae19b17d4d502aa0dca0639f95d0b08caaf0e5dce16f302 migrations/versions/db06fd3126ff_create_initial_schema.py
92ce287c8eb0aa508e0efd4553650b4ebd2474d1987587d6467c26e316e6a547 pytest.ini
4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525 requirements-dev.txt
98fab4585e4ac85841ee93836c9bcacbd09627e278cd54766ba0909a43952191 requirements.txt
599958169a2b3c35b72b9b27f54d1c156032b2fadf041cc0e905bd49e48d8c60 scripts/__init__.py
7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671 scripts/check_contract.py
20fb4ddaa3ab13bc5c350b6762298c6e26dcbf23fcd8e66e496c070d09cdec3b scripts/check_grade4.py
1ebdc8a4423305b6b972254e0b856b9381f7a5279aaa537a3c9d3cdb791adccf scripts/check_idempotency.py
c95f74ebfec55b047424d65ed201519a79cd3292b9067fcf6913f5ac1bf3ab0b scripts/check_nginx.py
4a4b5334ca1a717dfb341dc3ecc4d862b19fcec18d0909b087125800be9ab46e scripts/check_s3.py
c46b007560b879cf025a6162a7fe3e405690b23d2716654bb5b4079fa01fbb4a scripts/collect_evidence.py
0926fac2e73573a6e8b32c254bfc635132823aed9c4b429e91bf180e047bc32f scripts/deployment.py
4cf255992270adfdaa0a3fd304050ed858b7a49d9e3c8856337a310cf59f8596 scripts/expiry_demo.sh
21f2e3baa4d9615e0411edbecfbdc394d471b70b4f237d1e45557c9602b0ecad scripts/explain_history.py
af3422b5b039583980599765cea8315546a1a3de80671f78c7c205faa2bac65f scripts/failover_all.py
e06776d5e44eded34a7fc849ba4cabe955428cf9f924e077a48511656538e7b2 scripts/failure_demo.sh
0d132cc0528257b6bf3289607fd796ab2a176748d4921b6ac0c56da28b15154e scripts/init_local.py
0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b scripts/restart.example.sh
5dcdcb09e56e13e223f5b63e855d853a5a669e6a2b5ff4cb5d4800dd44231352 scripts/restart.sh
60d48ba1c8050a79f8a3725a0ea7bba34dadba3bef0b86bd2d2e6a768e6ca7d5 scripts/rolling_update.py
563376e1d5bfb235790466d5b51ad0559c8c6a0e27da4109ce82705b73f42670 scripts/s3_faults.py
1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811 scripts/stop-one.example.sh
c78f3fee788c907764162069ee09e3bd043b3b3eb3823f5bbb8f84dd1ab5bed0 scripts/stop-one.sh
b67cf021073a1b0fed6d536f3fc1f41d9bc34834090f02f2db3fdff8b8a6e356 scripts/upload_load.py
4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54 scripts/validate_openapi.py
0d23d308c846d2db45e14c6890e6b50be5bdd5a17e89a87b5b8e1e75b27aefc9 student-tests/conftest.py
8c11a6fbc9003c2d92225bfc676edfffe416e70d8b6972a864cf6c5c0b64c95b student-tests/helpers.py
b09d533125981d19774bff19b02ff096c3b367b8bda1dc62fc4647e8816063c7 student-tests/test_attachments.py
20cd1c8ea397fe558a9ded78723dd319cabe47f520a631375171c448a01b1de2 student-tests/test_auth.py
f57751337441fbc909549d553cd32e20d7f7918dc42f2000c15f9ba2706349a4 student-tests/test_chats.py
b9c2ed985bb2cf0261bfb928f0bcc2694c8980a5c55fea4c445d0ca410ca30c6 student-tests/test_concurrency.py
b8c9a1f7535318b946adff13b11ac2459622a1a9b23962f0981d0fc7793de0c9 student-tests/test_constraints.py
76b4fe648fd257eb603cfc06da76b59b783986f96a9b357bc87238210757ef34 student-tests/test_errors.py
045203d83f8ff50e48b8fad4328dba7f12de8adffebef597efadfd40c79ed5c1 student-tests/test_failures.py
2594b7bce03d36e6c32c1644e76f4d8f40e7099de6c687e13a74706e329bcefe student-tests/test_health.py
edaf79363adb942f9501d933deb86e39aed558f32c1a73188ae2d18d088378ae student-tests/test_history_plan.py
640a12067597a7e576d3db0fb6f9bbb14639a14d701f6a9b682ed97ad70cdf8e student-tests/test_idempotency.py
722951c437488d0c5d63839d418b2456c37a40c9e8dc55a52c7e8ea3a4066c3c student-tests/test_instance.py
c6d2ce0d47b2eae767a9f4d2b63333eb865c505cfaa3dffb5236877187318fdc student-tests/test_lifecycle.py
71e48fab2816753b213603bd977c9c1babb2b540596cd3401c1e62a6f538a3b5 student-tests/test_limits.py
cc040ae5ee2f196362fab4c7006711bf8d8c7f5e2f56acc8c74646ef2efb801d student-tests/test_messages.py
1ef7a28cb478169001a64974ff0632a55771c1f3faa02ea40a919cb722bf9a37 student-tests/test_migrations.py
2c573efd6b0a6204a8bac483bb6e2594c565f2d32faa04ebc3540298eb02af2d student-tests/test_pagination.py
0cea69086ff8ecc630eccabdb1a7ff79e72bb2df067cb4a59ce8ad31ba5f08a4 student-tests/test_s3_stream.py
cb981f15ad186faa2e1f8b6aecb7766eed66378b6db9dfc496dd1d76afc94b1b student-tests/test_sessions.py
bf840d8dcb6094412b9e2947df4a0a3c3f32c5d80af9df19693ed5b52b9c8d6d student-tests/test_users.py
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/__init__.py
8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d tests/client.py
99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 tests/fixtures/sample.png
a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13 tests/image_probe.py
6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933 tests/load.py
1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b tests/resilience.py
52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db tests/smoke.py
+396
View File
@@ -0,0 +1,396 @@
$ make check validate test-stand-up TEST_PROJECT=mtusi-messenger-lab4-test
.venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env up -d --build
Image mtusi-messenger:lab4 Building
Image mtusi-messenger:lab4 Building
Image mtusi-messenger:lab4 Building
Image mtusi-messenger-minio:lab4 Building
Image mtusi-messenger-minio-bootstrap:lab4 Building
#1 [internal] load local bake definitions
#1 reading from stdin 2.80kB done
#1 DONE 0.0s
#2 [api internal] load build definition from Dockerfile
#2 transferring dockerfile: 797B done
#2 DONE 0.0s
#3 [minio-bootstrap internal] load build definition from Dockerfile
#3 transferring dockerfile: 1.21kB done
#3 DONE 0.0s
#4 [minio internal] load metadata for docker.io/library/python:3.12.15-slim-bookworm
#4 DONE 1.5s
#5 [minio-bootstrap internal] load metadata for docker.io/library/golang:1.24.8-bookworm
#5 DONE 1.5s
#6 [minio internal] load .dockerignore
#6 transferring context: 196B done
#6 DONE 0.0s
#7 [minio-bootstrap internal] load build context
#7 transferring context: 3.50kB done
#7 DONE 0.1s
#8 [migrate internal] load build context
#8 transferring context: 318.96kB 0.0s done
#8 DONE 0.1s
#9 [minio compiler 1/3] FROM docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848
#9 resolve docker.io/library/golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 0.1s done
#9 DONE 0.2s
#10 [cleanup runtime 1/2] FROM docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70
#10 resolve docker.io/library/python:3.12.15-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 0.1s done
#10 DONE 0.2s
#11 [minio server 1/1] COPY --from=compiler /out/minio /usr/local/bin/minio
#11 CACHED
#12 [minio-bootstrap bootstrap 1/2] COPY --from=compiler /out/mc /usr/local/bin/mc
#12 CACHED
#13 [minio runtime 2/2] RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
#13 CACHED
#14 [minio compiler 3/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
#14 CACHED
#15 [minio compiler 2/3] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
#15 CACHED
#16 [minio-bootstrap bootstrap 2/2] COPY infra/minio/bootstrap.py /bootstrap.py
#16 CACHED
#17 [migrate 4/8] COPY requirements.txt .
#17 CACHED
#18 [migrate 5/8] RUN pip install -r requirements.txt
#18 CACHED
#19 [migrate 7/8] COPY migrations ./migrations
#19 CACHED
#20 [migrate 6/8] COPY alembic.ini ./
#20 CACHED
#21 [migrate 3/8] RUN useradd --system --uid 10001 --no-create-home appuser
#21 CACHED
#22 [migrate 2/8] WORKDIR /app
#22 CACHED
#23 [migrate 8/8] COPY app ./app
#23 CACHED
#24 [minio-bootstrap] exporting to image
#24 exporting layers done
#24 exporting manifest sha256:d7d356b436b22b1950d123996ec850fbe44e7b9b1e5c0f6dfee6c5ed8046fc30 0.0s done
#24 exporting config sha256:416fb93e9d28043ec257557a451b5b260a8ec2141fd6bffcf5d1fa090bc5aee0 0.0s done
#24 exporting attestation manifest sha256:7159f8f74e3f3ab0500179c06398cf1db149d22995073f2793fbcc3e705348c4
#24 exporting attestation manifest sha256:7159f8f74e3f3ab0500179c06398cf1db149d22995073f2793fbcc3e705348c4 0.2s done
#24 exporting manifest list sha256:a89064533cc824208502acd48147b7f4d6bff4b9d7e666994249e4be0a11bce9 0.1s done
#24 naming to docker.io/library/mtusi-messenger-minio-bootstrap:lab4
#24 naming to docker.io/library/mtusi-messenger-minio-bootstrap:lab4 0.0s done
#24 unpacking to docker.io/library/mtusi-messenger-minio-bootstrap:lab4
#24 unpacking to docker.io/library/mtusi-messenger-minio-bootstrap:lab4 0.1s done
#24 DONE 0.6s
#25 [minio] exporting to image
#25 exporting layers done
#25 exporting manifest sha256:91da74b8aa80d8f1967b0b1c34cfd963a7db8186f3f6b502396145eb1fc52c3f 0.0s done
#25 exporting config sha256:ae66a6f1ec14052043afc5d2bc4d41de5ad1986ea0ea7c44f9f02c7a9837f7d8 0.0s done
#25 exporting attestation manifest sha256:ac35b49ea51ebffe7f862b5fcfb5edb5fa9a1ce064b79aa2732473d216fc717f 0.2s done
#25 exporting manifest list sha256:4b32014b70b6c840fc6b88d421bcbbfe627f0c6b89ed7dd8d009a0b00e463f39 0.1s done
#25 naming to docker.io/library/mtusi-messenger-minio:lab4 0.0s done
#25 unpacking to docker.io/library/mtusi-messenger-minio:lab4 0.1s done
#25 DONE 0.6s
#26 [migrate] exporting to image
#26 exporting layers done
#26 exporting manifest sha256:bc6f252b407c7f0f55cef33ac619c87c0af8aa3444bfd48cff70ec5d13a9d3c6 0.1s done
#26 exporting config sha256:07e6fed33c1185350eaaf0ca71bef77b85b69b9f702cc9921b9ded39d0a059dc 0.1s done
#26 exporting attestation manifest sha256:2b7da561e7004b93f36c46abac5fbf5d7517b9f54885ed119fab6d19c1a153da 0.2s done
#26 exporting manifest list sha256:7e3e13a38949ace57409487e70b8510202c0d6ffdeecafcdb400a16c80c7a817 0.1s done
#26 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
#26 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
#26 DONE 0.7s
#27 [api] exporting to image
#27 exporting layers done
#27 exporting manifest sha256:5c384a19f40a3937220bb527abc5857f2db10a6b9215ac3440ee1516cc64e779 0.1s done
#27 exporting config sha256:4b39ae1766eacf6ef7c55627df3daadcb46ce8ee26f4dd63ada8047fa861ce87 0.1s done
#27 exporting attestation manifest sha256:8f6444f4580f96c70c2d6ea0cfd2d3113b93a6c569aea6cde5d714fafa39055a 0.2s done
#27 exporting manifest list sha256:01e56602fc2243976c8e491e5dfaebff3cf8aee1cfdf168e2328a442bc47dfca 0.1s done
#27 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
#27 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
#27 DONE 0.7s
#28 [cleanup] exporting to image
#28 exporting layers done
#28 exporting manifest sha256:6cc0026405e8326374b7eb6900ce3c4ce8c069dedafb7072793ca7fcba6378d4 0.1s done
#28 exporting config sha256:421efb8ea526249eff500a1e0a2936e2334d8244ef3d9558f1a70feffd7afc9b 0.1s done
#28 exporting attestation manifest sha256:70bb6141a85943d37fabc6631399a7bce798427ee873e9036b3099b27ae5edd9 0.2s done
#28 exporting manifest list sha256:7d88364818d984cdb1e1bdc4954841e51e0d842619513cc1951c842e281a9051 0.1s done
#28 naming to docker.io/library/mtusi-messenger:lab4 0.0s done
#28 unpacking to docker.io/library/mtusi-messenger:lab4 0.1s done
#28 DONE 0.7s
#29 [minio-bootstrap] resolving provenance for metadata file
#29 DONE 0.2s
#30 [minio] resolving provenance for metadata file
#30 DONE 0.2s
#31 [api] resolving provenance for metadata file
#31 DONE 0.2s
#32 [migrate] resolving provenance for metadata file
#32 DONE 0.1s
#33 [cleanup] resolving provenance for metadata file
#33 DONE 0.0s
Image mtusi-messenger-minio-bootstrap:lab4 Built
Image mtusi-messenger:lab4 Built
Image mtusi-messenger:lab4 Built
Image mtusi-messenger:lab4 Built
Image mtusi-messenger-minio:lab4 Built
Network mtusi-messenger-lab4-test_objects Creating
Network mtusi-messenger-lab4-test_default Creating
Volume mtusi-messenger-lab4-test_redisdata Creating
Network mtusi-messenger-lab4-test_objects Creating
Network mtusi-messenger-lab4-test_default Creating
Volume mtusi-messenger-lab4-test_redisdata Creating
Volume mtusi-messenger-lab4-test_pgdata Creating
Volume mtusi-messenger-lab4-test_pgdata Creating
Volume mtusi-messenger-lab4-test_miniodata Creating
Volume mtusi-messenger-lab4-test_miniodata Creating
Volume mtusi-messenger-lab4-test_pgdata Created
Volume mtusi-messenger-lab4-test_pgdata Created
Volume mtusi-messenger-lab4-test_miniodata Created
Volume mtusi-messenger-lab4-test_miniodata Created
Volume mtusi-messenger-lab4-test_redisdata Created
Volume mtusi-messenger-lab4-test_redisdata Created
Network mtusi-messenger-lab4-test_default Created
Network mtusi-messenger-lab4-test_default Created
Container mtusi-messenger-lab4-test-redis-1 Creating
Container mtusi-messenger-lab4-test-postgres-1 Creating
Network mtusi-messenger-lab4-test_objects Created
Network mtusi-messenger-lab4-test_objects Created
Container mtusi-messenger-lab4-test-minio-1 Creating
Container mtusi-messenger-lab4-test-postgres-1 Created
Container mtusi-messenger-lab4-test-migrate-1 Creating
Container mtusi-messenger-lab4-test-minio-1 Created
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Creating
Container mtusi-messenger-lab4-test-redis-1 Created
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Created
Container mtusi-messenger-lab4-test-migrate-1 Created
Container mtusi-messenger-lab4-test-api-1 Creating
Container mtusi-messenger-lab4-test-api-2 Creating
Container mtusi-messenger-lab4-test-cleanup-1 Creating
Container mtusi-messenger-lab4-test-api-1 Created
Container mtusi-messenger-lab4-test-cleanup-1 Created
Container mtusi-messenger-lab4-test-api-2 Created
Container mtusi-messenger-lab4-test-nginx-1 Creating
Container mtusi-messenger-lab4-test-nginx-1 Created
Container mtusi-messenger-lab4-test-minio-1 Starting
Container mtusi-messenger-lab4-test-postgres-1 Starting
Container mtusi-messenger-lab4-test-redis-1 Starting
Container mtusi-messenger-lab4-test-minio-1 Started
Container mtusi-messenger-lab4-test-minio-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Started
Container mtusi-messenger-lab4-test-postgres-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Started
Container mtusi-messenger-lab4-test-minio-1 Healthy
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Starting
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Started
Container mtusi-messenger-lab4-test-postgres-1 Healthy
Container mtusi-messenger-lab4-test-migrate-1 Starting
Container mtusi-messenger-lab4-test-migrate-1 Started
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
Container mtusi-messenger-lab4-test-migrate-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Waiting
Container mtusi-messenger-lab4-test-migrate-1 Waiting
Container mtusi-messenger-lab4-test-minio-1 Waiting
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Healthy
Container mtusi-messenger-lab4-test-postgres-1 Healthy
Container mtusi-messenger-lab4-test-minio-1 Healthy
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
Container mtusi-messenger-lab4-test-migrate-1 Exited
Container mtusi-messenger-lab4-test-migrate-1 Exited
Container mtusi-messenger-lab4-test-cleanup-1 Starting
Container mtusi-messenger-lab4-test-api-2 Starting
Container mtusi-messenger-lab4-test-cleanup-1 Started
Container mtusi-messenger-lab4-test-api-2 Started
Container mtusi-messenger-lab4-test-api-1 Starting
Container mtusi-messenger-lab4-test-api-1 Started
Container mtusi-messenger-lab4-test-api-2 Waiting
Container mtusi-messenger-lab4-test-api-1 Waiting
Container mtusi-messenger-lab4-test-api-2 Healthy
Container mtusi-messenger-lab4-test-api-1 Healthy
Container mtusi-messenger-lab4-test-nginx-1 Starting
Container mtusi-messenger-lab4-test-nginx-1 Started
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env exec -T postgres sh -c 'createdb -U "$POSTGRES_USER" "$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
$ make test-docker TEST_PROJECT=mtusi-messenger-lab4-test PYTEST_ARGS=--tb=short
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 DOCKER_TESTS=1 .venv/bin/python -m pytest -q --tb=short
........................................................................ [ 38%]
........................................................................ [ 76%]
............................................ [100%]
188 passed in 73.96s (0:01:13)
$ make test-stand-smoke test-stand-idempotency TEST_PROJECT=mtusi-messenger-lab4-test
.venv/bin/python tests/smoke.py --base-url "http://localhost:18080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
.venv/bin/python -m scripts.check_idempotency --base-url "http://localhost:18080"
PASS: 24 concurrent replays, 2 replicas, one message
$ make test-stand-persistence TEST_PROJECT=mtusi-messenger-lab4-test
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup About a minute ago Up About a minute 8080/tcp
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio About a minute ago Up About a minute (healthy)
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx About a minute ago Up About a minute (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres About a minute ago Up About a minute (healthy) 127.0.0.1:15432->5432/tcp
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis About a minute ago Up 9 seconds (healthy) 127.0.0.1:16379->6379/tcp
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 .venv/bin/python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "http://localhost:18080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup 2 minutes ago Up Less than a second 8080/tcp
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio 2 minutes ago Up 23 seconds (healthy)
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx 2 minutes ago Up 10 seconds (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up 22 seconds (healthy) 127.0.0.1:15432->5432/tcp
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis 2 minutes ago Up 23 seconds (healthy) 127.0.0.1:16379->6379/tcp
$ make test-stand-failover TEST_PROJECT=mtusi-messenger-lab4-test
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 17 seconds (healthy) 8080/tcp
mtusi-messenger-lab4-test-cleanup-1 mtusi-messenger:lab4 "python -m app.core.…" cleanup 2 minutes ago Up Less than a second 8080/tcp
mtusi-messenger-lab4-test-minio-1 mtusi-messenger-minio:lab4 "/usr/local/bin/mini…" minio 2 minutes ago Up 24 seconds (healthy)
mtusi-messenger-lab4-test-nginx-1 nginx:1.28.0-alpine "/docker-entrypoint.…" nginx 2 minutes ago Up 10 seconds (healthy) 80/tcp, 127.0.0.1:18080->8080/tcp
mtusi-messenger-lab4-test-postgres-1 postgres:16.10-bookworm "docker-entrypoint.s…" postgres 2 minutes ago Up 22 seconds (healthy) 127.0.0.1:15432->5432/tcp
mtusi-messenger-lab4-test-redis-1 redis:7.4.2-bookworm "/bin/sh /usr/local/…" redis 2 minutes ago Up 24 seconds (healthy) 127.0.0.1:16379->6379/tcp
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 .venv/bin/python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "http://localhost:18080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 4, "first_recovery_seconds": 1.885}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env ps -a api
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-messenger-lab4-test-api-1 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Up 34 seconds (healthy) 8080/tcp
mtusi-messenger-lab4-test-api-2 mtusi-messenger:lab4 "python -m app" api 2 minutes ago Exited (0) 13 seconds ago
COMPOSE_PROJECT_NAME=mtusi-messenger-lab4-test COMPOSE_FILE=compose.yaml:compose.test.yaml LB_HOST_PORT=18080 DB_HOST_PORT=15432 REDIS_HOST_PORT=16379 docker compose --env-file .env up -d --wait api
Container mtusi-messenger-lab4-test-minio-1 Running
Container mtusi-messenger-lab4-test-postgres-1 Running
Container mtusi-messenger-lab4-test-redis-1 Running
Container mtusi-messenger-lab4-test-api-1 Running
Container mtusi-messenger-lab4-test-minio-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Healthy
Container mtusi-messenger-lab4-test-migrate-1 Starting
Container mtusi-messenger-lab4-test-minio-1 Healthy
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Starting
Container mtusi-messenger-lab4-test-migrate-1 Started
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Started
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Waiting
Container mtusi-messenger-lab4-test-migrate-1 Waiting
Container mtusi-messenger-lab4-test-minio-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Healthy
Container mtusi-messenger-lab4-test-minio-1 Healthy
Container mtusi-messenger-lab4-test-postgres-1 Healthy
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
Container mtusi-messenger-lab4-test-migrate-1 Exited
Container mtusi-messenger-lab4-test-api-2 Starting
Container mtusi-messenger-lab4-test-api-2 Started
Container mtusi-messenger-lab4-test-api-2 Waiting
Container mtusi-messenger-lab4-test-api-1 Waiting
Container mtusi-messenger-lab4-test-minio-1 Waiting
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Waiting
Container mtusi-messenger-lab4-test-redis-1 Waiting
Container mtusi-messenger-lab4-test-migrate-1 Waiting
Container mtusi-messenger-lab4-test-postgres-1 Healthy
Container mtusi-messenger-lab4-test-minio-1 Healthy
Container mtusi-messenger-lab4-test-minio-bootstrap-1 Exited
Container mtusi-messenger-lab4-test-migrate-1 Exited
Container mtusi-messenger-lab4-test-redis-1 Healthy
Container mtusi-messenger-lab4-test-api-2 Healthy
Container mtusi-messenger-lab4-test-api-1 Healthy
$ .venv/bin/python -m scripts.check_s3 --base-url http://localhost:18080 --with-outage --with-faults
PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console, MinIO outage/recovery
PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup
$ .venv/bin/python -m scripts.upload_load --base-url http://localhost:18080 --concurrency 10 --output <temporary>/uploads.json
{
"concurrency": 10,
"rounds": 3,
"attempts": 30,
"fixture": {
"format": "PNG",
"width": 1500,
"height": 1500,
"size_bytes": 6761054,
"sha256": "49cb1a9ec3e54c1f22c649fb787c50186c54e449ade4a6f6758a50b2e374504d"
},
"statuses": {
"201": 12,
"429": 18
},
"elapsed_seconds": 5.138,
"successful_uploads": 12,
"p95_seconds": 2.143,
"memory": {
"bf64a4a13acb": {
"initial_bytes": 88752128,
"sampled_peak_bytes": 162426880,
"cgroup_peak_before_bytes": 161976320,
"cgroup_peak_after_bytes": 162689024,
"interval_ms": 20
},
"7e4180e529e1": {
"initial_bytes": 89436160,
"sampled_peak_bytes": 163692544,
"cgroup_peak_before_bytes": 157728768,
"cgroup_peak_after_bytes": 163692544,
"interval_ms": 20
}
},
"api_memory_limit": "1610612736",
"notes": "20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries"
}
$ .venv/bin/python -m scripts.check_grade4 --base-url http://localhost:18080
PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords
PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in 2.390s
Сборка нового образа API
Однократное применение совместимых миграций
Заменена реплика bf64a4a13acb
Заменена реплика 7e4180e529e1
Готово: 2 здоровых реплик нового образа
{"scenario": "rolling update", "replacements": 2, "migration_runs": 1, "successful_writes": 487, "failed_attempts": 13, "errors_by_kind": {"http_503": 13}, "elapsed_seconds": 25.341, "successful_rps": 19.218, "p95_seconds": 0.048}
{"scenario": "replica_failure", "phase": "before", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
{"scenario": "replica_failure", "phase": "stopped", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": false, "health": "unhealthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
{"scenario": "replica_failure", "phase": "restored", "target": "429367307e5f", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
{"instance": "429367307e5f", "successful_writes": 777, "failed_attempts": 87, "errors_by_kind": {"http_429": 70, "http_503": 17}, "elapsed_seconds": 13.965, "successful_rps": 55.64, "first_recovery_seconds": 1.865}
{"scenario": "replica_failure", "phase": "before", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
{"scenario": "replica_failure", "phase": "stopped", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": false, "health": "unhealthy"}]}
{"scenario": "replica_failure", "phase": "restored", "target": "20b4b219e7c8", "replicas": [{"instance": "429367307e5f", "running": true, "health": "healthy"}, {"instance": "20b4b219e7c8", "running": true, "health": "healthy"}]}
{"instance": "20b4b219e7c8", "successful_writes": 774, "failed_attempts": 96, "errors_by_kind": {"http_429": 79, "http_503": 17}, "elapsed_seconds": 14.127, "successful_rps": 54.79, "first_recovery_seconds": 2.042}
PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords
+5
View File
@@ -1,5 +1,10 @@
# Инфраструктура студента
В этом fork Compose включает Nginx, две API-реплики, PostgreSQL, Redis, MinIO,
миграции, bootstrap bucket и cleanup. Актуальные команды запуска и файловые
секреты — [RUNBOOK.md](../RUNBOOK.md), транзакции и ограничения MinIO —
[LAB4_PLAN.md](../LAB4_PLAN.md). Результаты прогона — [evidence](../evidence/README.md).
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
+22
View File
@@ -0,0 +1,22 @@
FROM golang:1.24.8-bookworm AS compiler
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM python:3.12.15-slim-bookworm AS runtime
RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
USER minio
FROM runtime AS server
COPY --from=compiler /out/minio /usr/local/bin/minio
LABEL org.opencontainers.image.source="https://github.com/minio/minio" \
org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z"
VOLUME ["/data"]
ENTRYPOINT ["/usr/local/bin/minio"]
FROM runtime AS bootstrap
COPY --from=compiler /out/mc /usr/local/bin/mc
COPY infra/minio/bootstrap.py /bootstrap.py
ENTRYPOINT ["python", "/bootstrap.py"]
+77
View File
@@ -0,0 +1,77 @@
"""Идемпотентный private bucket и отдельная ограниченная учётная запись API."""
import json
import os
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory
import time
from urllib.request import urlopen
def secret(name):
value = (Path("/run/secrets") / name).read_text().strip()
if not value:
raise ValueError("Пустой файл секрета")
return value
def main():
endpoint = os.environ["S3_ENDPOINT"]
bucket = os.environ["S3_BUCKET"]
root_user, root_password = secret("minio_root_user"), secret("minio_root_password")
access, password = secret("s3_access_key_id"), secret("s3_secret_access_key")
if access == root_user:
raise ValueError("API не должен использовать root credentials")
deadline = time.monotonic() + 60
while True:
try:
with urlopen(endpoint + "/minio/health/ready", timeout=2) as response:
if response.status == 200:
break
except OSError:
pass
if time.monotonic() >= deadline:
raise RuntimeError("MinIO не готов за 60 секунд")
time.sleep(1)
with TemporaryDirectory(prefix="minio-bootstrap-") as directory:
config = Path(directory)
# Credentials не передаются mc alias set через аргументы или окружение.
config_file = config / "config.json"
config_file.write_text(json.dumps({"version": "10", "aliases": {"storage": {
"url": endpoint, "accessKey": root_user, "secretKey": root_password,
"api": "S3v4", "path": "auto",
}}}))
config_file.chmod(0o600)
def mc(*args):
result = subprocess.run(["mc", "--config-dir", directory, *args],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
if result.returncode:
raise RuntimeError("Ошибка bootstrap MinIO; значения credentials скрыты")
mc("mb", "--ignore-existing", "storage/" + bucket)
mc("anonymous", "set", "none", "storage/" + bucket)
policy = {
"Version": "2012-10-17", "Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
"Resource": [f"arn:aws:s3:::{bucket}"]},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": [f"arn:aws:s3:::{bucket}/originals/*"]},
],
}
policy_file = config / "policy.json"
policy_file.write_text(json.dumps(policy))
mc("admin", "policy", "create", "storage", "messenger-attachments", str(policy_file))
# mc не поддерживает stdin для user add: вывод и ошибки команды скрыты.
mc("admin", "user", "add", "storage", access, password)
mc("admin", "policy", "attach", "storage", "messenger-attachments", "--user", access)
print("MinIO private bucket and API policy configured", flush=True)
if __name__ == "__main__":
try:
main()
except Exception as error:
print("MinIO bootstrap failed: " + type(error).__name__, flush=True)
raise SystemExit(1) from None
+140
View File
@@ -0,0 +1,140 @@
# Единая точка входа: round-robin между API-репликами, без sticky sessions.
# Директивы рассчитаны на закреплённый nginx:1.28.0-alpine.
worker_processes 2;
events {
worker_connections 1024;
}
http {
# В логе используем request_id ответа API, для собственных ошибок — id nginx.
map $upstream_http_x_request_id $response_request_id {
"" $request_id;
default $upstream_http_x_request_id;
}
log_format requests escape=json
'{"time":"$time_iso8601","request_id":"$response_request_id",'
'"method":"$request_method","uri":"$uri","status":$status,'
'"bytes":$body_bytes_sent,"request_time":$request_time,'
'"upstream":"$upstream_addr","upstream_status":"$upstream_status",'
'"upstream_time":"$upstream_response_time",'
'"instance_id":"$upstream_http_x_instance_id",'
'"limit_status":"$limit_req_status"}';
access_log /dev/stdout requests;
error_log /dev/stderr warn;
server_tokens off;
# DNS Docker возвращает адреса всех реплик сервиса api.
resolver 127.0.0.11 valid=5s ipv6=off;
resolver_timeout 1s;
# Общий для worker-процессов лимит на IP, который видит nginx.
# Клиентский X-Forwarded-For не используется в качестве ключа лимита.
limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s;
gzip on;
gzip_vary on;
gzip_min_length 1024;
gzip_comp_level 3;
gzip_types application/json;
upstream api_backend {
zone api_backend 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
# Кэш свободных соединений на worker, а не лимит запросов к API.
keepalive 16;
keepalive_timeout 4s;
}
server {
listen 8080;
client_header_timeout 10s;
client_body_timeout 10s;
# Размер ограничивает API при чтении, включая chunked-передачу.
client_max_body_size 0;
client_body_buffer_size 64k;
send_timeout 10s;
keepalive_timeout 30s;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 500ms;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
# Пассивное обнаружение отказов; Docker healthcheck сам не управляет LB.
# non_idempotent не включён: уже отправленные POST не повторяем.
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
# Окно перехода на другой backend, не общий deadline ответа.
proxy_next_upstream_timeout 2s;
# JSON и заголовки ошибок API передаются без подмены.
proxy_intercept_errors off;
error_page 429 = @rate_limited;
error_page 502 503 504 = @unavailable;
# Проверка самого LB для healthcheck в compose; не проксируется в API.
location = /lb-health {
access_log off;
default_type text/plain;
return 200 "ok\n";
}
# Проверки готовности/жизни не расходуют бюджет бизнес-запросов.
location /health/ {
proxy_pass http://api_backend;
}
location / {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
limit_req_log_level warn;
proxy_pass http://api_backend;
}
location ~ ^/api/v1/chats/[^/]+/attachments$ {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
proxy_request_buffering off;
proxy_buffering off;
proxy_read_timeout 65s;
proxy_send_timeout 15s;
# Ошибка одного upload не должна скрыть работающую реплику для чатов.
proxy_next_upstream off;
proxy_pass http://api_backend;
}
location ~ ^/api/v1/attachments/[^/]+/content$ {
limit_req zone=per_client burst=100 nodelay;
limit_req_status 429;
proxy_buffering off;
proxy_read_timeout 65s;
proxy_pass http://api_backend;
}
# Это ответы балансировщика: X-Instance-Id реплики у них отсутствует.
location @rate_limited {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header Retry-After 1 always;
return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}';
}
location @unavailable {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header Retry-After 1 always;
return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}';
}
}
}
+5
View File
@@ -0,0 +1,5 @@
dir /data
appendonly yes
appendfsync always
maxmemory 256mb
maxmemory-policy noeviction
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
password_hash=$(tr -d '\r\n' </run/secrets/redis_password | sha256sum | cut -d ' ' -f 1)
mkdir -p /run/redis
umask 077
printf 'user default reset on #%s ~* &* +@all\n' "$password_hash" >/run/redis/users.acl
chown redis:redis /run/redis/users.acl
chmod 600 /run/redis/users.acl
exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl
+68
View File
@@ -0,0 +1,68 @@
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from app.core.config import Settings
from app.models import Base
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name, disable_existing_loggers=False)
target_metadata = Base.metadata
def _database_url() -> str:
explicit = config.get_main_option("sqlalchemy.url")
if explicit:
return explicit
return Settings().get_database_url() # type: ignore
def run_migrations_offline() -> None:
context.configure(
url=_database_url(),
target_metadata=target_metadata,
literal_binds=True,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
section = config.get_section(config.config_ini_section, {})
section["sqlalchemy.url"] = _database_url().replace("%", "%%")
connectable = async_engine_from_config(
section, prefix="sqlalchemy.", poolclass=pool.NullPool
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+25
View File
@@ -0,0 +1,25 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
View File
@@ -0,0 +1,43 @@
"""Добавляет результаты идемпотентных запросов."""
from alembic import op
import sqlalchemy as sa
revision = "49d2c947a3f1"
down_revision = "abd273076068"
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Создаёт таблицу ключей без изменения существующих данных."""
op.create_table(
"idempotency_keys",
sa.Column("user_id", sa.UUID(), nullable=False),
sa.Column("method", sa.String(8), nullable=False),
sa.Column("path", sa.String(255), nullable=False),
sa.Column("key", sa.String(128), nullable=False),
sa.Column("request_hash", sa.String(64), nullable=False),
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"), nullable=False,
),
sa.ForeignKeyConstraint(
["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
),
sa.ForeignKeyConstraint(
["message_id"], ["messages.id"],
name=op.f("fk_idempotency_keys_message_id_messages"),
),
sa.PrimaryKeyConstraint(
"user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
),
)
op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"])
def downgrade() -> None:
"""Удаляет таблицу результатов запросов."""
op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys")
op.drop_table("idempotency_keys")
@@ -0,0 +1,61 @@
"""Attachment, связи сообщений и устойчивый журнал загрузок."""
from alembic import op
import sqlalchemy as sa
revision = "8f31b94a2d10"
down_revision = "49d2c947a3f1"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"attachments",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("owner_id", sa.UUID(), nullable=False),
sa.Column("object_key", sa.String(128), nullable=False),
sa.Column("filename", sa.String(255), nullable=False),
sa.Column("content_type", sa.String(32), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), server_default="ready", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")),
sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")),
sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")),
sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")),
sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
)
op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"])
op.create_table(
"message_attachments",
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column("attachment_id", sa.UUID(), nullable=False),
sa.Column("position", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")),
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")),
sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")),
)
op.create_table(
"upload_intents",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("object_key", sa.String(128), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
)
op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"])
def downgrade() -> None:
op.drop_index("ix_upload_intents_created_at", table_name="upload_intents")
op.drop_table("upload_intents")
op.drop_table("message_attachments")
op.drop_index("ix_attachments_chat_id", table_name="attachments")
op.drop_table("attachments")
@@ -0,0 +1,33 @@
"""add counters
Revision ID: abd273076068
Revises: db06fd3126ff
Create Date: 2026-10-26 22:01:01.910396
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = 'abd273076068'
down_revision: Union[str, None] = 'db06fd3126ff'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('counters',
sa.Column('name', sa.String(length=64), nullable=False),
sa.Column('value', sa.BigInteger(), nullable=False),
sa.PrimaryKeyConstraint('name', name=op.f('pk_counters'))
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('counters')
# ### end Alembic commands ###
@@ -0,0 +1,107 @@
"""create initial schema
Revision ID: db06fd3126ff
Revises:
Create Date: 2026-10-26 17:09:16.675339
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "db06fd3126ff"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"chats",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("title", sa.String(length=100), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")),
)
op.create_table(
"users",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("username", sa.String(length=32), nullable=False),
sa.Column("display_name", sa.String(length=100), nullable=False),
sa.Column("password_hash", sa.String(length=255), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
sa.UniqueConstraint("username", name=op.f("uq_users_username")),
)
op.create_table(
"chat_members",
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("user_id", sa.UUID(), nullable=False),
sa.ForeignKeyConstraint(
["chat_id"],
["chats.id"],
name=op.f("fk_chat_members_chat_id_chats"),
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users")
),
sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")),
)
op.create_index(
"ix_chat_members_user_id_chat_id",
"chat_members",
["user_id", "chat_id"],
unique=False,
)
op.create_table(
"messages",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("chat_id", sa.UUID(), nullable=False),
sa.Column("sender_id", sa.UUID(), nullable=False),
sa.Column("text", sa.Text(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.ForeignKeyConstraint(
["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats")
),
sa.ForeignKeyConstraint(
["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users")
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")),
)
op.create_index(
"ix_messages_chat_id_created_at_id",
"messages",
["chat_id", "created_at", "id"],
unique=False,
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages")
op.drop_table("messages")
op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members")
op.drop_table("chat_members")
op.drop_table("users")
op.drop_table("chats")
# ### end Alembic commands ###
+6
View File
@@ -0,0 +1,6 @@
[pytest]
testpaths = student-tests
python_files = test_*.py
asyncio_mode = auto
asyncio_default_fixture_loop_scope = session
asyncio_default_test_loop_scope = session
+52
View File
@@ -0,0 +1,52 @@
alembic==1.20.0
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
attrs==26.1.0
certifi==2026.7.22
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.143.3
h11==0.16.0
httpcore2==2.13.1
httpx2==2.13.1
idna==3.20
iniconfig==2.3.1
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.12.0
Mako==1.4.3
MarkupSafe==3.0.4
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
opentelemetry-api==1.45.1
packaging==26.3
pathable==0.4.4
pluggy==1.7.0
pydantic==2.14.1
pydantic_core==2.50.1
Pygments==2.21.0
pytest==9.1.1
PyYAML==6.0.3
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
six==1.17.0
SQLAlchemy==2.1.4
starlette==1.8.0
truststore==0.10.5
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.54.0
asyncpg==0.32.0
argon2-cffi==25.1.0
redis==8.1.0
greenlet==3.5.6
pydantic-settings==2.16.0
pytest-asyncio==1.4.1
httpx==0.28.1
minio==7.2.20
Pillow==13.0.0
+1
View File
@@ -0,0 +1 @@
"""Команды управления учебным стендом."""
+355
View File
@@ -0,0 +1,355 @@
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
import asyncio
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
from threading import Event
import time
from uuid import uuid4
import asyncpg
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from scripts.failover_all import exercise, failure_kind, history
from scripts.rolling_update import update
from tests.client import Client, Failure, arguments, check, run
PROBE = """import json, sys, urllib.request, urllib.error
payload = json.load(sys.stdin)
results = []
for item in payload:
data = json.dumps(item['body']).encode() if 'body' in item else None
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
try:
response = urllib.request.urlopen(request, timeout=7)
except urllib.error.HTTPError as error:
response = error
with response:
results.append({'status': response.status, 'body': json.loads(response.read())})
print(json.dumps(results))
"""
def probe(project, runner, requests):
"""Обращается к внутреннему API, передавая токен только через stdin."""
result = subprocess.run(
["docker", "exec", "-i", runner, "python", "-c", PROBE],
input=json.dumps(requests),
text=True,
capture_output=True,
timeout=10,
)
check(result.returncode == 0, "internal HTTP probe failed")
return json.loads(result.stdout)
def audit(project):
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
allowed = {
"postgres": {"db_password"},
"redis": {"redis_password"},
"migrate": {"db_password"},
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
"minio": {"minio_root_user", "minio_root_password"},
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
"nginx": set(),
}
ids = project.run("ps", "--all", "--quiet").splitlines()
containers = json.loads(project.docker("inspect", *ids))
for item in containers:
service = item["Config"]["Labels"]["com.docker.compose.service"]
check(service in allowed, "unexpected service")
host = item["HostConfig"]
check(
host["Memory"] > 0 and host["NanoCpus"] > 0,
service + ": missing CPU/RAM limit",
)
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
check(
not env_names
& {
"DB_PASSWORD",
"REDIS_PASSWORD",
"POSTGRES_PASSWORD",
"REDISCLI_AUTH",
"DATABASE_URL",
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
},
service + ": password in environment",
)
secrets = {
Path(mount["Destination"]).name
for mount in item["Mounts"]
if mount["Destination"].startswith("/run/secrets/")
}
check(secrets == allowed[service], service + ": incorrect secret scope")
if service in {"api", "minio"}:
check(not host["PortBindings"], service + " publishes ports")
for mount in item["Mounts"]:
if mount["Destination"].startswith("/run/secrets/"):
check(not mount["RW"], "writable secret mount")
check(
"container_name" not in project.config["services"][service],
"fixed container name",
)
check(
len(containers) == len(allowed) - 1 + len(project.replicas()),
"unexpected container count",
)
print(
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
flush=True,
)
async def graceful(project, base_url):
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
replicas = project.replicas()
target, runner = replicas[:2]
client = Client(base_url)
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
raw = json.dumps([actor["id"], "POST", path, key]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
database = project.config["services"]["postgres"]
env = database["environment"]
conn = await asyncpg.connect(
host="127.0.0.1",
port=int(database["ports"][0]["published"]),
database=env["POSTGRES_DB"],
user=env["POSTGRES_USER"],
password=(ROOT / "secrets/db_password").read_text().strip(),
timeout=3,
)
network = project.config['networks']['default']['name']
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
address = project.docker(
"inspect",
"--format",
address_template,
target.id,
)
check(bool(address), 'API has no address in the application network')
port = project.config["services"]["api"]["environment"]["APP_PORT"]
direct = f"http://{address}:{port}"
started = time.monotonic()
task = None
try:
async with conn.transaction():
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
task = asyncio.create_task(
asyncio.to_thread(
probe,
project,
runner.id,
[
{
"url": direct + path,
"method": "POST",
"body": {"text": "graceful-" + key},
"headers": {
"Authorization": "Bearer " + actor["token"],
"Idempotency-Key": key,
"Content-Type": "application/json",
},
}
],
)
)
deadline = time.monotonic() + 3
while not await conn.fetchval(
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
):
check(time.monotonic() < deadline, "POST did not reach database lock")
await asyncio.sleep(0.02)
started = time.monotonic()
await asyncio.to_thread(
project.docker, "kill", "--signal", "SIGTERM", target.id
)
results = await asyncio.to_thread(
probe,
project,
runner.id,
[
{"url": direct + "/health/ready"},
{"url": direct + "/health/live"},
{"url": direct + "/api/v1/users/me"},
],
)
check(
[item["status"] for item in results] == [503, 200, 503],
"incorrect drain readiness/liveness/new work",
)
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
result = (await task)[0]
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
state = json.loads(
project.docker("inspect", "--format", "{{json .State}}", target.id)
)
check(
not state["OOMKilled"] and state["ExitCode"] != 137,
"forced shutdown instead of graceful completion",
)
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
items = await asyncio.to_thread(history, client, actor, chat)
check(
result["body"]["id"] in {item["id"] for item in items},
"acknowledged in-flight message lost",
)
print(
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
flush=True,
)
finally:
await conn.close()
if task:
await asyncio.gather(task, return_exceptions=True)
if any(item.id == target.id and item.running for item in project.replicas()):
await asyncio.to_thread(project.docker, "stop", target.id)
await asyncio.to_thread(project.docker, "start", target.id)
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
class ObservedProject(ComposeProject):
"""Подсчитывает миграции и проверяет готовность перед заменой."""
def __init__(self):
self.migrations = 0
self.replacements = 0
self.replacement_started = []
super().__init__()
def run(self, *args, **kwargs):
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
self.migrations += 1
return super().run(*args, **kwargs)
def docker(self, *args, **kwargs):
if args[0] == "stop":
healthy = [
item
for item in self.replicas()
if item.id != args[1] and item.running and item.health == "healthy"
]
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
self.replacements += 1
self.replacement_started.append(time.monotonic())
return super().docker(*args, **kwargs)
def rolling(base_url):
"""Заменяет образ под постоянной записью с проверкой сохранности."""
project = ObservedProject()
before = project.replicas()
client = Client(base_url)
actor, _, _, chat = client.bootstrap()
stop = Event()
results, errors = [], []
def worker():
local = Client(base_url, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "rolling-" + key, key=key)
results.append(
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
)
key = uuid4().hex
except Failure as error:
errors.append(failure_kind(error))
stop.wait(0.08)
with ThreadPoolExecutor(max_workers=2) as pool:
jobs = [pool.submit(worker) for _ in range(2)]
try:
time.sleep(1)
started = time.monotonic()
update(project, base_url, revision="verification-" + uuid4().hex[:12])
finished = time.monotonic()
time.sleep(1)
finally:
stop.set()
for job in jobs:
job.result(timeout=5)
after = project.replicas()
check(
project.migrations == 1 and project.replacements == len(before),
"migration/replacement count incorrect",
)
check(
{item.image for item in before}.isdisjoint({item.image for item in after}),
"new image was not deployed",
)
check(
{item.id for item in before}.isdisjoint({item.id for item in after}),
"old containers remain",
)
writes = [row for row in results if started <= row[0] <= finished]
check(bool(writes), "no writes during update")
check(
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
"no writes during container replacement",
)
items = history(client, actor, chat)
check(
{row[2] for row in results} <= {item["id"] for item in items},
"rolling update lost confirmed messages",
)
check(
len(items) == len({item["text"] for item in items}),
"rolling update created duplicate messages",
)
latencies = sorted(row[3] for row in writes)
print(
json.dumps(
{
"scenario": "rolling update",
"replacements": project.replacements,
"migration_runs": project.migrations,
"successful_writes": len(writes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(finished - started, 3),
"successful_rps": round(len(writes) / (finished - started), 3),
"p95_seconds": round(
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
),
}
),
flush=True,
)
def main():
"""Выполняет проверки только на отдельном тестовом проекте."""
def configure(parser):
parser.add_argument(
"--phase", choices=("all", "update-and-failover"), default="all"
)
args = arguments("Grade 4 integration checks", configure)
project = ComposeProject()
check(project.name.endswith("-test"), "use isolated test Compose project")
with deployment_lock(project):
audit(project)
if args.phase == "all":
asyncio.run(graceful(project, args.base_url))
rolling(args.base_url)
args.duration, args.concurrency = 10, 2
exercise(project, args)
audit(project)
if __name__ == "__main__":
run(main)
+39
View File
@@ -0,0 +1,39 @@
"""Проверяет конкурентные повторы через балансировщик."""
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from tests.client import Client, arguments, check, run
def main() -> None:
"""Отправляет один ключ параллельно через разные реплики."""
args = arguments("Concurrent idempotency through load balancer")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = uuid4().hex
body = "parallel-" + key
def send(_):
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send, range(24)))
ids = {result["json"]["id"] for result in results}
replicas = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel retries created duplicate messages")
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
history = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]["items"]
check(
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
)
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
if __name__ == "__main__":
run(main)
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
from concurrent.futures import ThreadPoolExecutor
import gzip
import json
from pathlib import Path
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
IMAGE = "nginx:1.28.0-alpine"
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
def check(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
class Stand:
def __init__(self, directory: str) -> None:
self.directory = Path(directory)
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
self.base_url = ""
backend = self.directory / "backend.conf"
backend.write_text(
"worker_processes 1;\n"
"events { worker_connections 1024; }\n"
"http {\n"
" log_format backend 'BACKEND $request_method $uri';\n"
" access_log /dev/stdout backend;\n"
" server {\n"
" listen 8080;\n"
" default_type application/json;\n"
" add_header X-Instance-Id $hostname always;\n"
" add_header X-Request-Id $request_id always;\n"
" add_header Retry-After 7 always;\n"
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
" location = /api/limited {\n"
' return 429 \'{"error":{"code":"rate_limited",'
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
" }\n"
" location = /api/post-failure {\n"
' return 503 \'{"error":{"code":"unavailable",'
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
" }\n"
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
" }\n"
"}\n"
)
self.compose_file = self.directory / "compose.json"
self.compose_file.write_text(
json.dumps(
{
"services": {
"api": {
"image": IMAGE,
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
},
"nginx": {
"image": IMAGE,
"volumes": [
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
],
"ports": ["127.0.0.1::8080"],
},
}
}
)
)
def compose(self, *args: str) -> str:
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
"/dev/null",
"-f",
str(self.compose_file),
"-p",
self.project,
*args,
],
cwd=self.directory,
capture_output=True,
text=True,
timeout=45,
)
if result.returncode:
raise RuntimeError(result.stdout + result.stderr)
return result.stdout + result.stderr
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
request = Request(
self.base_url + path,
method=method,
headers={"Accept-Encoding": encoding},
data=b"{}" if method == "POST" else None,
)
try:
response = urlopen(request, timeout=3)
except HTTPError as error:
response = error
except (URLError, TimeoutError):
return 0, {}, b""
with response:
return (
response.status,
{key.lower(): value for key, value in response.headers.items()},
response.read(),
)
def wait_instances(self, count: int) -> set[str]:
seen: set[str] = set()
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
status, headers, _ = self.call("/health/live")
if status == 200:
seen.add(headers["x-instance-id"])
if len(seen) == count:
return seen
time.sleep(0.05)
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
def check_error(response, status: int, code: str) -> None:
actual_status, headers, body = response
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
check(
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
)
error = json.loads(body)["error"]
check(error["code"] == code, "Неверный код ошибки")
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
def run_checks(stand: Stand) -> None:
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
address = stand.compose("port", "nginx", "8080").strip()
stand.base_url = "http://" + address
stand.wait_instances(2)
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
stand.compose("exec", "-T", "nginx", "nginx", "-t")
print("PASS: nginx -t и балансировка двух реплик")
status, headers, body = stand.call("/api/large", encoding="gzip")
check(
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
)
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
check(
"accept-encoding" in headers.get("vary", "").lower(),
"Нет Vary: Accept-Encoding",
)
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
small = stand.call("/health/live", encoding="gzip")
check(
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
)
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
limited = stand.call("/api/limited")
check_error(limited, 429, "rate_limited")
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
print("PASS: JSON и заголовки ошибки API передаются без подмены")
failure = stand.call("/api/post-failure", method="POST")
check_error(failure, 503, "unavailable")
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
print("PASS: отправленный POST не повторяется на другой реплике")
with ThreadPoolExecutor(max_workers=32) as pool:
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
rejected = [response for response in responses if response[0] == 429]
check(bool(rejected), "Не удалось вызвать ограничение частоты")
check(
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
)
for response in rejected:
check_error(response, 429, "rate_limited")
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
with ThreadPoolExecutor(max_workers=16) as pool:
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
check(all(response[0] == 200 for response in health), "Лимит затронул health")
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
stand.compose(
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
)
instances = stand.wait_instances(3)
print("PASS: третья реплика обнаружена через DNS без reload nginx")
target = sorted(instances)[0]
containers = stand.compose("ps", "-q", "api").splitlines()
matching = [container for container in containers if container.startswith(target)]
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
# Все id взяты только из уникального временного Compose-проекта.
started = time.monotonic()
subprocess.run(
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
)
recovery = None
for _ in range(20):
status, headers, _ = stand.call("/health/live")
if status == 200:
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
if recovery is None:
recovery = time.monotonic() - started
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
stand.compose("stop", "--timeout", "1", "api")
unavailable = stand.call("/health/ready")
check_error(unavailable, 503, "unavailable")
check(
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
)
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
print("PASS: недоступность всех backend возвращает JSON 503")
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
check(bool(entries), "Нет JSON access log")
check(
all("request_id" in entry and "upstream_time" in entry for entry in entries),
"Неполный лог",
)
check(
any(entry["limit_status"] == "REJECTED" for entry in entries),
"Лимит не отражён в логе",
)
print("PASS: JSON access log содержит время backend и результат лимита")
def main() -> None:
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
stand = Stand(directory)
try:
run_checks(stand)
except Exception:
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
raise
finally:
stand.compose("down", "--remove-orphans", "--timeout", "1")
if __name__ == "__main__":
main()
+137
View File
@@ -0,0 +1,137 @@
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
import json
import base64
from pathlib import Path
import subprocess
import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r'''
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.storage.s3 import create_s3_storage
from app.models.attachment import Attachment
from minio.error import S3Error
async def main():
settings = Settings()
engine = create_db_engine(settings)
storage = create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
key = record.object_key
results = {}
for method in ('GET', 'PUT'):
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
method=method, data=b'x' if method == 'PUT' else None)
try:
with urlopen(request, timeout=3) as response:
results[method] = response.status
except HTTPError as error:
results[method] = error.code
error.close()
try:
storage._client.get_bucket_policy(settings.s3.bucket)
results['admin_denied'] = False
except S3Error as error:
results['admin_denied'] = error.code == 'AccessDenied'
try:
buckets = storage._client.list_buckets()
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
except S3Error as error:
results['other_buckets_denied'] = error.code == 'AccessDenied'
try:
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
results['console_unreachable'] = False
except HTTPError as error:
results['console_unreachable'] = False
error.close()
except URLError:
results['console_unreachable'] = True
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
for name in ('minio_root_user', 'minio_root_password'))
print(json.dumps(results))
finally:
await storage.aclose()
await engine.dispose()
asyncio.run(main())
'''
def main():
def configure(parser):
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
args = arguments('MinIO private bucket and failure checks', configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
replicas = project.replicas()
check(bool(replicas), 'API not running')
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
input=json.dumps({'attachment_id': attachment['id']}),
text=True, capture_output=True, timeout=20)
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
statuses = json.loads(result.stdout)
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
check(statuses['console_unreachable'], 'console reachable from API network')
check(statuses['temp_originals_absent'], 'API retained temporary originals')
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
env = project.config['services']['minio']['environment']
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
if args.with_faults:
project.run('stop', 'cleanup')
try:
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
result = subprocess.run(
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
'image': base64.b64encode(original).decode()}),
text=True, capture_output=True, timeout=60,
)
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
finally:
project.run('start', 'cleanup')
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
if args.with_outage:
try:
project.run('stop', 'minio')
client.call('GET', path, expected=503, actor=alice)
finally:
project.run('start', 'minio')
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
+ (', MinIO outage/recovery' if args.with_outage else ''))
if args.with_faults:
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
if __name__ == '__main__':
run(main)
+204
View File
@@ -0,0 +1,204 @@
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
from datetime import datetime, timezone
import hashlib
from importlib.metadata import version
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
from tempfile import TemporaryDirectory
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
PROJECT = 'mtusi-messenger-lab4-test'
EVIDENCE = ROOT / 'evidence'
ENVIRONMENT = {
**os.environ,
'COMPOSE_PROJECT_NAME': PROJECT,
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
'LB_HOST_PORT': '18080',
}
COMPOSE = ['docker', 'compose', '--env-file', '.env']
NONSECRET_SETTINGS = {
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
'S3_CLEANUP_INTERVAL_SECONDS',
}
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
def timestamp():
return datetime.now(timezone.utc).isoformat()
def invoke(command):
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
def read_command(command):
result = invoke(command)
if result.returncode:
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
return result.stdout.strip()
def source_manifest():
names = {
'.dockerignore', '.env.example', 'AGENTS.md', 'CODE_STYLE.md',
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
}
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
if path.is_file() and '__pycache__' not in path.parts
and path.suffix != '.md' and not path.name.startswith('.'))
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
def sanitize(text, secrets, staging):
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
for secret in secrets:
clean = clean.replace(secret, '<redacted>')
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
return clean
def container_snapshot(images):
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
rows = []
if ids:
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
rows = [json.loads(line) for line in output.splitlines()]
for row in rows:
image = row['image_id']
if image not in images:
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
return rows
def remaining_resources():
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
f'label=com.docker.compose.project={PROJECT}']).splitlines()
for resource in ('container', 'network', 'volume')}
def extract_metrics(output, upload_file):
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
if pytest is None:
raise RuntimeError('В выводе нет успешного итогового результата pytest')
return {
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
'uploads': json.loads(upload_file.read_text()),
'public_failover': [row for row in rows if 'observed_instances_before' in row],
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
}
def main():
if any(remaining_resources().values()):
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
started = timestamp()
manifest = source_manifest()
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
secrets = [value for value in secrets if value]
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
services = {name: {
'image': service.get('image'),
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
'tmpfs': service.get('tmpfs', []),
} for name, service in config['services'].items()}
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
staging = Path(folder)
upload_file = staging / 'uploads.json'
def execute(command):
begin = timestamp()
result = invoke(command)
output = sanitize(result.stdout, secrets, staging)
display = sanitize(shlex.join(command), secrets, staging)
commands.append({'command': display, 'started_at_utc': begin,
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
if result.returncode:
print(output[-5000:], flush=True)
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
return f'$ {display}\n{output}\n'
cleanup_output = ''
try:
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
snapshots['started'] = container_snapshot(images)
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
snapshots['before_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
snapshots['after_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
snapshots['after_failover'] = container_snapshot(images)
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:18080', '--with-outage', '--with-faults']))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:18080', '--concurrency', '10', '--output', str(upload_file)]))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:18080']))
snapshots['after_update_and_failures'] = container_snapshot(images)
nginx_output = execute(['make', 'test-nginx'])
finally:
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
remaining = remaining_resources()
if any(remaining.values()):
raise RuntimeError('После очистки остались ресурсы тестового проекта')
if source_manifest() != manifest:
raise RuntimeError('Во время прогона изменились проверяемые исходники')
output = ''.join(combined)
metrics = extract_metrics(output, upload_file)
run = {
'started_at_utc': started, 'finished_at_utc': timestamp(),
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
'project': PROJECT, 'runtime': {'python': platform.python_version(),
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
'compose': read_command(['docker', 'compose', 'version'])},
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
'services': services, 'container_snapshots': snapshots, 'images': images,
'commands': commands, 'remaining_test_resources': remaining,
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
}
artifacts = {
'test-output.txt': output, 'nginx-output.txt': nginx_output,
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
}
for name, content in artifacts.items():
if any(secret in content for secret in secrets):
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
EVIDENCE.mkdir(exist_ok=True)
for name, content in artifacts.items():
(EVIDENCE / name).write_text(content)
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
if __name__ == '__main__':
main()

Some files were not shown because too many files have changed in this diff Show More