forked from mtusi2026/mtusi2026lab5
pure version of lab5
This commit is contained in:
+5
-5
@@ -1,8 +1,8 @@
|
||||
# Скопируйте в .env для локального окружения; .env не коммитится.
|
||||
# Настройки приложения, Compose и очереди lab5.
|
||||
# Настройки приложения, Compose и очереди.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
APP_ENV=development
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
MAX_MESSAGES=100000
|
||||
@@ -35,7 +35,7 @@ DB_STATEMENT_TIMEOUT_MS=5000
|
||||
|
||||
# --- Балансировка и идемпотентность ---
|
||||
API_REPLICAS=2
|
||||
API_IMAGE=mtusi-messenger:lab5
|
||||
API_IMAGE=messenger:local
|
||||
API_BUILD_REVISION=development
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
SHUTDOWN_DRAIN_SECONDS=1
|
||||
@@ -48,7 +48,7 @@ LB_HOST_PORT=18081
|
||||
DB_HOST_PORT=25432
|
||||
REDIS_HOST_PORT=26379
|
||||
|
||||
# --- Lab4: MinIO, ограниченный upload и очистка журнала ---
|
||||
# --- MinIO, ограниченный upload и очистка журнала ---
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
@@ -72,7 +72,7 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
|
||||
MINIO_BROWSER=off
|
||||
|
||||
# Lab5: отдельный RabbitMQ broker и ограниченный worker.
|
||||
# Отдельный RabbitMQ broker и ограниченный worker.
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
|
||||
+1
-3
@@ -1,7 +1,5 @@
|
||||
FROM python:3.12.15-slim-bookworm
|
||||
|
||||
LABEL io.mtusi.messenger.secrets="file-v1"
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
@@ -19,7 +17,7 @@ COPY migrations ./migrations
|
||||
COPY app ./app
|
||||
|
||||
ARG API_BUILD_REVISION
|
||||
LABEL io.mtusi.messenger.revision=$API_BUILD_REVISION
|
||||
LABEL org.opencontainers.image.revision=$API_BUILD_REVISION
|
||||
|
||||
USER appuser
|
||||
|
||||
|
||||
@@ -4,14 +4,12 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(if $(wildcard ven
|
||||
PYTEST_ARGS ?=
|
||||
BASE_URL ?= http://localhost:$(or $(LB_HOST_PORT),8080)
|
||||
CA_FILE ?=
|
||||
SERVICE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
COMPOSE = docker compose --env-file .env
|
||||
|
||||
# Отдельный тестовый стенд: свой compose-проект, свои порты, открытые БД/Redis
|
||||
# (compose.test.yaml). COMPOSE_FILE подхватывают и scripts/*.sh.
|
||||
TEST_PROJECT ?= mtusi-messenger-lab5-test
|
||||
TEST_PROJECT ?= messenger-test
|
||||
TEST_LB_PORT ?= 28080
|
||||
TEST_DB_PORT ?= $(DB_HOST_PORT)
|
||||
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
|
||||
@@ -27,58 +25,49 @@ TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/r
|
||||
TEST_REDIS_DB ?= 15
|
||||
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
|
||||
|
||||
define wait_ready
|
||||
@for i in $$(seq 1 90); do \
|
||||
curl -sf "$(1)/health/ready" >/dev/null && exit 0; sleep 1; \
|
||||
done; echo "API $(1) не готов за 90 секунд" >&2; exit 1
|
||||
endef
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover \
|
||||
init test-grade4 test-s3 load-uploads cleanup-dry-run cleanup-apply \
|
||||
up down restart restore ps logs migrate migration instances \
|
||||
test-stand-up test-stand-down test-stand-destroy test-stand-ps \
|
||||
test-stand-smoke test-stand-persistence test-stand-failover \
|
||||
test-own test-docker test-limits test-concurrency test-migrations test-all \
|
||||
explain expiry-demo failure-demo test-nginx test-lab3-grade3 test-stand-idempotency \
|
||||
rolling-update failover-all test-lab3-grade4
|
||||
.PHONY: help init up down restart ps logs migrate migration worker-stop worker-start \
|
||||
cleanup-dry-run cleanup-apply check validate test test-unit test-pipeline \
|
||||
test-nginx test-storage test-outbox test-jobs test-persistence test-failover \
|
||||
test-suite test-stand-up test-stand-destroy
|
||||
|
||||
help:
|
||||
@echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply"
|
||||
@echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads"
|
||||
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
|
||||
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
|
||||
@echo "Тестовый: test-stand-up | test-stand-down | test-stand-destroy | test-stand-ps"
|
||||
@echo " test-stand-smoke | test-stand-persistence | test-stand-failover"
|
||||
@echo "Тесты: test-own | test-docker | test-limits | test-concurrency | test-migrations | test-all"
|
||||
@echo "Демо: explain | expiry-demo | failure-demo (на тестовом стенде)"
|
||||
@echo "Nginx: test-nginx (отдельный временный стенд без БД и приложения)"
|
||||
@echo "База lab3: test-lab3-grade3 (все проверки уровня 3 с удалением тестовых volumes)"
|
||||
@echo "База lab3: rolling-update | failover-all"
|
||||
@echo " test-lab3-grade4 (проверки lab3 на отдельном стенде с очисткой)"
|
||||
@echo "Миграции: migration m='описание' (нужен поднятый тестовый стенд)"
|
||||
|
||||
# --- Проверки API и сценариев отказа (tests/) ---
|
||||
@echo "Запуск: init | up | down | restart | ps | logs SERVICE=worker"
|
||||
@echo "Обслуживание: migrate | migration m='описание' | worker-stop | worker-start"
|
||||
@echo "Очистка объектов: cleanup-dry-run | cleanup-apply"
|
||||
@echo "Проверки: check | validate | test | test-pipeline | test-unit PYTEST_ARGS=..."
|
||||
@echo "Сервисы: test-nginx | test-storage | test-outbox | test-jobs"
|
||||
@echo "Отказы API (по запросу): test-persistence | test-failover"
|
||||
@echo "Автономный прогон с удалением тестового стенда: test-suite"
|
||||
|
||||
init:
|
||||
$(PYTHON) scripts/init_local.py
|
||||
|
||||
test-grade4:
|
||||
$(MAKE) check validate
|
||||
$(MAKE) test-stand-up
|
||||
$(MAKE) test-docker
|
||||
$(MAKE) test-stand-smoke
|
||||
$(MAKE) test-stand-idempotency
|
||||
$(MAKE) test-stand-persistence
|
||||
$(MAKE) test-stand-failover
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)"
|
||||
up:
|
||||
$(COMPOSE) up -d --build --wait --wait-timeout 120
|
||||
|
||||
test-s3:
|
||||
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
|
||||
load-uploads:
|
||||
$(PYTHON) -m scripts.upload_load --base-url "$(BASE_URL)" --concurrency 10
|
||||
restart:
|
||||
scripts/restart.sh
|
||||
|
||||
ps:
|
||||
$(COMPOSE) ps $(SERVICE)
|
||||
|
||||
logs:
|
||||
$(COMPOSE) logs -f $(SERVICE)
|
||||
|
||||
migrate:
|
||||
$(COMPOSE) run --rm --build migrate
|
||||
|
||||
migration:
|
||||
DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)"
|
||||
|
||||
worker-stop:
|
||||
$(COMPOSE) stop worker
|
||||
|
||||
worker-start:
|
||||
$(COMPOSE) start worker
|
||||
|
||||
cleanup-dry-run:
|
||||
$(COMPOSE) run --rm --no-deps cleanup python -m app.core.cleanup --once
|
||||
@@ -92,161 +81,49 @@ check:
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
|
||||
test-nginx:
|
||||
$(PYTHON) scripts/check_nginx.py
|
||||
|
||||
test-lab3-grade3:
|
||||
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
|
||||
$(MAKE) check validate; \
|
||||
$(MAKE) test-stand-up; \
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
|
||||
|
||||
test-lab3-grade4:
|
||||
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
|
||||
$(MAKE) check validate; \
|
||||
$(MAKE) test-stand-up; \
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover; \
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
# Все сохранённые регрессионные тесты запускаются явно; PYTEST_ARGS выбирает подмножество.
|
||||
test-unit:
|
||||
$(TEST_ENV) $(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
test-pipeline:
|
||||
$(TEST_ENV) $(PYTHON) -m pytest -q student-tests/test_attachments.py student-tests/test_jobs.py student-tests/test_image_pipeline.py $(PYTEST_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
test-nginx:
|
||||
$(PYTHON) scripts/check_nginx.py
|
||||
|
||||
# --- Основной стенд ---
|
||||
test-storage:
|
||||
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
rolling-update:
|
||||
$(PYTHON) -m scripts.rolling_update --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
failover-all:
|
||||
$(PYTHON) -m scripts.failover_all --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
up:
|
||||
$(COMPOSE) up -d --build
|
||||
$(call wait_ready,$(BASE_URL))
|
||||
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
|
||||
restart:
|
||||
scripts/restart.sh
|
||||
$(call wait_ready,$(BASE_URL))
|
||||
|
||||
# Вернуть остановленную stop-one.sh реплику (compose up стартует остановленные контейнеры).
|
||||
restore:
|
||||
$(COMPOSE) up -d --wait api
|
||||
|
||||
ps:
|
||||
$(COMPOSE) ps
|
||||
|
||||
logs:
|
||||
$(COMPOSE) logs -f api nginx
|
||||
|
||||
migrate:
|
||||
$(COMPOSE) run --rm --build migrate
|
||||
|
||||
# Показать, какие реплики отвечают через LB (по заголовку X-Instance-Id).
|
||||
instances:
|
||||
@for i in $$(seq 1 10); do \
|
||||
curl -s -o /dev/null -D - "$(BASE_URL)/health/live" | grep -i '^x-instance-id' || echo "нет X-Instance-Id"; \
|
||||
done
|
||||
|
||||
# Генерация миграции против БД тестового стенда (основной стенд БД не публикует).
|
||||
migration:
|
||||
DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) $(PYTHON) -m alembic revision --autogenerate -m "$(m)"
|
||||
|
||||
# --- Тестовый стенд ---
|
||||
|
||||
test-stand-up:
|
||||
$(TEST_COMPOSE) up -d --build
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
$(TEST_COMPOSE) exec -T postgres sh -c 'createdb -U "$$POSTGRES_USER" "$$POSTGRES_DB"_test 2>&1 | grep -v "already exists"; true'
|
||||
|
||||
test-stand-down:
|
||||
$(TEST_COMPOSE) down
|
||||
|
||||
test-stand-destroy:
|
||||
$(TEST_COMPOSE) down -v
|
||||
|
||||
test-stand-ps:
|
||||
$(TEST_COMPOSE) ps
|
||||
|
||||
test-stand-smoke:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-stand-idempotency:
|
||||
$(PYTHON) -m scripts.check_idempotency --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-stand-persistence:
|
||||
$(TEST_COMPOSE) ps
|
||||
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_COMPOSE) ps
|
||||
|
||||
test-stand-failover:
|
||||
$(TEST_COMPOSE) ps
|
||||
$(TEST_ENV) $(PYTHON) tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_COMPOSE) ps -a api
|
||||
$(TEST_COMPOSE) up -d --wait api
|
||||
|
||||
# --- Собственные тесты (student-tests/) ---
|
||||
|
||||
test-own:
|
||||
$(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
|
||||
test-docker:
|
||||
$(TEST_ENV) DOCKER_TESTS=1 $(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
|
||||
test-limits:
|
||||
$(PYTHON) -m pytest student-tests/test_limits.py -q
|
||||
|
||||
test-concurrency:
|
||||
$(PYTHON) -m pytest student-tests/test_concurrency.py -q
|
||||
|
||||
test-migrations:
|
||||
$(PYTHON) -m pytest student-tests/test_migrations.py -q
|
||||
|
||||
test-all: test-stand-up test-docker test-stand-smoke test-stand-persistence
|
||||
|
||||
# --- Демо (на тестовом стенде) ---
|
||||
|
||||
explain:
|
||||
$(TEST_ENV) DB_HOST=localhost DB_PORT=$(TEST_DB_PORT) PYTHONPATH=. $(PYTHON) scripts/explain_history.py
|
||||
|
||||
expiry-demo: test-stand-up
|
||||
$(TEST_ENV) SESSION_TTL_SECONDS=3 $(COMPOSE) up -d api
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
BASE_URL=$(TEST_BASE_URL) TTL=3 scripts/expiry_demo.sh
|
||||
$(TEST_COMPOSE) up -d api
|
||||
$(call wait_ready,$(TEST_BASE_URL))
|
||||
|
||||
failure-demo: test-stand-up
|
||||
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
|
||||
|
||||
# --- Очередь lab5 ---
|
||||
.PHONY: worker-stop worker-start jobs-logs
|
||||
worker-stop:
|
||||
$(COMPOSE) stop worker
|
||||
|
||||
worker-start:
|
||||
$(COMPOSE) start worker
|
||||
|
||||
jobs-logs:
|
||||
$(COMPOSE) logs -f publisher worker broker
|
||||
|
||||
# Уже работающий изолированный стенд; сам цель его не поднимает.
|
||||
.PHONY: test-outbox-recovery
|
||||
test-outbox-recovery:
|
||||
test-outbox:
|
||||
$(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-jobs
|
||||
test-jobs:
|
||||
$(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)"
|
||||
$(PYTHON) -m scripts.check_jobs --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(or $(ACTION_SCRIPT),scripts/restart.sh)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(or $(ACTION_SCRIPT),scripts/stop-one.sh)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# Отдельные PostgreSQL/Redis нужны для локальных тестов; тестовая БД создаётся явно.
|
||||
test-stand-up:
|
||||
$(TEST_COMPOSE) up -d --build --wait --wait-timeout 120
|
||||
$(TEST_COMPOSE) exec -T postgres sh -c 'psql -U "$$POSTGRES_USER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '\''$${POSTGRES_DB}_test'\''" | grep -q 1 || createdb -U "$$POSTGRES_USER" "$${POSTGRES_DB}_test"'
|
||||
|
||||
test-stand-destroy:
|
||||
$(TEST_COMPOSE) down --volumes --remove-orphans
|
||||
|
||||
# Проверки текущего pipeline; полный регрессионный набор и отказы API — отдельные цели.
|
||||
test-suite:
|
||||
@set -eu; trap '$(TEST_COMPOSE) down --volumes --remove-orphans' EXIT; \
|
||||
$(MAKE) check validate; \
|
||||
$(MAKE) test-stand-up; \
|
||||
$(TEST_ENV) $(MAKE) test-pipeline; \
|
||||
$(TEST_ENV) $(MAKE) test BASE_URL="$(TEST_BASE_URL)"; \
|
||||
$(TEST_ENV) $(MAKE) test-storage BASE_URL="$(TEST_BASE_URL)"; \
|
||||
$(TEST_ENV) $(MAKE) test-outbox BASE_URL="$(TEST_BASE_URL)"; \
|
||||
$(TEST_ENV) $(MAKE) test-jobs BASE_URL="$(TEST_BASE_URL)";
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
Автор: Григорян Илья Мурадович, БПИ2303.
|
||||
Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026.
|
||||
Дальнейшая цель — 5; исследовательский трек ещё не выбран.
|
||||
Использована помощь Codex при реализации, проверках и исправлениях.
|
||||
Commit/PR: заполнить после публикации.
|
||||
- Автор, группа: Григорян Илья Мурадович, БПИ2303.
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **5**, трек — **TODO**. Все пункты уровней **3 и 4** реализованы и проверены.
|
||||
- Commit/PR: TODO после публикации.
|
||||
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI, SQLAlchemy, Pillow, aio-pika; [RabbitMQ](https://www.rabbitmq.com/docs/reliability), [Pillow](https://pillow.readthedocs.io/en/stable/); помощь Codex при реализации и проверках.
|
||||
|
||||
## Запуск
|
||||
## Запуск из чистого clone
|
||||
|
||||
Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt.
|
||||
Нужны Python 3.12, pip и Docker Compose. Версии зависимостей закреплены в requirements.txt;
|
||||
основные образы — PostgreSQL 16.10, Redis 7.4.2, RabbitMQ 4.4.0 и nginx 1.28.0.
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
@@ -16,210 +16,83 @@ python3 -m venv .venv
|
||||
make init
|
||||
make up
|
||||
make test
|
||||
make down
|
||||
```
|
||||
|
||||
`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие.
|
||||
Все environment-параметры Compose подставляются из .env; образец — .env.example.
|
||||
Пароли и S3 credentials читаются из файлов secrets, не включаются в задания.
|
||||
API не получает credentials брокера. По умолчанию HTTP доступен на
|
||||
http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты.
|
||||
`make init` создаёт отсутствующие .env и secrets/, миграции и bootstrap выполняются
|
||||
при запуске. Все настройки берутся из .env; образец — .env.example.
|
||||
HTTP: http://localhost:18081. Внутренние хранилища и брокер наружу не открыты.
|
||||
`make down` сохраняет данные; `docker compose down --volumes` намеренно удаляет их.
|
||||
При обновлении схемы сначала остановить старые процессы, сохранив volumes.
|
||||
|
||||
При обновлении ранее запущенного стенда сначала остановить старые процессы
|
||||
через `make down` без удаления volumes, затем выполнить `make up`.
|
||||
Migrate применяет ревизии до a62c950e18b3 перед стартом приложения.
|
||||
Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые
|
||||
вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP.
|
||||
При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs
|
||||
из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы
|
||||
обоих брокеров во время перехода должны быть остановлены.
|
||||
## Архитектура и решения
|
||||
|
||||
## Инфраструктура и устойчивый приём
|
||||
API → приватный MinIO staging + PostgreSQL job/outbox → RabbitMQ → worker →
|
||||
конечные объекты MinIO + PostgreSQL metadata. Redis хранит сессии.
|
||||
Сохранены аутентификация, чаты, сообщения, пагинация и идемпотентность предыдущих работ.
|
||||
Загрузка возвращает **202 queued** после устойчивого приёма. Состояния:
|
||||
queued / processing / ready / failed; скачивание до ready возвращает 409.
|
||||
|
||||
| Компонент | Реализация | Данные |
|
||||
| --- | --- | --- |
|
||||
| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий |
|
||||
| Балансировка | nginx 1.28.0 | единая HTTP-точка входа |
|
||||
| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit |
|
||||
| Сессии | Redis 7.4.2, AOF always | redisdata |
|
||||
| Объекты | приватный MinIO | staging, originals, variants |
|
||||
| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata |
|
||||
| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ |
|
||||
| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL |
|
||||
| Cleanup | отдельный процесс | незавершённые upload и terminal staging |
|
||||
RabbitMQ использует устойчивую очередь, сохранение сообщений на диск и подтверждения.
|
||||
Publisher отмечает публикацию после подтверждения брокера; worker подтверждает доставку
|
||||
после фиксации результата. Outbox восстанавливает разрыв COMMIT → publish.
|
||||
Фиксированные ключи объектов, блокировка job и lease защищают от повторов и устаревшего worker.
|
||||
|
||||
Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP.
|
||||
Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером
|
||||
приёма сообщения (publisher confirm) и только затем фиксирует published_attempt.
|
||||
При неизвестном результате публикации возможен дубль; намерение остаётся в БД.
|
||||
Подтверждение worker (consumer ack) следует после устойчивого результата или
|
||||
устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ
|
||||
возвращает неподтверждённые доставки в очередь.
|
||||
Original не перекодируется. Варианты: JPEG thumbnail ≤256×256, центральный JPEG crop
|
||||
128×128 и WebP ≤1280×1280. JPEG quality=85, WebP quality=80/method=4; EXIF orientation
|
||||
учитывается, EXIF/ICC/XMP удаляются. Для JPEG alpha заменяется белой подложкой,
|
||||
WebP сохраняет прозрачность. Лимиты: 10 MiB, 40 млн пикселей, timeout 60 секунд,
|
||||
768 MiB и 20 секунд CPU на дочерний процесс; concurrency=2, prefetch=1, lease=90 секунд.
|
||||
|
||||
До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет
|
||||
его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL.
|
||||
Job одновременно служит transactional outbox — устойчивым намерением публикации.
|
||||
Worker и broker не входят в синхронный путь upload и readiness API. Остановка
|
||||
worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT
|
||||
API проверяет БД, чтобы не удалить оригинал уже принятого задания.
|
||||
Повторы ограничены пятью попытками. Задержка со случайным разбросом:
|
||||
c=min(120, 5×2^(n−1)), delay∈[c/2,c]. Постоянные ошибки завершаются сразу.
|
||||
Реестр failed и аудит находятся в PostgreSQL; явный replay проверяет исходник,
|
||||
выделяет новый бюджет и не запускает уже готовую job. Staging активных jobs сохраняется;
|
||||
ready очищается через сутки, failed — через семь суток.
|
||||
|
||||
Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id,
|
||||
pipeline_version=1, attempt, created_at, schema_version, type и необязательный
|
||||
trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются.
|
||||
Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409.
|
||||
Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены.
|
||||
|
||||
## Изображения и ограничения
|
||||
|
||||
Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без
|
||||
перекодирования. Все варианты создаются в отдельном ограниченном процессе:
|
||||
|
||||
| Вариант | Формат и размеры | Параметры |
|
||||
| --- | --- | --- |
|
||||
| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha |
|
||||
| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается |
|
||||
| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется |
|
||||
|
||||
Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся
|
||||
в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA.
|
||||
Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256.
|
||||
|
||||
Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60;
|
||||
IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс.
|
||||
Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс;
|
||||
S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией.
|
||||
|
||||
## Повторы, идемпотентность и очистка
|
||||
|
||||
Состояния: queued → processing → ready; временный отказ возвращает queued,
|
||||
постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code.
|
||||
JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit.
|
||||
Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS,
|
||||
JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling].
|
||||
База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД;
|
||||
дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при
|
||||
публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные;
|
||||
timeout, временные ошибки S3/сети и файловой системы повторяются.
|
||||
|
||||
Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает
|
||||
истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job,
|
||||
проверяет актуальность token и lease, держит блокировку до записи original,
|
||||
трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки.
|
||||
Устаревшая попытка не может начать запись. Фиксированные ключи по
|
||||
(attachment_id, pipeline_version, variant) исключают второй набор объектов.
|
||||
Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один
|
||||
раз с идемпотентным эффектом, не exactly-once выполнение.
|
||||
|
||||
WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал.
|
||||
Publisher и worker используют init:true: Python не является PID 1, поэтому
|
||||
fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу
|
||||
без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload.
|
||||
|
||||
Queued/processing staging не удаляется. Для ready retention — 86400 секунд;
|
||||
для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток).
|
||||
Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым
|
||||
original, объект сохраняется. Частичные конечные объекты failed автоматически
|
||||
не удаляются; после replay используются те же ключи.
|
||||
|
||||
## Реестр failed и операторский replay
|
||||
|
||||
Failed registry находится в PostgreSQL. Команды требуют операторского доступа
|
||||
к Docker/credentials стенда; публичного endpoint для replay нет:
|
||||
Операторские команды:
|
||||
|
||||
```sh
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID
|
||||
```
|
||||
|
||||
JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored,
|
||||
configuration_fixed, manual_review. Actor — аудиторская метка, не система входа.
|
||||
Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256,
|
||||
атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не
|
||||
сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id
|
||||
с теми же параметрами возвращает already_applied даже после ready; конфликт
|
||||
параметров отклоняется. После неоднозначного ответа повторяют тот же request-id.
|
||||
Готовую job с новым request-id повторить нельзя. После удаления staging replay
|
||||
отклоняется с source_expired; отсутствующий объект даёт source_missing.
|
||||
JOB_ID и REQUEST_UUID заменить UUID. Повтор команды с тем же request-id безопасен.
|
||||
|
||||
## Проверки уровня 3 и 4
|
||||
## Доказательства критериев
|
||||
|
||||
Полный воспроизводимый прогон на отдельном стенде:
|
||||
`make test-suite` поднимает отдельный стенд, проверяет pipeline и удаляет контейнеры,
|
||||
сети и volumes при завершении, включая ошибку. Полная сохранённая регрессия запускается
|
||||
отдельно через `make test-unit`; отдельные сервисы — `test-nginx` и `test-storage`.
|
||||
Проверки pipeline: 42 теста. Полная сохранённая регрессия: 201 тест.
|
||||
Публичные smoke/persistence/failover прошли. Для отдельных pytest-прогонов нужен
|
||||
`make test-stand-up`; после проверки — `make test-stand-destroy`.
|
||||
|
||||
```sh
|
||||
make test-grade4 PYTHON=.venv/bin/python
|
||||
```
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | 202 → ready, неизменный original, JPEG thumbnail | evidence/verification.json |
|
||||
| Устойчивый приём и сохранение возможностей lab4 | DOCKER_TESTS=1 make test-unit; make test-persistence | Данные и сессия сохранены, загрузка асинхронная | evidence/verification.json |
|
||||
| Отдельные broker/worker, поздний ack | make test-jobs | Сообщения сохранены, очередь подтверждена | evidence/jobs.json |
|
||||
| Состояния, ограниченные повторы, идемпотентность | make test-pipeline; make test-jobs | failed после бюджета, дубли не создают новые объекты | evidence/verification.json |
|
||||
| Остановка worker и рестарт API/брокера | make test-jobs | То же вложение стало ready, сообщение пережило рестарт | evidence/jobs.json |
|
||||
| Crop/WebP и ограничения worker | make test-pipeline; make test-jobs | Форматы/размеры, metadata и лимиты проверены | evidence/verification.json |
|
||||
| Разрыв COMMIT → publish | make test-outbox | SIGKILL publisher, затем ready без нового upload | evidence/verification.json |
|
||||
| Backoff/jitter, failed registry, replay | make test-jobs | Постоянные/временные ошибки различаются, replay безопасен | evidence/jobs.json |
|
||||
| Crash после PUT до ack, lease и cleanup | make test-jobs; make test-pipeline | Тот же результат, prefetch=1; активный staging сохранён | evidence/jobs.json; evidence/verification.json |
|
||||
| Выбранный трек 5 | TODO | TODO | TODO |
|
||||
|
||||
Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные
|
||||
тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две
|
||||
реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД
|
||||
для pytest выполняется только в выделенной базе *_test, Redis — DB15.
|
||||
Публичные tests/ и обязательные contracts/ не изменены.
|
||||
## Эксперименты и отказы
|
||||
|
||||
Отдельные сценарии на уже работающем изолированном стенде:
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| Crash после PUT до COMMIT | Повтор той же job | ready на второй попытке, те же четыре объекта и SHA-256 | После lease/retry; потерь нет |
|
||||
| Crash после COMMIT до ack | Готовый дубль без обработки | Metadata и Last-Modified не изменились | После доставки; потерь нет |
|
||||
| Временная / постоянная ошибка | Ограниченный retry / немедленный failed | Пять / одна попытка; явный replay завершился ready | В пределах бюджета; потерь нет |
|
||||
|
||||
```sh
|
||||
make test-outbox-recovery BASE_URL=http://localhost:28080
|
||||
make test-jobs BASE_URL=http://localhost:28080
|
||||
```
|
||||
Трек 5 — **TODO**: гипотеза, методика, измерения и выводы.
|
||||
|
||||
При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты
|
||||
в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта,
|
||||
создают синтетические данные и возвращают штатные worker/publisher в finally.
|
||||
Перезапуски сохраняют volumes. Сброса пользовательских данных нет.
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json;
|
||||
подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json.
|
||||
Секреты, tokens и исходные broker payload в evidence не записываются.
|
||||
|
||||
## Фактический результат проверок
|
||||
|
||||
Все перечисленные проверки завершились успешно на отдельном проекте
|
||||
`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes:
|
||||
|
||||
| Проверка | Результат |
|
||||
| --- | --- |
|
||||
| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков |
|
||||
| Дополнительные timeout и failed retention/SHA-256 | 2 passed |
|
||||
| Контракт, OpenAPI и Compose config | PASS |
|
||||
| Публичный smoke | PASS |
|
||||
| 24 конкурентных повтора сообщения через две API-реплики | PASS |
|
||||
| Публичные persistence и failover | PASS |
|
||||
| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload |
|
||||
| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume |
|
||||
| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов |
|
||||
| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified |
|
||||
| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен |
|
||||
| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён |
|
||||
| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 |
|
||||
| Неизменность публичных tests/contracts/README и whitespace | PASS |
|
||||
|
||||
Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены
|
||||
в последующие полные запуски make test-grade4. Тесты намеренно меняют только
|
||||
выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях
|
||||
используют собственную БД тестового Compose-проекта.
|
||||
|
||||
Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885,
|
||||
42.633 секунды; все попали в соответствующие интервалы jitter. Задержка
|
||||
последней failed-попытки сохраняется для диагностики, автоматический retry
|
||||
после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно.
|
||||
|
||||
Во время проверки исправлена проблема fault hooks с PID 1 через init:true.
|
||||
Старые собственные проверки lab4 адаптированы к обязательному переходу
|
||||
202 queued → ready; публичные проверки не изменены. Цель make test-grade4
|
||||
переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры
|
||||
не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий.
|
||||
|
||||
## Границы результата
|
||||
|
||||
Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes.
|
||||
PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта.
|
||||
Ожидание в очереди не ограничено; admission control и исследование перегрузки
|
||||
относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение
|
||||
алгоритмов требует решения о новой версии/миграции результатов.
|
||||
|
||||
Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability),
|
||||
[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms),
|
||||
[aio-pika](https://docs.aio-pika.com/quick-start.html),
|
||||
[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html),
|
||||
[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp).
|
||||
Один Docker-хост, без репликации хранилищ и защиты от потери диска. Очередь не имеет
|
||||
admission control; pipeline фиксирован на версии 1. Доставка как минимум один раз,
|
||||
с идемпотентным результатом. Исследование для уровня 5 — **TODO**.
|
||||
|
||||
@@ -29,7 +29,7 @@ def validate_idempotency_key(key: str | None) -> str:
|
||||
def request_fingerprint(new_message: CreateMessage) -> str:
|
||||
"""Хеширует нормализованное содержимое запроса."""
|
||||
body = new_message.model_dump(mode="json")
|
||||
# Сохраняет отпечатки текстовых запросов lab3 и равенство absent / [].
|
||||
# Сохраняет прежние отпечатки текстовых запросов и равенство absent / [].
|
||||
if not body["attachment_ids"]:
|
||||
body.pop("attachment_ids")
|
||||
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
name: mtusi-messenger-lab5
|
||||
name: messenger
|
||||
|
||||
x-api-image: &api-image
|
||||
build:
|
||||
@@ -73,7 +73,7 @@ services:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: server
|
||||
image: mtusi-messenger-minio:lab5
|
||||
image: messenger-minio:local
|
||||
command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
@@ -104,7 +104,7 @@ services:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: bootstrap
|
||||
image: mtusi-messenger-minio-bootstrap:lab5
|
||||
image: messenger-minio-bootstrap:local
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
|
||||
S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"results": [
|
||||
{
|
||||
"scenario": "worker_stopped_restart_all",
|
||||
"attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707",
|
||||
"attachment_id": "f5cc75ba-d2c7-43ff-8cf4-123b723118b8",
|
||||
"queue_before": {
|
||||
"name": "attachment-jobs",
|
||||
"messages_unacknowledged": 0,
|
||||
@@ -19,13 +19,13 @@
|
||||
"messages_ready": 1
|
||||
},
|
||||
"after": {
|
||||
"job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf",
|
||||
"job_id": "9adcea8e-9046-4736-9763-c24bb6daf417",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:27:44.556734+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:26:56.662360+00:00",
|
||||
"updated_at": "2026-11-10 04:36:45.543612+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:35:56.727228+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -33,41 +33,41 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:27:44.548000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:45.525000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:27:44.552000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:45.533000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:27:44.543000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:45.521000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:27:44.536000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:45.508000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "crash-after-object",
|
||||
"attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc",
|
||||
"attachment_id": "1927c264-c721-4de1-8914-32d9beab48e5",
|
||||
"exit_code": 137,
|
||||
"before": {
|
||||
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
|
||||
"job_id": "bda1d00d-41c7-4d28-b2e5-07759be55985",
|
||||
"status": "processing",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:27:49.144648+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:27:47.218612+00:00",
|
||||
"updated_at": "2026-11-10 04:36:50.666631+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:36:48.673346+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -75,36 +75,36 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:27:49.912000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:51.443000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:27:49.917000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:51.448000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:27:49.907000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:51.434000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:27:49.894000+00:00"
|
||||
"last_modified": "2026-11-10 04:36:51.420000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
|
||||
"job_id": "bda1d00d-41c7-4d28-b2e5-07759be55985",
|
||||
"status": "ready",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:24.765539+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:23.142176+00:00",
|
||||
"updated_at": "2026-11-10 04:38:26.694238+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:38:25.019759+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -112,41 +112,41 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:24.750000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:26.675000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:24.758000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:26.686000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:24.742000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:26.666000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:24.725000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:26.656000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "crash-before-ack",
|
||||
"attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1",
|
||||
"attachment_id": "e02d817d-9615-4058-ae5c-63ab895aa6f5",
|
||||
"exit_code": 137,
|
||||
"before": {
|
||||
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
|
||||
"job_id": "9bfea78c-ce05-4a00-bf76-a2213ed3a8df",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:29.833142+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
|
||||
"updated_at": "2026-11-10 04:38:32.063984+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:38:29.436074+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -154,36 +154,36 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:29.824000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.054000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:29.829000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.059000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:29.818000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.048000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:29.805000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.032000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
|
||||
"job_id": "9bfea78c-ce05-4a00-bf76-a2213ed3a8df",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:29.833142+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
|
||||
"updated_at": "2026-11-10 04:38:32.063984+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:38:29.436074+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -191,102 +191,102 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:29.824000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.054000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:29.829000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.059000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:29.818000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.048000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:29.805000+00:00"
|
||||
"last_modified": "2026-11-10 04:38:32.032000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "fail-transient",
|
||||
"attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea",
|
||||
"attachment_id": "a9c15ce8-cacf-43c0-8373-8e3ff5fa23aa",
|
||||
"samples": [
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "queued",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:45.863494+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:50.476742+00:00",
|
||||
"updated_at": "2026-11-10 04:38:46.283315+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:38:50.994899+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "queued",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:51.216904+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:58.053724+00:00",
|
||||
"updated_at": "2026-11-10 04:38:51.039278+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:38:59.940067+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "queued",
|
||||
"attempt": 3,
|
||||
"published_attempt": 3,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:58.279470+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:15.602067+00:00",
|
||||
"updated_at": "2026-11-10 04:39:00.122276+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:39:19.754304+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "queued",
|
||||
"attempt": 4,
|
||||
"published_attempt": 4,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:30:16.386336+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:47.271081+00:00",
|
||||
"updated_at": "2026-11-10 04:39:20.250087+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:39:56.121121+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "failed",
|
||||
"attempt": 5,
|
||||
"published_attempt": 5,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:30:47.622716+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:30.255216+00:00",
|
||||
"updated_at": "2026-11-10 04:39:56.526646+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:41:15.809038+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
}
|
||||
],
|
||||
"after": {
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"job_id": "ffa9abca-645a-4ee2-ab2a-7bd06a7e0a59",
|
||||
"status": "ready",
|
||||
"attempt": 6,
|
||||
"published_attempt": 6,
|
||||
"limit": 10,
|
||||
"updated_at": "2026-11-10 03:30:59.696883+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:53.135190+00:00",
|
||||
"updated_at": "2026-11-10 04:40:09.651133+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:40:03.031585+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -294,25 +294,25 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:30:59.687000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:09.642000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:30:59.692000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:09.647000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:30:59.683000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:09.637000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:30:59.670000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:09.624000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -320,29 +320,29 @@
|
||||
},
|
||||
{
|
||||
"scenario": "fail-permanent",
|
||||
"attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9",
|
||||
"attachment_id": "d8a6e312-e468-4e4b-a7ee-1d84751d5843",
|
||||
"samples": [
|
||||
{
|
||||
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
|
||||
"job_id": "0da2ef52-65a8-4a74-8d43-015666e2975c",
|
||||
"status": "failed",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:31:12.333162+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:16.827024+00:00",
|
||||
"updated_at": "2026-11-10 04:40:24.826416+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:40:29.630307+00:00",
|
||||
"last_error_code": "injected_permanent",
|
||||
"permanent": true,
|
||||
"objects": []
|
||||
}
|
||||
],
|
||||
"after": {
|
||||
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
|
||||
"job_id": "0da2ef52-65a8-4a74-8d43-015666e2975c",
|
||||
"status": "ready",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 6,
|
||||
"updated_at": "2026-11-10 03:31:24.282794+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:17.137221+00:00",
|
||||
"updated_at": "2026-11-10 04:40:40.004814+00:00",
|
||||
"next_attempt_at": "2026-11-10 04:40:30.084006+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
@@ -350,25 +350,25 @@
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:31:24.272000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:39.988000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:31:24.278000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:39.996000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:31:24.267000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:39.982000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:31:24.254000+00:00"
|
||||
"last_modified": "2026-11-10 04:40:39.968000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -157,7 +157,7 @@
|
||||
{
|
||||
"name": "worker_fault_retry_replay",
|
||||
"exit_code": 0,
|
||||
"evidence": "lab5-jobs.json",
|
||||
"evidence": "jobs.json",
|
||||
"scenarios": [
|
||||
"worker_stopped_restart_all",
|
||||
"crash-after-object",
|
||||
@@ -267,5 +267,42 @@
|
||||
"Single Docker host, no disk-loss or cluster failover claim",
|
||||
"Grade 5 experiments not performed",
|
||||
"No production-scale performance/security claim"
|
||||
]
|
||||
],
|
||||
"post_test_cleanup": {
|
||||
"date": "2026-11-10",
|
||||
"project_removed": "mtusi-lab5-verification",
|
||||
"test_volumes_removed": 4,
|
||||
"conserver_removed": true,
|
||||
"old_lab3_image_removed": true,
|
||||
"old_empty_networks_removed": true,
|
||||
"current_lab5_images_retained": true
|
||||
},
|
||||
"maintenance_checks": {
|
||||
"status": "passed",
|
||||
"command": "make test-suite",
|
||||
"project": "messenger-test",
|
||||
"pipeline_tests": 42,
|
||||
"checks": [
|
||||
"contract",
|
||||
"OpenAPI",
|
||||
"Compose startup",
|
||||
"public smoke",
|
||||
"MinIO private access/bootstrap",
|
||||
"outbox recovery",
|
||||
"worker_stopped_restart_all",
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent"
|
||||
],
|
||||
"standalone_nginx": "passed",
|
||||
"test_resources_removed": true,
|
||||
"obsolete_image_tags_removed": [
|
||||
"mtusi-messenger-minio-bootstrap:lab5",
|
||||
"mtusi-messenger-minio:lab5",
|
||||
"mtusi-messenger:lab5"
|
||||
],
|
||||
"regression_tests_preserved": true,
|
||||
"note": "Earlier recorded commands retain their original names as historical evidence; operational scripts and resource names are now neutral."
|
||||
}
|
||||
}
|
||||
+13
-24
@@ -1,27 +1,16 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура
|
||||
|
||||
Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis
|
||||
только на localhost. Nginx направляет запросы на две API-реплики.
|
||||
compose.yaml запускает две API-реплики за nginx, PostgreSQL, Redis сессий,
|
||||
RabbitMQ, приватный MinIO, publisher, worker и cleanup. Все environment-настройки
|
||||
берутся из .env; credentials передаются файлами secrets. Миграции и bootstrap
|
||||
завершаются до старта зависимых сервисов.
|
||||
|
||||
PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2
|
||||
с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь,
|
||||
persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata.
|
||||
Broker и MinIO не публикуют внешние порты. API не получает credentials брокера.
|
||||
Publisher читает outbox PostgreSQL и ждёт подтверждения RabbitMQ. Worker фиксирует
|
||||
результат в MinIO/PostgreSQL до ack; prefetch=1 на consumer. init:true обеспечивает
|
||||
корректные сигналы. Broker и хранилища не публикуют внешние порты.
|
||||
|
||||
Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет
|
||||
original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack.
|
||||
Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого
|
||||
consumer. Publisher/worker используют init:true для корректных сигналов и
|
||||
воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает.
|
||||
|
||||
Все environment-настройки Compose берутся из .env; образец — .env.example.
|
||||
Credentials передаются файлами secrets; make init создаёт только отсутствующие.
|
||||
Migrate и minio-bootstrap завершаются до старта зависимых сервисов.
|
||||
Перед обновлением остановить старые процессы без удаления volumes. Миграция
|
||||
a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ.
|
||||
|
||||
Restart hook сохраняет volumes и состояние заранее остановленного worker.
|
||||
Failover hook останавливает выбранную API-реплику. Retry, retention, операторские
|
||||
команды, ограничения и результаты проверок описаны в REPORT.md.
|
||||
Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии —
|
||||
make test-outbox-recovery и make test-jobs на выбранном работающем стенде.
|
||||
make up/down управляют основным проектом messenger и сохраняют volumes.
|
||||
make test-suite использует отдельный messenger-test и удаляет его вместе с
|
||||
volumes и сетями при завершении. compose.test.yaml открывает тестовые PostgreSQL
|
||||
и Redis только на localhost. Полная регрессия доступна через make test-unit,
|
||||
отдельные проверки — make test-nginx, test-storage, test-outbox и test-jobs.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации."""
|
||||
"""Состояния, варианты и устойчивые jobs с намерением публикации."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений."""
|
||||
"""Ключи crop/WebP и дообработка старых ready-вложений."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
|
||||
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from scripts.failover_all import exercise, failure_kind, history
|
||||
from scripts.rolling_update import update
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
PROBE = """import json, sys, urllib.request, urllib.error
|
||||
payload = json.load(sys.stdin)
|
||||
results = []
|
||||
for item in payload:
|
||||
data = json.dumps(item['body']).encode() if 'body' in item else None
|
||||
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=7)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
results.append({'status': response.status, 'body': json.loads(response.read())})
|
||||
print(json.dumps(results))
|
||||
"""
|
||||
|
||||
|
||||
def probe(project, runner, requests):
|
||||
"""Обращается к внутреннему API, передавая токен только через stdin."""
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", runner, "python", "-c", PROBE],
|
||||
input=json.dumps(requests),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
check(result.returncode == 0, "internal HTTP probe failed")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def audit(project):
|
||||
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
|
||||
allowed = {
|
||||
"postgres": {"db_password"},
|
||||
"redis": {"redis_password"},
|
||||
"migrate": {"db_password"},
|
||||
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"minio": {"minio_root_user", "minio_root_password"},
|
||||
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"nginx": set(),
|
||||
}
|
||||
ids = project.run("ps", "--all", "--quiet").splitlines()
|
||||
containers = json.loads(project.docker("inspect", *ids))
|
||||
for item in containers:
|
||||
service = item["Config"]["Labels"]["com.docker.compose.service"]
|
||||
check(service in allowed, "unexpected service")
|
||||
host = item["HostConfig"]
|
||||
check(
|
||||
host["Memory"] > 0 and host["NanoCpus"] > 0,
|
||||
service + ": missing CPU/RAM limit",
|
||||
)
|
||||
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
|
||||
check(
|
||||
not env_names
|
||||
& {
|
||||
"DB_PASSWORD",
|
||||
"REDIS_PASSWORD",
|
||||
"POSTGRES_PASSWORD",
|
||||
"REDISCLI_AUTH",
|
||||
"DATABASE_URL",
|
||||
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
|
||||
},
|
||||
service + ": password in environment",
|
||||
)
|
||||
secrets = {
|
||||
Path(mount["Destination"]).name
|
||||
for mount in item["Mounts"]
|
||||
if mount["Destination"].startswith("/run/secrets/")
|
||||
}
|
||||
check(secrets == allowed[service], service + ": incorrect secret scope")
|
||||
if service in {"api", "minio"}:
|
||||
check(not host["PortBindings"], service + " publishes ports")
|
||||
for mount in item["Mounts"]:
|
||||
if mount["Destination"].startswith("/run/secrets/"):
|
||||
check(not mount["RW"], "writable secret mount")
|
||||
check(
|
||||
"container_name" not in project.config["services"][service],
|
||||
"fixed container name",
|
||||
)
|
||||
check(
|
||||
len(containers) == len(allowed) - 1 + len(project.replicas()),
|
||||
"unexpected container count",
|
||||
)
|
||||
print(
|
||||
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
async def graceful(project, base_url):
|
||||
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
|
||||
replicas = project.replicas()
|
||||
target, runner = replicas[:2]
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
raw = json.dumps([actor["id"], "POST", path, key]).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
||||
database = project.config["services"]["postgres"]
|
||||
env = database["environment"]
|
||||
conn = await asyncpg.connect(
|
||||
host="127.0.0.1",
|
||||
port=int(database["ports"][0]["published"]),
|
||||
database=env["POSTGRES_DB"],
|
||||
user=env["POSTGRES_USER"],
|
||||
password=(ROOT / "secrets/db_password").read_text().strip(),
|
||||
timeout=3,
|
||||
)
|
||||
network = project.config['networks']['default']['name']
|
||||
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
|
||||
address = project.docker(
|
||||
"inspect",
|
||||
"--format",
|
||||
address_template,
|
||||
target.id,
|
||||
)
|
||||
check(bool(address), 'API has no address in the application network')
|
||||
port = project.config["services"]["api"]["environment"]["APP_PORT"]
|
||||
direct = f"http://{address}:{port}"
|
||||
started = time.monotonic()
|
||||
task = None
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{
|
||||
"url": direct + path,
|
||||
"method": "POST",
|
||||
"body": {"text": "graceful-" + key},
|
||||
"headers": {
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Idempotency-Key": key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
deadline = time.monotonic() + 3
|
||||
while not await conn.fetchval(
|
||||
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
|
||||
):
|
||||
check(time.monotonic() < deadline, "POST did not reach database lock")
|
||||
await asyncio.sleep(0.02)
|
||||
started = time.monotonic()
|
||||
await asyncio.to_thread(
|
||||
project.docker, "kill", "--signal", "SIGTERM", target.id
|
||||
)
|
||||
results = await asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{"url": direct + "/health/ready"},
|
||||
{"url": direct + "/health/live"},
|
||||
{"url": direct + "/api/v1/users/me"},
|
||||
],
|
||||
)
|
||||
check(
|
||||
[item["status"] for item in results] == [503, 200, 503],
|
||||
"incorrect drain readiness/liveness/new work",
|
||||
)
|
||||
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
|
||||
result = (await task)[0]
|
||||
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
|
||||
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
|
||||
state = json.loads(
|
||||
project.docker("inspect", "--format", "{{json .State}}", target.id)
|
||||
)
|
||||
check(
|
||||
not state["OOMKilled"] and state["ExitCode"] != 137,
|
||||
"forced shutdown instead of graceful completion",
|
||||
)
|
||||
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
|
||||
items = await asyncio.to_thread(history, client, actor, chat)
|
||||
check(
|
||||
result["body"]["id"] in {item["id"] for item in items},
|
||||
"acknowledged in-flight message lost",
|
||||
)
|
||||
print(
|
||||
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
if task:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
if any(item.id == target.id and item.running for item in project.replicas()):
|
||||
await asyncio.to_thread(project.docker, "stop", target.id)
|
||||
await asyncio.to_thread(project.docker, "start", target.id)
|
||||
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
|
||||
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
|
||||
|
||||
|
||||
class ObservedProject(ComposeProject):
|
||||
"""Подсчитывает миграции и проверяет готовность перед заменой."""
|
||||
|
||||
def __init__(self):
|
||||
self.migrations = 0
|
||||
self.replacements = 0
|
||||
self.replacement_started = []
|
||||
super().__init__()
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
|
||||
self.migrations += 1
|
||||
return super().run(*args, **kwargs)
|
||||
|
||||
def docker(self, *args, **kwargs):
|
||||
if args[0] == "stop":
|
||||
healthy = [
|
||||
item
|
||||
for item in self.replicas()
|
||||
if item.id != args[1] and item.running and item.health == "healthy"
|
||||
]
|
||||
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
|
||||
self.replacements += 1
|
||||
self.replacement_started.append(time.monotonic())
|
||||
return super().docker(*args, **kwargs)
|
||||
|
||||
|
||||
def rolling(base_url):
|
||||
"""Заменяет образ под постоянной записью с проверкой сохранности."""
|
||||
project = ObservedProject()
|
||||
before = project.replicas()
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
stop = Event()
|
||||
results, errors = [], []
|
||||
|
||||
def worker():
|
||||
local = Client(base_url, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "rolling-" + key, key=key)
|
||||
results.append(
|
||||
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
|
||||
)
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
errors.append(failure_kind(error))
|
||||
stop.wait(0.08)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(2)]
|
||||
try:
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
update(project, base_url, revision="verification-" + uuid4().hex[:12])
|
||||
finished = time.monotonic()
|
||||
time.sleep(1)
|
||||
finally:
|
||||
stop.set()
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
after = project.replicas()
|
||||
check(
|
||||
project.migrations == 1 and project.replacements == len(before),
|
||||
"migration/replacement count incorrect",
|
||||
)
|
||||
check(
|
||||
{item.image for item in before}.isdisjoint({item.image for item in after}),
|
||||
"new image was not deployed",
|
||||
)
|
||||
check(
|
||||
{item.id for item in before}.isdisjoint({item.id for item in after}),
|
||||
"old containers remain",
|
||||
)
|
||||
writes = [row for row in results if started <= row[0] <= finished]
|
||||
check(bool(writes), "no writes during update")
|
||||
check(
|
||||
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
|
||||
"no writes during container replacement",
|
||||
)
|
||||
items = history(client, actor, chat)
|
||||
check(
|
||||
{row[2] for row in results} <= {item["id"] for item in items},
|
||||
"rolling update lost confirmed messages",
|
||||
)
|
||||
check(
|
||||
len(items) == len({item["text"] for item in items}),
|
||||
"rolling update created duplicate messages",
|
||||
)
|
||||
latencies = sorted(row[3] for row in writes)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"replacements": project.replacements,
|
||||
"migration_runs": project.migrations,
|
||||
"successful_writes": len(writes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(finished - started, 3),
|
||||
"successful_rps": round(len(writes) / (finished - started), 3),
|
||||
"p95_seconds": round(
|
||||
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
|
||||
),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Выполняет проверки только на отдельном тестовом проекте."""
|
||||
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--phase", choices=("all", "update-and-failover"), default="all"
|
||||
)
|
||||
|
||||
args = arguments("Grade 4 integration checks", configure)
|
||||
project = ComposeProject()
|
||||
check(project.name.endswith("-test"), "use isolated test Compose project")
|
||||
with deployment_lock(project):
|
||||
audit(project)
|
||||
if args.phase == "all":
|
||||
asyncio.run(graceful(project, args.base_url))
|
||||
rolling(args.base_url)
|
||||
args.duration, args.concurrency = 10, 2
|
||||
exercise(project, args)
|
||||
audit(project)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,39 +0,0 @@
|
||||
"""Проверяет конкурентные повторы через балансировщик."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Отправляет один ключ параллельно через разные реплики."""
|
||||
args = arguments("Concurrent idempotency through load balancer")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
body = "parallel-" + key
|
||||
|
||||
def send(_):
|
||||
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send, range(24)))
|
||||
ids = {result["json"]["id"] for result in results}
|
||||
replicas = {result["headers"]["X-Instance-Id"] for result in results}
|
||||
check(len(ids) == 1, "parallel retries created duplicate messages")
|
||||
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
|
||||
history = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
check(
|
||||
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
|
||||
)
|
||||
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
@@ -79,7 +79,7 @@ def verify_result(client, actor, attachment, original):
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='lab5-worker-fault-'+uuid4().hex
|
||||
name='messenger-worker-fault-'+uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
@@ -106,7 +106,7 @@ def crash(client, actor, chat, mode):
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='lab5-retry-fault-'+uuid4().hex
|
||||
name='messenger-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
@@ -189,10 +189,11 @@ def persistence(client,actor,chat):
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/lab5-jobs.json')
|
||||
parser.add_argument('--output',default='evidence/jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,timeout=40)
|
||||
client=Client(args.base_url,args.ca_file,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
@@ -25,7 +25,7 @@ def check(condition: bool, message: str) -> None:
|
||||
class Stand:
|
||||
def __init__(self, directory: str) -> None:
|
||||
self.directory = Path(directory)
|
||||
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
|
||||
self.project = "messenger-nginx-check-" + uuid4().hex[:12]
|
||||
self.base_url = ""
|
||||
backend = self.directory / "backend.conf"
|
||||
backend.write_text(
|
||||
@@ -246,7 +246,7 @@ def run_checks(stand: Stand) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
|
||||
with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory:
|
||||
stand = Stand(directory)
|
||||
try:
|
||||
run_checks(stand)
|
||||
@@ -254,7 +254,7 @@ def main() -> None:
|
||||
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
|
||||
raise
|
||||
finally:
|
||||
stand.compose("down", "--remove-orphans", "--timeout", "1")
|
||||
stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -34,7 +34,7 @@ def main() -> None:
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
@@ -42,7 +42,7 @@ def main() -> None:
|
||||
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "lab5-outbox-fault-" + uuid4().hex
|
||||
fault_container = "messenger-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
|
||||
+4
-20
@@ -1,8 +1,6 @@
|
||||
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
|
||||
|
||||
import json
|
||||
import base64
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
@@ -70,12 +68,15 @@ asyncio.run(main())
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
|
||||
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
|
||||
args = arguments('MinIO private bucket and failure checks', configure)
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call('GET', '/health/live', schema='Health')
|
||||
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
|
||||
for _ in range(2):
|
||||
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
|
||||
@@ -96,21 +97,6 @@ def main():
|
||||
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content'
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
|
||||
if args.with_faults:
|
||||
project.run('stop', 'cleanup')
|
||||
try:
|
||||
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
|
||||
result = subprocess.run(
|
||||
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
|
||||
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
|
||||
'image': base64.b64encode(original).decode()}),
|
||||
text=True, capture_output=True, timeout=60,
|
||||
)
|
||||
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
|
||||
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
|
||||
finally:
|
||||
project.run('start', 'cleanup')
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
|
||||
if args.with_outage:
|
||||
try:
|
||||
project.run('stop', 'minio')
|
||||
@@ -129,8 +115,6 @@ def main():
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
|
||||
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
|
||||
+ (', MinIO outage/recovery' if args.with_outage else ''))
|
||||
if args.with_faults:
|
||||
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from importlib.metadata import version
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT = 'mtusi-messenger-lab5-test'
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
ENVIRONMENT = {
|
||||
**os.environ,
|
||||
'COMPOSE_PROJECT_NAME': PROJECT,
|
||||
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
|
||||
'LB_HOST_PORT': '28080',
|
||||
}
|
||||
COMPOSE = ['docker', 'compose', '--env-file', '.env']
|
||||
NONSECRET_SETTINGS = {
|
||||
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
|
||||
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
|
||||
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
|
||||
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
|
||||
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
|
||||
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
|
||||
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
|
||||
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
|
||||
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
|
||||
'S3_CLEANUP_INTERVAL_SECONDS',
|
||||
}
|
||||
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
|
||||
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
|
||||
|
||||
|
||||
def timestamp():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def invoke(command):
|
||||
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
|
||||
def read_command(command):
|
||||
result = invoke(command)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def source_manifest():
|
||||
names = {
|
||||
'.dockerignore', '.env.example',
|
||||
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
|
||||
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
|
||||
}
|
||||
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
|
||||
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
|
||||
if path.is_file() and '__pycache__' not in path.parts
|
||||
and path.suffix != '.md' and not path.name.startswith('.'))
|
||||
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
|
||||
|
||||
|
||||
def sanitize(text, secrets, staging):
|
||||
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
|
||||
for secret in secrets:
|
||||
clean = clean.replace(secret, '<redacted>')
|
||||
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
|
||||
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
|
||||
return clean
|
||||
|
||||
|
||||
def container_snapshot(images):
|
||||
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
rows = []
|
||||
if ids:
|
||||
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
|
||||
rows = [json.loads(line) for line in output.splitlines()]
|
||||
for row in rows:
|
||||
image = row['image_id']
|
||||
if image not in images:
|
||||
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
|
||||
return rows
|
||||
|
||||
|
||||
def remaining_resources():
|
||||
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
|
||||
f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
for resource in ('container', 'network', 'volume')}
|
||||
|
||||
|
||||
def extract_metrics(output, upload_file):
|
||||
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
|
||||
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
|
||||
if pytest is None:
|
||||
raise RuntimeError('В выводе нет успешного итогового результата pytest')
|
||||
return {
|
||||
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
|
||||
'uploads': json.loads(upload_file.read_text()),
|
||||
'public_failover': [row for row in rows if 'observed_instances_before' in row],
|
||||
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
|
||||
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if any(remaining_resources().values()):
|
||||
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
|
||||
started = timestamp()
|
||||
manifest = source_manifest()
|
||||
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
|
||||
secrets = [value for value in secrets if value]
|
||||
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
|
||||
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
|
||||
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
|
||||
services = {name: {
|
||||
'image': service.get('image'),
|
||||
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
|
||||
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
|
||||
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
|
||||
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
|
||||
'tmpfs': service.get('tmpfs', []),
|
||||
} for name, service in config['services'].items()}
|
||||
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
|
||||
staging = Path(folder)
|
||||
upload_file = staging / 'uploads.json'
|
||||
|
||||
def execute(command):
|
||||
begin = timestamp()
|
||||
result = invoke(command)
|
||||
output = sanitize(result.stdout, secrets, staging)
|
||||
display = sanitize(shlex.join(command), secrets, staging)
|
||||
commands.append({'command': display, 'started_at_utc': begin,
|
||||
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
|
||||
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
|
||||
if result.returncode:
|
||||
print(output[-5000:], flush=True)
|
||||
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
|
||||
return f'$ {display}\n{output}\n'
|
||||
|
||||
cleanup_output = ''
|
||||
try:
|
||||
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['started'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
|
||||
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['before_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_failover'] = container_snapshot(images)
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
|
||||
snapshots['after_update_and_failures'] = container_snapshot(images)
|
||||
nginx_output = execute(['make', 'test-nginx'])
|
||||
finally:
|
||||
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
|
||||
remaining = remaining_resources()
|
||||
if any(remaining.values()):
|
||||
raise RuntimeError('После очистки остались ресурсы тестового проекта')
|
||||
if source_manifest() != manifest:
|
||||
raise RuntimeError('Во время прогона изменились проверяемые исходники')
|
||||
output = ''.join(combined)
|
||||
metrics = extract_metrics(output, upload_file)
|
||||
run = {
|
||||
'started_at_utc': started, 'finished_at_utc': timestamp(),
|
||||
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
|
||||
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
|
||||
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
|
||||
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
|
||||
'project': PROJECT, 'runtime': {'python': platform.python_version(),
|
||||
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
|
||||
'compose': read_command(['docker', 'compose', 'version'])},
|
||||
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
|
||||
'services': services, 'container_snapshots': snapshots, 'images': images,
|
||||
'commands': commands, 'remaining_test_resources': remaining,
|
||||
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
|
||||
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
|
||||
}
|
||||
artifacts = {
|
||||
'test-output.txt': output, 'nginx-output.txt': nginx_output,
|
||||
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
|
||||
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
|
||||
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
|
||||
}
|
||||
for name, content in artifacts.items():
|
||||
if any(secret in content for secret in secrets):
|
||||
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
for name, content in artifacts.items():
|
||||
(EVIDENCE / name).write_text(content)
|
||||
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,17 +1,10 @@
|
||||
"""Общие операции с репликами текущего Compose-проекта."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
@@ -65,43 +58,3 @@ class ComposeProject:
|
||||
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
|
||||
for row in map(json.loads, rows)
|
||||
]
|
||||
|
||||
def wait_healthy(self, ids: set[str], timeout: float = 40) -> None:
|
||||
"""Ожидает готовность конкретных реплик."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
current = {replica.id: replica for replica in self.replicas()}
|
||||
if not ids <= current.keys():
|
||||
raise RuntimeError("Контейнер API исчез во время развёртывания")
|
||||
if all(current[id].running and current[id].health == "healthy" for id in ids):
|
||||
return
|
||||
if any(not current[id].running for id in ids):
|
||||
raise RuntimeError("Контейнер API завершился до готовности")
|
||||
time.sleep(0.25)
|
||||
raise TimeoutError("API не стал healthy за отведённое время")
|
||||
|
||||
def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None:
|
||||
"""Ожидает ответа реплики через балансировщик."""
|
||||
context = ssl.create_default_context(cafile=ca_file)
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response:
|
||||
if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id:
|
||||
return
|
||||
except (HTTPError, URLError, TimeoutError, OSError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError("Реплика не появилась в ответах балансировщика")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def deployment_lock(project: ComposeProject):
|
||||
"""Исключает параллельные изменения одного стенда."""
|
||||
path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock"
|
||||
with path.open("a") as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None
|
||||
yield
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}"
|
||||
API="$BASE_URL/api/v1"
|
||||
USERNAME="expiry_$(date +%s)"
|
||||
PASSWORD="expiry-password-123"
|
||||
|
||||
code() {
|
||||
curl -s -o /dev/null -w '%{http_code}' "$@"
|
||||
}
|
||||
|
||||
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
|
||||
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}"
|
||||
SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")
|
||||
TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])')
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT"
|
||||
echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")"
|
||||
sleep $((TTL + 1))
|
||||
printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")"
|
||||
echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
|
||||
NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
|
||||
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")"
|
||||
@@ -1,132 +0,0 @@
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных.
|
||||
|
||||
Данные создаются в одной транзакции и откатываются в конце: база не меняется.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine
|
||||
|
||||
TARGET_MESSAGES = 10_000
|
||||
NOISE_MESSAGES = 40_000
|
||||
PAGE = 51
|
||||
|
||||
HISTORY_FIRST_PAGE = """
|
||||
SELECT * FROM messages
|
||||
WHERE chat_id = :chat_id
|
||||
ORDER BY created_at, id
|
||||
LIMIT :page
|
||||
"""
|
||||
|
||||
HISTORY_AFTER_CURSOR = """
|
||||
SELECT * FROM messages
|
||||
WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id)
|
||||
ORDER BY created_at, id
|
||||
LIMIT :page
|
||||
"""
|
||||
|
||||
|
||||
async def seed(connection: AsyncConnection) -> dict[str, object]:
|
||||
user_id, target_id, noise_id = uuid4(), uuid4(), uuid4()
|
||||
await connection.execute(
|
||||
text(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) "
|
||||
"VALUES (:id, :username, 'Explain', 'x')"
|
||||
),
|
||||
{"id": user_id, "username": "explain_" + uuid4().hex[:8]},
|
||||
)
|
||||
await connection.execute(
|
||||
text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"),
|
||||
{"a": target_id, "b": noise_id},
|
||||
)
|
||||
await connection.execute(
|
||||
text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"),
|
||||
{"a": target_id, "b": noise_id, "u": user_id},
|
||||
)
|
||||
for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)):
|
||||
await connection.execute(
|
||||
text(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, "
|
||||
"now() - make_interval(secs => :count - g) "
|
||||
"FROM generate_series(1, :count) AS g"
|
||||
),
|
||||
{"chat_id": chat_id, "user_id": user_id, "count": count},
|
||||
)
|
||||
await connection.execute(text("ANALYZE messages"))
|
||||
|
||||
middle = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"SELECT created_at, id FROM messages WHERE chat_id = :c "
|
||||
"ORDER BY created_at, id OFFSET :o LIMIT 1"
|
||||
),
|
||||
{"c": target_id, "o": TARGET_MESSAGES // 2},
|
||||
)
|
||||
).one()
|
||||
return {
|
||||
"chat_id": target_id,
|
||||
"after_created_at": middle.created_at,
|
||||
"after_id": middle.id,
|
||||
"page": PAGE,
|
||||
}
|
||||
|
||||
|
||||
async def explain(
|
||||
connection: AsyncConnection, title: str, query: str, params: dict[str, object]
|
||||
) -> None:
|
||||
rows = await connection.execute(
|
||||
text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params
|
||||
)
|
||||
print(f"== {title}")
|
||||
print(query.strip())
|
||||
print()
|
||||
for row in rows:
|
||||
print(row[0])
|
||||
print()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
engine = create_db_engine(Settings()) # type: ignore[call-arg]
|
||||
async with engine.connect() as connection:
|
||||
transaction = await connection.begin()
|
||||
params = await seed(connection)
|
||||
total = (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one()
|
||||
print(
|
||||
f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}"
|
||||
)
|
||||
print()
|
||||
|
||||
first_page = {"chat_id": params["chat_id"], "page": PAGE}
|
||||
await explain(
|
||||
connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page
|
||||
)
|
||||
await explain(
|
||||
connection,
|
||||
"page after cursor in the middle of history",
|
||||
HISTORY_AFTER_CURSOR,
|
||||
params,
|
||||
)
|
||||
|
||||
await connection.execute(text("SET LOCAL enable_indexscan = off"))
|
||||
await connection.execute(text("SET LOCAL enable_bitmapscan = off"))
|
||||
await explain(
|
||||
connection,
|
||||
"same page with index scans disabled (for comparison)",
|
||||
HISTORY_AFTER_CURSOR,
|
||||
params,
|
||||
)
|
||||
|
||||
await transaction.rollback()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,148 +0,0 @@
|
||||
"""Сценарий поочерёдного отказа всех API-реплик под записью."""
|
||||
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from threading import Event, Lock
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def failure_kind(error: Failure) -> str:
|
||||
"""Различает HTTP-отказы, ошибки соединения и проверки контракта."""
|
||||
match = re.search(r"received (\d{3})$", str(error))
|
||||
if match:
|
||||
return "http_" + match.group(1)
|
||||
return "connection" if "connection/TLS failure" in str(error) else "contract"
|
||||
|
||||
|
||||
def snapshot(project: ComposeProject, phase: str, target: str) -> list:
|
||||
"""Печатает состояние API без окружения и секретов."""
|
||||
replicas = project.replicas()
|
||||
print(json.dumps({
|
||||
"scenario": "replica_failure", "phase": phase, "target": target,
|
||||
"replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas],
|
||||
}), flush=True)
|
||||
return replicas
|
||||
|
||||
|
||||
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
|
||||
"""Читает всю историю без повторов курсора."""
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
||||
items = []
|
||||
cursors = set()
|
||||
for _ in range(1000):
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items
|
||||
check(cursor not in cursors, "cursor cycle")
|
||||
cursors.add(cursor)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}"
|
||||
raise Failure("history exceeded safety bound")
|
||||
|
||||
|
||||
def exercise(project: ComposeProject, args) -> None:
|
||||
"""Останавливает каждую реплику и восстанавливает её перед следующей."""
|
||||
replicas = project.replicas()
|
||||
check(len(replicas) >= 2, "at least two API replicas are required")
|
||||
project.wait_healthy({item.id for item in replicas})
|
||||
client = Client(args.base_url, args.ca_file, timeout=2)
|
||||
for item in replicas:
|
||||
project.wait_routed(item.instance_id, client.base, args.ca_file)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
confirmed = set()
|
||||
|
||||
for target in replicas:
|
||||
snapshot(project, "before", target.instance_id)
|
||||
stop = Event()
|
||||
lock = Lock()
|
||||
successes = []
|
||||
errors = []
|
||||
|
||||
def worker():
|
||||
local = Client(client.base, args.ca_file, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "failover-" + key, key=key)
|
||||
with lock:
|
||||
successes.append((started, time.monotonic(), result["json"]["id"]))
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
with lock:
|
||||
errors.append(failure_kind(error))
|
||||
time.sleep(0.1)
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(args.concurrency)]
|
||||
try:
|
||||
time.sleep(2)
|
||||
stop_started = time.monotonic()
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
||||
env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id},
|
||||
check=True, timeout=10, capture_output=True,
|
||||
)
|
||||
stop_finished = time.monotonic()
|
||||
stopped = snapshot(project, "stopped", target.instance_id)
|
||||
check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped")
|
||||
check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API")
|
||||
time.sleep(args.duration)
|
||||
finally:
|
||||
stop.set()
|
||||
try:
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
finally:
|
||||
project.docker("start", target.id)
|
||||
project.wait_healthy({item.id for item in replicas})
|
||||
project.wait_routed(target.instance_id, client.base, args.ca_file)
|
||||
snapshot(project, "restored", target.instance_id)
|
||||
|
||||
elapsed = max(end for _, end, _ in successes) - started if successes else 0
|
||||
recovered = [end for begin, end, _ in successes if begin >= stop_finished]
|
||||
check(bool(recovered), "no confirmed writes after failure")
|
||||
recovery = min(recovered) - stop_started
|
||||
check(recovery <= 10, "writes did not recover within 10 seconds")
|
||||
confirmed.update(id for _, _, id in successes)
|
||||
items = history(client, actor, chat)
|
||||
check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost")
|
||||
texts = [item["text"] for item in items]
|
||||
check(len(texts) == len(set(texts)), "uncertain retry created duplicates")
|
||||
print(json.dumps({
|
||||
"instance": target.instance_id,
|
||||
"successful_writes": len(successes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0,
|
||||
"first_recovery_seconds": round(recovery, 3),
|
||||
}), flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Готовит параметры эксперимента отказа реплик."""
|
||||
def configure(parser):
|
||||
parser.add_argument("--duration", type=float, default=15)
|
||||
parser.add_argument("--concurrency", type=int, default=2)
|
||||
|
||||
args = arguments("Fail each API replica under write load", configure)
|
||||
check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload")
|
||||
project = ComposeProject()
|
||||
with deployment_lock(project):
|
||||
exercise(project, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
API="$BASE_URL/api/v1"
|
||||
USERNAME="demo_$(date +%s)"
|
||||
PASSWORD="demo-password-123"
|
||||
|
||||
compose() {
|
||||
docker compose --env-file .env "$@"
|
||||
}
|
||||
|
||||
code() {
|
||||
curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
attempts=0
|
||||
until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do
|
||||
attempts=$((attempts + 1))
|
||||
[ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
|
||||
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}"
|
||||
TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
|
||||
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "== baseline"
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== redis stopped"
|
||||
compose stop redis >/dev/null 2>&1
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")"
|
||||
echo "GET /users/me (no token) -> $(code "$API/users/me")"
|
||||
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
|
||||
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
|
||||
compose start redis >/dev/null 2>&1
|
||||
wait_healthy redis
|
||||
echo "-- redis started again (AOF restored)"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== postgres stopped"
|
||||
compose stop postgres >/dev/null 2>&1
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")"
|
||||
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
|
||||
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
|
||||
compose start postgres >/dev/null 2>&1
|
||||
wait_healthy postgres
|
||||
echo "-- postgres started again"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== logout"
|
||||
echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
@@ -3,7 +3,7 @@ set -eu
|
||||
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
|
||||
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
|
||||
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
|
||||
# Allow operator unseal if required in lab7; do not print credentials.
|
||||
# Do not print credentials.
|
||||
# Exit 0 only after the intended action; test waits for API readiness separately.
|
||||
printf '%s
|
||||
' 'TODO: implement restart.sh for your infrastructure' >&2
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Поочерёдно заменяет API с одной дополнительной репликой."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from scripts.deployment import ComposeProject, deployment_lock
|
||||
|
||||
|
||||
def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None:
|
||||
"""Собирает образ, применяет миграции и заменяет реплики."""
|
||||
original = project.replicas()
|
||||
if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original):
|
||||
raise RuntimeError("До обновления нужны минимум две здоровые API-реплики")
|
||||
old_images = {item.image for item in original}
|
||||
if len(old_images) != 1:
|
||||
raise RuntimeError("Сначала восстановите единый образ работающих реплик")
|
||||
old_image = old_images.pop()
|
||||
profile = project.docker(
|
||||
"image", "inspect", old_image,
|
||||
"--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}',
|
||||
)
|
||||
if profile != "file-v1":
|
||||
raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up")
|
||||
count = len(original)
|
||||
print("Сборка нового образа API", flush=True)
|
||||
build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else ()
|
||||
project.run("build", *build_args, "api", timeout=300)
|
||||
tag = project.config["services"]["api"]["image"]
|
||||
new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}")
|
||||
print("Однократное применение совместимых миграций", flush=True)
|
||||
project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate")
|
||||
if old_image == new_image:
|
||||
print("Образ не изменился; замена контейнеров не требуется")
|
||||
return
|
||||
|
||||
try:
|
||||
for old in original:
|
||||
project.run(
|
||||
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
|
||||
"--no-recreate", "--scale", f"api={count + 1}", "api",
|
||||
)
|
||||
current = project.replicas()
|
||||
additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image]
|
||||
if not additions:
|
||||
raise RuntimeError("Новая реплика не создана")
|
||||
project.wait_healthy({item.id for item in additions})
|
||||
for added in additions:
|
||||
project.wait_routed(added.instance_id, base_url, ca_file)
|
||||
survivors = {item.id for item in project.replicas() if item.id != old.id}
|
||||
project.wait_healthy(survivors)
|
||||
project.docker("stop", old.id)
|
||||
project.docker("rm", old.id)
|
||||
print(f"Заменена реплика {old.instance_id}", flush=True)
|
||||
final = project.replicas()
|
||||
if len(final) != count or any(item.image != new_image for item in final):
|
||||
raise RuntimeError("Состав реплик после обновления не соответствует плану")
|
||||
project.wait_healthy({item.id for item in final})
|
||||
except Exception:
|
||||
print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr)
|
||||
project.run(
|
||||
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
|
||||
"--force-recreate", "--scale", f"api={count}", "api",
|
||||
env={"API_IMAGE": old_image},
|
||||
)
|
||||
project.wait_healthy({item.id for item in project.replicas()})
|
||||
raise
|
||||
|
||||
dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines()
|
||||
if old_image in dangling:
|
||||
references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}")
|
||||
if not references:
|
||||
try:
|
||||
project.docker("image", "rm", old_image)
|
||||
except RuntimeError:
|
||||
print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr)
|
||||
print(f"Готово: {count} здоровых реплик нового образа")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Запускает обновление с блокировкой проекта."""
|
||||
parser = argparse.ArgumentParser(description="Rolling update of API replicas")
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--ca-file", default=None)
|
||||
parser.add_argument("--revision", default=None)
|
||||
args = parser.parse_args()
|
||||
project = ComposeProject()
|
||||
with deployment_lock(project):
|
||||
update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,108 +0,0 @@
|
||||
"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from minio.error import S3Error
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.domain import attachments as domain
|
||||
from app.errors.exceptions import unavailable
|
||||
from app.models.attachment import UploadIntent
|
||||
from app.schemas.user import User
|
||||
from app.storage import attachments as repository
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def object_exists(storage, key):
|
||||
exists = True
|
||||
try:
|
||||
await storage.stat_object(key)
|
||||
except S3Error as error:
|
||||
if error.code not in {'NoSuchKey', 'NoSuchObject'}:
|
||||
raise
|
||||
exists = False
|
||||
return exists
|
||||
|
||||
|
||||
async def main():
|
||||
payload = json.load(sys.stdin)
|
||||
settings = Settings()
|
||||
engine = create_db_engine(settings)
|
||||
factory = create_session_factory(engine)
|
||||
storage = create_s3_storage(settings.s3)
|
||||
user = User.model_validate(payload['user'])
|
||||
chat_id = UUID(payload['chat_id'])
|
||||
original = base64.b64decode(payload['image'])
|
||||
keys = []
|
||||
create_record = repository.create_attachment
|
||||
put_file, delete_object = storage.put_file, storage.delete_object
|
||||
|
||||
async def capture_put(key, path, **metadata):
|
||||
keys.append(key)
|
||||
await put_file(key, path, **metadata)
|
||||
|
||||
async def fail_metadata(*args, **kwargs):
|
||||
raise OperationalError('injected metadata failure after real S3 PUT', {}, None)
|
||||
|
||||
async def fail_delete(*args):
|
||||
raise unavailable('injected compensation failure')
|
||||
|
||||
async def chunks():
|
||||
yield original
|
||||
|
||||
async def attempt():
|
||||
raised = False
|
||||
async with factory() as session:
|
||||
try:
|
||||
await domain.upload_attachment(session, storage, user, chat_id,
|
||||
filename='fault.png', content_type='image/png', chunks=chunks())
|
||||
except OperationalError:
|
||||
raised = True
|
||||
assert raised, 'injected database failure was not raised'
|
||||
|
||||
storage.put_file = capture_put
|
||||
repository.create_attachment = fail_metadata
|
||||
try:
|
||||
await attempt()
|
||||
assert not await object_exists(storage, keys[-1]), 'compensation left original'
|
||||
async with factory() as session:
|
||||
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1]))
|
||||
assert intent is None, 'compensated upload left intent'
|
||||
storage.delete_object = fail_delete
|
||||
await attempt()
|
||||
key = keys[-1]
|
||||
assert await object_exists(storage, key), 'fault did not leave real S3 object'
|
||||
storage.delete_object = delete_object
|
||||
async with factory() as session:
|
||||
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert result.deleted == 0, 'fresh object deleted before grace period'
|
||||
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key))
|
||||
assert intent is not None, 'durable intent missing'
|
||||
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
|
||||
await session.commit()
|
||||
dry_run = await domain.reconcile_attachments(session, storage)
|
||||
assert dry_run.candidates == 1 and dry_run.deleted == 0
|
||||
assert await object_exists(storage, key), 'dry-run removed original'
|
||||
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert applied.deleted == 1 and not await object_exists(storage, key)
|
||||
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert repeated.examined == repeated.deleted == 0
|
||||
print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True,
|
||||
'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True}))
|
||||
finally:
|
||||
repository.create_attachment = create_record
|
||||
storage.put_file, storage.delete_object = put_file, delete_object
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
|
||||
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
from threading import Barrier
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
MONITOR = r'''
|
||||
import json, pathlib, sys, threading
|
||||
root = pathlib.Path('/sys/fs/cgroup')
|
||||
def current():
|
||||
return int((root / 'memory.current').read_text())
|
||||
def peak():
|
||||
path = root / 'memory.peak'
|
||||
return int(path.read_text()) if path.exists() else None
|
||||
stop = threading.Event()
|
||||
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
|
||||
initial, before = current(), peak()
|
||||
sampled = initial
|
||||
print('ready', flush=True)
|
||||
while not stop.wait(0.02):
|
||||
sampled = max(sampled, current())
|
||||
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
|
||||
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--rounds', type=int, default=3)
|
||||
parser.add_argument('--output', type=Path)
|
||||
args = arguments('Concurrent attachment uploads and API memory', configure)
|
||||
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=65)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
|
||||
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, format='PNG')
|
||||
data = buffer.getvalue()
|
||||
check(len(data) <= 10485760, 'fixture exceeds upload limit')
|
||||
monitors = []
|
||||
measurements = {}
|
||||
rows = []
|
||||
started = time.monotonic()
|
||||
try:
|
||||
for replica in project.replicas():
|
||||
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
monitors.append((replica.instance_id, process))
|
||||
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
|
||||
check(bool(monitors), 'API not running')
|
||||
for _ in range(args.rounds):
|
||||
barrier = Barrier(args.concurrency)
|
||||
def upload(index):
|
||||
local = Client(args.base_url, args.ca_file, timeout=65)
|
||||
barrier.wait(timeout=15)
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
|
||||
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
except Failure:
|
||||
# При раннем отказе сервер может закрыть ещё отправляемое тело.
|
||||
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
rows.extend(pool.map(upload, range(args.concurrency)))
|
||||
finally:
|
||||
elapsed = time.monotonic() - started
|
||||
for alias, process in monitors:
|
||||
output, _ = process.communicate(input='', timeout=10)
|
||||
check(process.returncode == 0, 'memory monitor failed')
|
||||
measurements[alias] = json.loads(output)
|
||||
succeeded = [row for row in rows if row['status'] == 201]
|
||||
check(bool(succeeded), 'all uploads were rejected')
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
for row in succeeded:
|
||||
check(row['json']['sha256'] == digest, 'upload checksum differs')
|
||||
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
|
||||
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
|
||||
latencies = sorted(row['elapsed'] for row in rows)
|
||||
result = {
|
||||
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
|
||||
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
|
||||
'statuses': dict(Counter(str(row['status']) for row in rows)),
|
||||
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
|
||||
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
|
||||
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
|
||||
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
|
||||
}
|
||||
encoded = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(encoded + '\n')
|
||||
print(encoded)
|
||||
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -11,9 +11,9 @@ validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
|
||||
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
schema = json.loads(path.read_text())
|
||||
|
||||
@@ -132,7 +132,7 @@ async def clean_db(db_engine: AsyncEngine, redis_client: Redis) -> None:
|
||||
|
||||
def build_settings(database_url: str, **overrides: object) -> Settings:
|
||||
values: dict[str, object] = {
|
||||
# Регрессия lab3 проверяет PostgreSQL/Redis; MinIO подменяется отдельно.
|
||||
# Регрессия проверяет PostgreSQL/Redis; MinIO подменяется отдельно.
|
||||
"s3_enabled": False,
|
||||
"app_host": "127.0.0.1",
|
||||
"app_port": 8080,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
||||
"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
|
||||
"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
|
||||
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
Reference in New Issue
Block a user