From 175751760018974fe9963c781607a95ee13cdcb7 Mon Sep 17 00:00:00 2001 From: yasn0clown Date: Wed, 4 Nov 2026 17:43:29 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=B0=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 59 +- .gitattributes | 10 + .github/pull_request_template.md | 28 +- Dockerfile | 38 + Makefile | 103 ++- REPORT.md | 260 +++++- compose.yaml | 396 +++++++++ docs/lab3/REPORT.md | 231 +++++ docs/lab3/evidence/README.md | 25 + docs/lab3/evidence/SUMMARY.txt | 34 + docs/lab3/evidence/check.txt | 7 + docs/lab3/evidence/failover.txt | 66 ++ docs/lab3/evidence/failover_series.json | 294 +++++++ docs/lab3/evidence/failover_series.txt | 17 + docs/lab3/evidence/failover_series_kill.json | 294 +++++++ docs/lab3/evidence/instances.txt | 22 + docs/lab3/evidence/lb_errors.txt | 17 + docs/lab3/evidence/lb_log_sample.txt | 62 ++ docs/lab3/evidence/load.json | 17 + docs/lab3/evidence/load.txt | 28 + docs/lab3/evidence/migrations.txt | 26 + docs/lab3/evidence/persistence.txt | 24 + docs/lab3/evidence/rolling_back.json | 147 ++++ docs/lab3/evidence/rolling_update.json | 147 ++++ docs/lab3/evidence/rolling_update.txt | 29 + docs/lab3/evidence/shutdown_timeline.txt | 17 + docs/lab3/evidence/smoke.txt | 3 + docs/lab3/evidence/stand.txt | 33 + docs/lab3/evidence/student-tests.txt | 134 +++ evidence/README.md | 6 + evidence/SUMMARY.txt | 46 + evidence/check.txt | 7 + evidence/consistency.txt | 75 ++ evidence/failover.txt | 66 ++ evidence/failover_series.json | 294 +++++++ evidence/failover_series.txt | 17 + evidence/failover_series_kill.json | 298 +++++++ evidence/instances.txt | 22 + evidence/lb_log_sample.txt | 62 ++ evidence/load.json | 17 + evidence/load.txt | 29 + evidence/persistence.txt | 48 ++ evidence/reconcile.txt | 19 + evidence/rolling_back.json | 147 ++++ evidence/rolling_update.json | 147 ++++ evidence/rolling_update.txt | 29 + evidence/s3_bootstrap.txt | 225 +++++ evidence/s3_outage.json | 98 +++ evidence/s3_outage.txt | 24 + evidence/s3_private.txt | 67 ++ evidence/sandbox/check.txt | 6 + evidence/sandbox/consistency.txt | 17 + evidence/sandbox/failover.txt | 12 + evidence/sandbox/failover_series.json | 268 ++++++ evidence/sandbox/load.json | 17 + evidence/sandbox/persistence.txt | 7 + evidence/sandbox/reconcile.txt | 17 + evidence/sandbox/s3_bootstrap.txt | 45 + evidence/sandbox/s3_outage.json | 98 +++ evidence/sandbox/s3_outage_hang.json | 98 +++ evidence/sandbox/s3_private.txt | 54 ++ evidence/sandbox/smoke.txt | 3 + evidence/sandbox/student-tests.txt | 120 +++ evidence/sandbox/upload_memory_c10.json | 43 + .../sandbox/upload_memory_c20_limit4.json | 39 + .../sandbox/upload_memory_c20_nolimit.json | 39 + evidence/smoke.txt | 3 + evidence/stand.txt | 42 + evidence/student-tests.txt | 168 ++++ evidence/upload_memory.txt | 108 +++ evidence/upload_memory_c10.json | 51 ++ evidence/upload_memory_c20.json | 52 ++ infra/README.md | 7 + infra/haproxy/haproxy.cfg | 104 +++ migrations/001_init.sql | 66 ++ .../002_session_cache_invalidations.sql | 13 + migrations/003_idempotency_keys.sql | 40 + migrations/004_attachments.sql | 68 ++ requirements.txt | 10 + scripts/collect_evidence.py | 204 +++++ scripts/init_secrets.py | 78 ++ scripts/ops.py | 473 ++++++++++ scripts/restart.cmd | 9 + scripts/restart.sh | 10 + scripts/sql/explain_history.sql | 93 ++ scripts/stop-one.cmd | 9 + scripts/stop-one.sh | 10 + src/__init__.py | 0 src/config.py | 187 ++++ src/db.py | 130 +++ src/images.py | 284 ++++++ src/main.py | 812 ++++++++++++++++++ src/migrate.py | 25 + src/models.py | 104 +++ src/objectstore.py | 226 +++++ src/reconcile.py | 237 +++++ src/runtime.py | 39 + src/s3_bootstrap.py | 226 +++++ src/security.py | 97 +++ src/server.py | 65 ++ src/sessions.py | 240 ++++++ src/storage.py | 565 ++++++++++++ student-tests/common.py | 108 +++ student-tests/dependency_failure.py | 206 +++++ student-tests/failover_series.py | 163 ++++ student-tests/fixtures/sample.jpg | Bin 0 -> 13121 bytes student-tests/fixtures/sample.png | Bin 0 -> 111817 bytes student-tests/fixtures/sample_progressive.jpg | Bin 0 -> 8520 bytes student-tests/imagegen.py | 93 ++ student-tests/loadgen.py | 169 ++++ student-tests/race.py | 202 +++++ student-tests/revocation_partition.py | 37 + student-tests/rolling_update_load.py | 95 ++ student-tests/run_all.py | 49 ++ student-tests/s3_outage.py | 143 +++ student-tests/test_api.py | 138 +++ student-tests/test_attachment_consistency.py | 306 +++++++ student-tests/test_attachments.py | 299 +++++++ student-tests/test_auth.py | 109 +++ student-tests/test_concurrency.py | 80 ++ student-tests/test_expiry.py | 49 ++ student-tests/test_idempotency.py | 172 ++++ student-tests/test_session_cache.py | 149 ++++ student-tests/test_storage_idempotency.py | 144 ++++ student-tests/test_storage_rollback.py | 101 +++ student-tests/upload_memory.py | 232 +++++ student-tests/wait_url.py | 18 + 127 files changed, 13079 insertions(+), 32 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 docs/lab3/REPORT.md create mode 100644 docs/lab3/evidence/README.md create mode 100644 docs/lab3/evidence/SUMMARY.txt create mode 100644 docs/lab3/evidence/check.txt create mode 100644 docs/lab3/evidence/failover.txt create mode 100644 docs/lab3/evidence/failover_series.json create mode 100644 docs/lab3/evidence/failover_series.txt create mode 100644 docs/lab3/evidence/failover_series_kill.json create mode 100644 docs/lab3/evidence/instances.txt create mode 100644 docs/lab3/evidence/lb_errors.txt create mode 100644 docs/lab3/evidence/lb_log_sample.txt create mode 100644 docs/lab3/evidence/load.json create mode 100644 docs/lab3/evidence/load.txt create mode 100644 docs/lab3/evidence/migrations.txt create mode 100644 docs/lab3/evidence/persistence.txt create mode 100644 docs/lab3/evidence/rolling_back.json create mode 100644 docs/lab3/evidence/rolling_update.json create mode 100644 docs/lab3/evidence/rolling_update.txt create mode 100644 docs/lab3/evidence/shutdown_timeline.txt create mode 100644 docs/lab3/evidence/smoke.txt create mode 100644 docs/lab3/evidence/stand.txt create mode 100644 docs/lab3/evidence/student-tests.txt create mode 100644 evidence/SUMMARY.txt create mode 100644 evidence/check.txt create mode 100644 evidence/consistency.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failover_series.json create mode 100644 evidence/failover_series.txt create mode 100644 evidence/failover_series_kill.json create mode 100644 evidence/instances.txt create mode 100644 evidence/lb_log_sample.txt create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/reconcile.txt create mode 100644 evidence/rolling_back.json create mode 100644 evidence/rolling_update.json create mode 100644 evidence/rolling_update.txt create mode 100644 evidence/s3_bootstrap.txt create mode 100644 evidence/s3_outage.json create mode 100644 evidence/s3_outage.txt create mode 100644 evidence/s3_private.txt create mode 100644 evidence/sandbox/check.txt create mode 100644 evidence/sandbox/consistency.txt create mode 100644 evidence/sandbox/failover.txt create mode 100644 evidence/sandbox/failover_series.json create mode 100644 evidence/sandbox/load.json create mode 100644 evidence/sandbox/persistence.txt create mode 100644 evidence/sandbox/reconcile.txt create mode 100644 evidence/sandbox/s3_bootstrap.txt create mode 100644 evidence/sandbox/s3_outage.json create mode 100644 evidence/sandbox/s3_outage_hang.json create mode 100644 evidence/sandbox/s3_private.txt create mode 100644 evidence/sandbox/smoke.txt create mode 100644 evidence/sandbox/student-tests.txt create mode 100644 evidence/sandbox/upload_memory_c10.json create mode 100644 evidence/sandbox/upload_memory_c20_limit4.json create mode 100644 evidence/sandbox/upload_memory_c20_nolimit.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/stand.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/upload_memory.txt create mode 100644 evidence/upload_memory_c10.json create mode 100644 evidence/upload_memory_c20.json create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_session_cache_invalidations.sql create mode 100644 migrations/003_idempotency_keys.sql create mode 100644 migrations/004_attachments.sql create mode 100644 requirements.txt create mode 100644 scripts/collect_evidence.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/ops.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/sql/explain_history.sql create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/images.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/objectstore.py create mode 100644 src/reconcile.py create mode 100644 src/runtime.py create mode 100644 src/s3_bootstrap.py create mode 100644 src/security.py create mode 100644 src/server.py create mode 100644 src/sessions.py create mode 100644 src/storage.py create mode 100644 student-tests/common.py create mode 100644 student-tests/dependency_failure.py create mode 100644 student-tests/failover_series.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/fixtures/sample.png create mode 100644 student-tests/fixtures/sample_progressive.jpg create mode 100644 student-tests/imagegen.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/race.py create mode 100644 student-tests/revocation_partition.py create mode 100644 student-tests/rolling_update_load.py create mode 100644 student-tests/run_all.py create mode 100644 student-tests/s3_outage.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachment_consistency.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_session_cache.py create mode 100644 student-tests/test_storage_idempotency.py create mode 100644 student-tests/test_storage_rollback.py create mode 100644 student-tests/upload_memory.py create mode 100644 student-tests/wait_url.py diff --git a/.env.example b/.env.example index d99d26a2..62aa4b62 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,71 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Секреты сюда НЕ пишутся: их создаёт scripts/init_secrets.py в каталоге secrets/. APP_HOST=0.0.0.0 +# Публичный порт LB на 127.0.0.1 (единственная точка входа API) APP_PORT=8080 +# Страница состояния HAProxy (UP/DOWN реплик) на 127.0.0.1, только чтение +LB_STATS_PORT=8404 APP_ENV=lab +# Тег образа API +APP_VERSION=lab4 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +# TTL экземпляра api-short-ttl (профиль ttl) для автотеста истечения +SHORT_SESSION_TTL_SECONDS=2 +# Файлы с секретами внутри контейнера (Compose secrets) DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# Источник истины для сессий — PostgreSQL; Redis — общий кэш проверок токена SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url +SESSION_CACHE_TTL_SECONDS=60 +# Число реплик API за балансировщиком (HAProxy держит до 6 слотов) API_REPLICAS=2 +# Хранение результата Idempotency-Key, не меньше 24 ч по контракту IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# Пул соединений и таймауты (на реплику) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +CACHE_TIMEOUT_SECONDS=0.25 +CACHE_RETRY_AFTER_SECONDS=5 +# Readiness и остановка (согласованы с HAProxy: inter 1s, fall 2, timeout check 1.5s) +READY_DB_TIMEOUT_SECONDS=1 +DRAIN_SECONDS=3 +GRACEFUL_TIMEOUT_SECONDS=10 +KEEP_ALIVE_SECONDS=30 +# Одновременных хешей Argon2id на реплику (19 MiB каждый, лимит памяти 256M) +PASSWORD_HASH_CONCURRENCY=4 + +# lab4: S3 и вложения +# Закреплённая версия образа chrislusf/seaweedfs (не latest) +SEAWEEDFS_VERSION=4.50 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments +# Ключи — файлы в secrets/ (создаёт scripts/init_secrets.py), внутри контейнера: S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Таймауты клиента S3: недоступность S3 -> быстрый 503 только для файловых операций +S3_CONNECT_TIMEOUT_SECONDS=2 +S3_READ_TIMEOUT_SECONDS=10 +S3_MAX_ATTEMPTS=2 +# Общий бюджет вызова S3 из запроса, включая DNS (скачивание < timeout server HAProxy 10 с) +S3_DOWNLOAD_DEADLINE_SECONDS=5 +S3_UPLOAD_DEADLINE_SECONDS=10 +# После отказа S3 файловые запросы N секунд сразу получают 503 (circuit breaker) +S3_CIRCUIT_OPEN_SECONDS=5 +# Лимиты контракта (можно только уменьшить) MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Одновременных загрузок на реплику (остальные ждут слот, затем 503 + Retry-After) +MAX_CONCURRENT_UPLOADS=4 +UPLOAD_QUEUE_TIMEOUT_SECONDS=5 +# Максимальное время приёма тела загрузки (медленный/зависший клиент -> 400) +UPLOAD_READ_TIMEOUT_SECONDS=60 +# Одновременных разборов изображения на реплику +IMAGE_DECODE_CONCURRENCY=2 +# Сборщик незавершённых загрузок (pending) и объектов-сирот, см. src/reconcile.py +ATTACHMENT_GC_INTERVAL_SECONDS=300 +ATTACHMENT_GC_GRACE_SECONDS=900 +ORPHAN_SWEEP_INTERVAL_SECONDS=3600 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d72808bb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# Скрипты и конфиги, которые выполняются внутри Linux-контейнеров или sh, — всегда LF, +# даже при checkout на Windows (иначе HAProxy/sh спотыкаются о CRLF). +*.sh text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.py text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf +# Пакетные файлы Windows — CRLF. +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..42a08d1e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Попов Константин Дмитриевич, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: — -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесено всё API и инфраструктура lab3 (HAProxy, 2+ реплики, идемпотентность, drain, rolling update). Добавлены приватное S3-хранилище SeaweedFS 4.50 с идемпотентным bootstrap бакета и отдельными ограниченными ключами API; вложения (миграция 004, составные FK «файл и сообщение из одного чата»); синхронная загрузка с потоковым лимитом 10 MiB, проверкой PNG/JPEG по содержимому без полной распаковки, лимитом 40 Мп и безопасными именами; авторизованное потоковое скачивание; компенсация и сборщик для согласованности БД/S3 (dry-run/apply); лимит одновременных загрузок; тесты уровня 4 и эксперимент памяти. +Команды запуска из чистого clone: `python scripts/init_secrets.py`, `cp .env.example .env`, `docker compose --env-file .env up -d --build`, `python scripts/ops.py wait`, `python scripts/ops.py wait-s3`; проверки — `python scripts/collect_evidence.py` (см. REPORT.md). +Результаты проверок и ссылка на REPORT.md: REPORT.md, разделы «Доказательства критериев» и «Эксперименты»; evidence/SUMMARY.txt, evidence/. +Ограничения / известные ошибки: один host/БД/S3 без репликации; тело загрузки до 10 MiB × 4 слота в памяти (измерено); progressive JPEG проверяется структурно; прогрев SeaweedFS 15–30 с после рестарта (hooks его ждут). Подробно — REPORT.md. +Источники и использование ИИ: REPORT.md. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e6a95f80 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# Lab4: API мессенджера (FastAPI + PostgreSQL + Redis + S3/SeaweedFS), реплики за HAProxy. +FROM python:3.12.7-slim-bookworm + +# lab4: настройки glibc malloc — меньше арен на потоки и возврат больших буферов +# ОС сразу после загрузки (замер: базовый RSS 124 -> 88 MiB, см. REPORT.md). +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + MALLOC_ARENA_MAX=2 \ + MALLOC_MMAP_THRESHOLD_=1048576 \ + MALLOC_TRIM_THRESHOLD_=1048576 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --disable-pip-version-check -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +# Готовность (а не только жизнь процесса): БД доступна, схема мигрирована, +# реплика не в фазе осушения. Интервалы в compose.yaml переопределяют эти. +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" || exit 1 + +# exec-форма: Python — PID 1 и получает SIGTERM напрямую. +# src/server.py: SIGTERM -> осушение DRAIN_SECONDS (readiness 503) -> graceful stop. +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src.server"] diff --git a/Makefile b/Makefile index 5d86d171..71df80b3 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,16 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover \ + secrets up wait ps down migrate restore rolling-update logs-lb \ + test-student test-idempotency test-failover-series test-failover-series-kill \ + test-rolling test-storage test-expiry test-failures test-revocation explain evidence \ + test-attachments test-consistency test-s3-outage test-upload-memory s3-check wait-s3 reconcile reconcile-apply help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up/wait/ps/down/restore/rolling-update: stand; test-*: own checks (see REPORT.md)" + @echo "lab4: test-attachments test-consistency test-s3-outage test-upload-memory s3-check reconcile" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +34,98 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# ---------------------------------------------------------------- Решение +COMPOSE ?= docker compose --env-file .env + +secrets: + $(PYTHON) scripts/init_secrets.py + @test -f .env || cp .env.example .env + +up: secrets + $(COMPOSE) config --quiet + $(COMPOSE) up -d --build + $(PYTHON) scripts/ops.py wait + $(PYTHON) scripts/ops.py wait-s3 + +wait: + $(PYTHON) scripts/ops.py wait + +ps: + $(PYTHON) scripts/ops.py ps + +# Безопасная остановка: volumes (PostgreSQL pgdata и S3 s3data) сохраняются. +down: + $(COMPOSE) --profile ttl --profile faults down + +migrate: + $(COMPOSE) run --rm -T migrate + +restore: + $(PYTHON) scripts/ops.py restore + +rolling-update: + $(PYTHON) scripts/ops.py rolling-update + +logs-lb: + $(COMPOSE) logs --tail 50 lb + +test-student: + $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" + +test-idempotency: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_idempotency.py -v + +test-failover-series: + $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover_series.json + +test-failover-series-kill: + $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --mode kill --output evidence/failover_series_kill.json + +test-rolling: + $(PYTHON) student-tests/rolling_update_load.py --base-url "$(BASE_URL)" --output evidence/rolling_update.json + +test-storage: + $(PYTHON) scripts/ops.py in-network storage + +test-expiry: + $(PYTHON) scripts/ops.py in-network expiry + +test-failures: + $(PYTHON) student-tests/dependency_failure.py --base-url "$(BASE_URL)" --output evidence/dependency_failure.json + +test-revocation: + $(PYTHON) student-tests/revocation_partition.py --base-url "$(BASE_URL)" | tee evidence/revocation_partition.txt + +explain: + $(COMPOSE) exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 < scripts/sql/explain_history.sql > evidence/explain_history.txt + @echo "saved evidence/explain_history.txt" + +# Все проверки на compose-стенде с записью результатов в evidence/ (~6-8 минут). +evidence: + $(PYTHON) scripts/collect_evidence.py + +# ---------------------------------------------------------------- lab4: вложения и S3 +test-attachments: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_attachments.py -v + +test-consistency: + $(PYTHON) scripts/ops.py in-network consistency + +test-s3-outage: + $(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" --output evidence/s3_outage.json + +test-upload-memory: + $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" --concurrency 10 --rounds 3 --docker --output evidence/upload_memory_c10.json + +s3-check: + $(PYTHON) scripts/ops.py s3-check + +wait-s3: + $(PYTHON) scripts/ops.py wait-s3 + +reconcile: + $(PYTHON) scripts/ops.py reconcile + +reconcile-apply: + $(PYTHON) scripts/ops.py reconcile --apply diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..91b8569e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,271 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0, **boto3 1.35.99**, **Pillow 11.1.0**; PostgreSQL 16.4, Redis 7.4.1, HAProxy 2.8.30, **SeaweedFS 4.50**. Документация: Amazon S3 (PutObject, Content-MD5, ListObjectsV2), SeaweedFS wiki (Amazon S3 API, S3 credentials/identities, `weed server`), спецификация PNG (ISO/IEC 15948: чанки, CRC, Adam7), JPEG (ITU T.81: маркеры SOF/SOS/EOI), Pillow (`Image.open`, `draft`), glibc `mallopt` (MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_). Помощь ИИ (Claude) — при проектировании схемы согласованности БД/S3, проверки изображений, тестов, экспериментов и при написании отчёта. + +Решения lab3 (балансировщик, реплики, идемпотентность, drain, rolling update) перенесены без изменений семантики; их отчёт и доказательства сохранены в [docs/lab3/REPORT.md](docs/lab3/REPORT.md). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`; **`chrislusf/seaweedfs:4.50`** (тег задаётся `SEAWEEDFS_VERSION` в `.env`, не `latest`). Нужны Docker Engine / Docker Desktop с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Оркестрация — **Compose + HAProxy** (Swarm не используется). + +```sh +# 1. Секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py # + 4 файла S3-ключей и secrets/s3_config.json +cp .env.example .env # Windows: copy .env.example .env + +# 2. Сборка и запуск: postgres+redis -> migrate; s3 -> s3-init (bucket) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Готовность: 2 реплики healthy и UP в HAProxy; S3 реально пишет/читает +python scripts/ops.py wait +python scripts/ops.py wait-s3 +python scripts/ops.py ps +``` + +Прогрев: первая сборка 1–3 мин, далее стенд готов за ~30–40 с (healthcheck PostgreSQL и SeaweedFS, миграции, bootstrap бакета, `start_period` API, `rise 2` в HAProxy). + +```sh +# Публичные проверки +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh # Windows: scripts\restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh # Windows: scripts\stop-one.cmd +python scripts/ops.py restore # вернуть реплику, затем failover повторить для второй + +# Собственные проверки уровня 4 (lab4) +python student-tests/run_all.py --base-url http://localhost:8080 # lab2-3 + test_attachments.py (17 тестов) +python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы +python scripts/ops.py s3-check # приватный бакет вне API, права ключей, закрытые админ-порты +python scripts/ops.py reconcile # сверка БД<->S3, dry-run (--apply — исправить) +python student-tests/s3_outage.py --output evidence/s3_outage.json +python student-tests/upload_memory.py --concurrency 10 --docker --output evidence/upload_memory_c10.json + +# Всё перечисленное (и проверки lab3) одной командой с записью в evidence/: +python scripts/collect_evidence.py # ~12–15 мин; --quick ~7–8 мин +``` + +**Остановка.** Безопасная (данные PostgreSQL и S3 сохраняются): `docker compose --env-file .env --profile ttl --profile faults down`. Намеренное удаление всех учебных данных (volumes `pgdata`, `s3data`) — только вручную: `docker compose --env-file .env --profile ttl --profile faults down -v`. Hooks и тесты volumes не удаляют. + +**Действия оператора.** Создать секреты до первого запуска (пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` существующие файлы не перезаписывает; S3-ключи не связаны с данными и могут быть заменены — см. docstring скрипта). После `stop-one` вернуть реплику `python scripts/ops.py restore`. Если тег `chrislusf/seaweedfs:4.50` недоступен в вашем registry-зеркале, укажите соседний проверенный тег в `SEAWEEDFS_VERSION`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` + 127.0.0.1:8080 (единственная публичная точка входа) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки /health/ready + │ POST .../attachments: timeout server 90 s, остальное 10 s + ▼ сеть backend + ┌──────── api x2 (read-only rootfs, tmpfs /tmp 16m, 0.5 CPU / 320M) ────────┐ + │ загрузка: поток -> лимит 10 MiB -> проверка PNG/JPEG -> [1] pending в БД │ + │ -> [2] S3 PUT (Content-MD5) -> [3] ready в БД -> 201 │ + │ скачивание: проверка членства -> поток из S3 (64 KiB) -> клиент │ + │ фон: сборщик pending/сирот (advisory lock, одна реплика за раз) │ + └──────┬───────────────────────┬────────────────────────────┬───────────────┘ + ▼ backend ▼ backend ▼ storage (internal: true) + postgres 16 (pgdata) redis 7 (кэш сессий) s3: SeaweedFS 4.50 (s3data) + users, chats, messages, только S3 API :8333 в сети storage; + sessions, idempotency_keys, master/filer/volume UI — 127.0.0.1 + attachments, message_attachments внутри контейнера; без анонимов + ▲ ▲ + migrate (one-shot) s3-init (one-shot, admin-ключи): + бакет + проверки прав и приватности +``` + +Порядок старта: `postgres`, `redis` (healthy) → `migrate`; `s3` (healthy) → `s3-init` (оба `service_completed_successfully`) → `api` (healthy) → `lb`. + +**Источник истины.** PostgreSQL — для всех метаданных, включая вложения (владелец, чат, имя, тип, размер, SHA-256, размеры в пикселях, ключ объекта, состояние). Байты оригинала — только в приватном бакете `messenger-attachments`. На API нет ни volume, ни каталога для файлов: rootfs read-only, `/tmp` — tmpfs 16 MiB (тело загрузки держится в памяти процесса до записи в S3, не на диске; `python scripts/ops.py api-fs`). + +### Выбор S3-сервера: SeaweedFS + +MinIO с осени 2025 г. не публикует community-образы и бинарники для свежих релизов, а консоль администратора у него встроена в тот же процесс. SeaweedFS (Apache-2.0): версия закрепляется тегом, S3-identities с правами на конкретный бакет задаются статическим JSON-конфигом (`-s3.config`), а все административные HTTP-интерфейсы (master 9333, filer 8888, volume 8080) привязываются к 127.0.0.1 внутри контейнера (`-ip.bind=127.0.0.1`), наружу в сеть `storage` выставлен только S3 (`-s3.ip.bind=0.0.0.0`). Встроенный IAM API отключён (`-s3.iam=false`), автосоздание бакетов — тоже (`-s3.autoCreateBucket=false`). + +| Роль | Сервис | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | haproxy:2.8.30-alpine | `/lb-health`; бэкенды по `/health/ready` | — | backend; 127.0.0.1:8080, :8404 | — | +| API ×2 | `api` | сборка `Dockerfile` | `/health/ready` (PostgreSQL + схема) | нет (read-only) | backend + storage; без ports | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key | +| БД | `postgres` | postgres:16.4-alpine | `pg_isready` | `pgdata` | backend | postgres_password | +| Кэш сессий | `redis` | redis:7.4.1-alpine | `redis-cli ping` | нет (только кэш) | backend | redis_password | +| S3 | `s3` | chrislusf/seaweedfs:4.50 | `wget /healthz` (+ `ops.py wait-s3`) | `s3data` | storage (internal); только :8333 | secret s3_config (identities) | +| Миграции | `migrate` | сборка | one-shot | — | backend | database_url | +| Bucket bootstrap | `s3-init` | сборка | one-shot | — | storage | admin + api S3-ключи | +| Тестовые отказы | `api-fault-*` (профиль `faults`) | образ API | `/health/ready` | — | backend + storage | как у api | + +### Модель данных (`migrations/004_attachments.sql`) + +- `attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key UNIQUE, state pending|ready, created_at, updated_at)`. CHECK-ограничения повторяют контракт: имя 1–255, тип png/jpeg, размер 1..10 485 760, `width*height ≤ 40 000 000`, sha256 в нижнем регистре. +- `object_key = "originals/" + случайный UUID`. Он **не** совпадает с `id` вложения, не содержит `filename` и не раскрывается в API. +- `message_attachments(message_id, attachment_id, chat_id, position)`: порядок `attachment_ids` сохраняется. **Составные внешние ключи** `(message_id, chat_id) → messages(id, chat_id)` и `(attachment_id, chat_id) → attachments(id, chat_id)` на уровне БД запрещают привязать файл чужого чата. `ON DELETE RESTRICT` не даёт удалить вложение, на которое ссылается сообщение. +- `messages_text_len` расширен до 0..4000: `text=""` допустим только вместе с вложениями (проверяет API). Миграция только расширяющая — реплики lab3 работают на новой схеме (rolling update). + +### Загрузка, `201` и границы транзакций + +`POST /api/v1/chats/{id}/attachments?filename=…` (сырые bytes, `src/main.py`): + +1. Дешёвые проверки до чтения тела: аутентификация → `filename` (400) → `Content-Type` только `image/png|image/jpeg` (415) → `Content-Length > 10 MiB` → 413 без чтения → членство в чате (404) → слот загрузки (`MAX_CONCURRENT_UPLOADS=4` на реплику; ожидание ≤ 5 с, иначе 503 + `Retry-After`). +2. Тело читается потоком; как только счётчик превысил 10 MiB — 413 (и для chunked без `Content-Length`). Остаток тела (до 10 MiB) дочитывается в никуда, чтобы клиент получил 413, а не обрыв. Общий дедлайн приёма — `UPLOAD_READ_TIMEOUT_SECONDS=60` (медленный клиент → 400). Обрыв клиентом → ничего не записано. +3. Проверка содержимого (`src/images.py`, ≤ 2 одновременных на реплику): + - сигнатура PNG/JPEG; **сигнатура должна совпасть с заявленным Content-Type** (подмена MIME → 415); + - размеры из IHDR / SOFn **до** распаковки: > 40 Мп → 413 (pixel bomb 8000×6000 размером ~50 КБ отклоняется за миллисекунды); + - PNG: все CRC, порядок IHDR…IDAT…IEND, допустимые bit depth/color type, **потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»**: объём должен в точности совпасть с ожидаемым (с учётом Adam7). Так обнаруживаются обрезанные и испорченные данные, а zip-бомба обрывается, как только вывод превысил ожидание; память O(64 KiB) при любом размере картинки; + - JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI → обрезанный файл = 415). Baseline дополнительно декодируется Pillow в режиме `draft` (1/8 масштаба). Progressive Pillow не декодирует: libjpeg держит коэффициенты всего кадра, замер — **119 MiB** на 39.7 Мп против **4.9 MiB** у baseline; + - независимый парсер Pillow должен увидеть тот же формат и размеры. +4. `[1]` `INSERT attachments … state='pending'` (с проверкой членства в том же операторе) — **COMMIT до S3**. +5. `[2]` S3 `PutObject` с `Content-MD5` (S3 сам отклоняет повреждённую передачу — BadDigest; проверено) и `x-amz-meta-sha256`. +6. `[3]` `UPDATE … SET state='ready' WHERE id=… AND state='pending'` — **COMMIT** → `201 Attachment {status: ready, variants: [], error_code: null}`. + +**Когда честно вернуть 201:** только после шага `[3]`. Байты к этому моменту записаны в S3 (PUT подтверждён с проверкой MD5), а метаданные зафиксированы в PostgreSQL. Ответ 201 до любого из этих событий означал бы, что «готовое» вложение может не скачаться. + +**S3 PUT прошёл, а COMMIT `[3]` — нет.** Остаются строка `pending` (не видна в API, прикрепить нельзя) и объект. При ошибке БД API сразу компенсирует: удаляет объект и пытается удалить строку, клиент получает 503. Если процесс умер (компенсации нет), строку и объект уберёт сборщик после grace period. Обе ветки проверены внедрением отказа (`api-fault-db`, `api-fault-crash`). + +### Согласованность БД и S3: компенсация + сборщик (`src/reconcile.py`) + +Главный инвариант: **ключ объекта записан в БД раньше, чем объект появится в бакете.** Поэтому объект без строки может появиться только после того, как строку удалили. Ключ одноразовый, удалённая строка не возрождается. + +| Ситуация | Кто исправляет | Как | +| --- | --- | --- | +| Ошибка S3 в `[2]` | запрос | DELETE своей pending-строки, 503 | +| Ошибка БД в `[3]` | запрос | DELETE объекта (+ попытка DELETE строки), 503 | +| Процесс умер между `[1]`/`[2]`/`[3]`, обрыв, сеть | сборщик, проход A | pending старше grace: **сначала** `DELETE … WHERE state='pending' RETURNING object_key` (атомарный захват; если запрос успел `[3]`, строка уже ready и не трогается), **потом** DELETE объекта | +| Объект без строки (сбой удаления в A, ручная запись) | сборщик, проход B | листинг `originals/` страницами; нет строки **и** `LastModified` старше grace → повторная проверка → DELETE | +| ready-строка без объекта | только отчёт (C) | байты не восстановить; авто-удаление метаданных опасно — `missing_objects` в отчёте | + +- **Grace period** `ATTACHMENT_GC_GRACE_SECONDS=900` (минимум 300, конфиг меньше не примет). Это намного больше самой долгой загрузки: чтение ≤ 60 с + проверка + S3 PUT (connect 2 с + read 10 с) × 2 попытки ≈ 90 с. Если загрузка всё же переживёт grace, `[3]` вернёт 0 строк → API удалит свой объект и ответит 503. +- **Свежие и используемые вложения не удаляются:** сборщик трогает только `pending` старше grace. ready-строки не удаляет никогда, а на используемые ещё и стоит FK RESTRICT. Тесты: `test_fresh_pending_is_kept`, `test_ready_and_referenced_are_never_collected` (прямой `DELETE` получает ForeignKeyViolation), `test_pending_cannot_be_attached`. +- Фоновый проход A — каждые 300 с, полный обход бакета B — раз в час. Выполняет одна реплика за раз (`pg_try_advisory_lock`). Ручной запуск: `python scripts/ops.py reconcile` (**dry-run** по умолчанию, ничего не меняет) и `… reconcile --apply`. Повторный `--apply` — no-op (проверено). Стоимость обхода на стенде: 238 объектов — 0.036 с (листинг + один `SELECT … = ANY` на страницу из 1000 ключей). + +### Ограничение памяти и потоковая обработка + +- Лимит байт применяется **во время** чтения (на 10 MiB + 1 байт). В памяти одной загрузки — тело (≤ 10 MiB) + кратковременно куски до склейки. Проверка PNG — O(64 KiB), JPEG baseline — ≤ ~5 MiB. Тело уходит в S3 как `BytesIO` поверх тех же bytes, без копии. +- Одновременно на реплику ≤ 4 загрузки (`MAX_CONCURRENT_UPLOADS`). Остальные ждут слот ≤ 5 с **до чтения тела**, поэтому их байты не накапливаются в памяти. Скачивание идёт потоком из S3 кусками по 64 KiB (`StreamingResponse`), без буферизации файла. +- Найдено и исправлено замерами (см. «Эксперименты»): `zlib.unconsumed_tail` копировал остаток IDAT на каждом шаге (пик 2× файла и квадратичное копирование), а botocore склеивал заголовки с телом `bytes` в один буфер. В образе также настроен glibc malloc (`MALLOC_ARENA_MAX=2`, фиксированный порог mmap 1 MiB), чтобы большие буферы возвращались ОС сразу. + +### Безопасность бакета и ключей + +- **Bootstrap идемпотентен** (`src/s3_bootstrap.py`, сервис `s3-init`): `CreateBucket` → `BucketAlreadyOwnedByYou` считается успехом (`evidence/sandbox/s3_bootstrap.txt`: повторный запуск — `already exists (idempotent no-op)`). После создания бакета s3-init проверяет права ключей и приватность; при нарушении он завершается с ошибкой, и API не стартует. +- **Отдельные ключи API** (`messenger_api`) имеют только `Read/Write/List` на `messenger-attachments`. `CreateBucket`/`DeleteBucket` → AccessDenied. Admin-ключи (`bootstrap_admin`) смонтированы только в `s3-init`. Identity `anonymous` не объявлена. +- **Проверка приватности вне API** (`python scripts/ops.py s3-check`): из сети `storage`, без подписи, к S3 напрямую. Анонимный GET реального объекта последнего вложения → 403, анонимный листинг бакета и списка бакетов → 403. Тот же объект с ключами API → 200 и совпадающий SHA-256. +- **Админ-консоль не открыта в общей сети:** порты 9333/8888/8080/19333/18888 с адреса `s3` закрыты (проверка в том же скрипте). Сеть `storage` — `internal: true`. LB в неё не входит, порты S3 на хост не публикуются. +- **Случайный ключ и приватный бакет не заменяют ACL приложения:** доступ к `GET /attachments/{id}` и `/content` проверяется по членству в чате на каждый запрос (посторонний → 404 без раскрытия существования). Download идёт через API без redirect и без подписанных URL. Ответ содержит `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `Content-Disposition` с безопасно закодированным именем. + +### Недоступность S3 и readiness + +S3 **не входит** в readiness: если бы реплики при падении S3 становились not-ready, LB вывел бы обе, и вместе с файлами пропали бы чат и вход. Вместо этого файловые операции быстро отвечают `503 unavailable` + `Retry-After` (таймауты клиента S3: connect 2 с, read 10 с, ≤ 2 попытки), а всё остальное работает. Критичные зависимости readiness — по-прежнему PostgreSQL и мигрированная схема. + +Таймауты boto3 не покрывают DNS: в Docker остановленный контейнер `s3` пропадает из DNS, и разрешение имени висит секундами. В первом прогоне на Docker Desktop загрузка ждала 12–13 с, а скачивание превысило `timeout server` HAProxy (10 с) и получило **504 от LB** вместо 503. Исправление (`src/main.py`, `s3_call`; `src/objectstore.py`, `CircuitBreaker`): +- у каждого вызова S3 из запроса есть общий дедлайн: скачивание 5 с, загрузка 10 с; +- после отказа circuit breaker 5 с сразу отвечает 503, затем делает одну пробную попытку; +- у API в compose `dns_opt: timeout:1, attempts:1`. + +Проверено на худшем случае — S3, который принимает соединение и молчит: upload 503 за 10.0 с, download 503 за 5.0 с (`evidence/sandbox/s3_outage_hang.json`). При S3, который сразу отказывает: 1.8–2.3 с и 0.01 с (breaker открыт). + +**Особенность SeaweedFS после рестарта** (замер): `/healthz` отвечает сразу, но первые операции блокируются. Первое чтение после рестарта шло ~15 с, первая запись ~29 с: выборы Raft-лидера (`electionTimeout` 10 с), затем heartbeat volume-сервера. С `-master.electionTimeout=2s` первая запись — ~19 с. Поэтому restart-hook и `ops.py wait-s3` ждут **реальной** готовности: проба put/get/delete и чтение существующего объекта, каждая операция < 2 с (`s3_bootstrap.py --wait-ready`); на стенде готовность наступала через 14.5–17.1 с. Штатная остановка SeaweedFS длится ~22 с, поэтому `stop_grace_period: 10s` (как у Docker по умолчанию) и затем SIGKILL. Подтверждённые записи к этому моменту уже в page cache ОС; тест сохранности после такого рестарта проходит. + +### Что перенесено из lab3 и что изменено + +Перенесены без изменения семантики: всё API lab2–3, HAProxy-конфигурация (обнаружение реплик через DNS, drain, `retries`/`redispatch`), идемпотентность, миграции one-shot, rolling update, hooks и собственные тесты lab2–3 (47 HTTP-тестов + 12 тестов хранилища проходят на lab4). + +Изменено в lab4: + +1. Переход контракта 3 → 4: поля `attachment_ids` в запросе и ответе сообщения (отсутствие = `[]` и для отпечатка идемпотентности; порядок значим), три новых endpoint. +2. Сервисы `s3`, `s3-init`, volume `s3data`, сеть `storage`; секреты S3; профиль `faults`. +3. HAProxy: для `POST …/attachments` — `timeout server 90s` (`http-request set-timeout`), остальным запросам — прежние 10 с. +4. Лимит памяти API 256M → 320M (4 загрузки по 10 MiB, замер ниже); glibc malloc tuning в `Dockerfile`. +5. `scripts/ops.py`: restart включает S3 и ждёт его прогрева; новые `wait-s3`, `s3-check`, `reconcile`, `api-fs`, `in-network consistency`. `collect_evidence.py` дополнен проверками lab4. + +### Соответствие `.env.example` и реальной конфигурации + +Переменные lab3 — без изменений (см. [docs/lab3/REPORT.md](docs/lab3/REPORT.md)). Новые: + +| Переменная | Значение | Где используется | +| --- | --- | --- | +| `SEAWEEDFS_VERSION` | 4.50 | тег образа `s3` | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | http://s3:8333, us-east-1, messenger-attachments | api, s3-init, `src/config.py` | +| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | /run/secrets/… | ключи API (compose secrets) | +| `S3_CONNECT_TIMEOUT_SECONDS`, `S3_READ_TIMEOUT_SECONDS`, `S3_MAX_ATTEMPTS` | 2, 10, 2 | `src/objectstore.py` | +| `S3_DOWNLOAD_DEADLINE_SECONDS`, `S3_UPLOAD_DEADLINE_SECONDS`, `S3_CIRCUIT_OPEN_SECONDS` | 5, 10, 5 | общий бюджет вызова S3 (вкл. DNS), circuit breaker | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | 10485760, 40000000 | не выше контракта (иначе API не стартует) | +| `MAX_CONCURRENT_UPLOADS`, `UPLOAD_QUEUE_TIMEOUT_SECONDS` | 4, 5 | слоты загрузки на реплику | +| `UPLOAD_READ_TIMEOUT_SECONDS` | 60 | дедлайн приёма тела | +| `IMAGE_DECODE_CONCURRENCY` | 2 | разборов изображения на реплику | +| `ATTACHMENT_GC_INTERVAL_SECONDS`, `ATTACHMENT_GC_GRACE_SECONDS`, `ORPHAN_SWEEP_INTERVAL_SECONDS` | 300, 900, 3600 | `src/reconcile.py` | +| `FAULT_INJECTION` | пусто | только сервисы `api-fault-*` | ## Доказательства критериев +**Где получены результаты.** Код проверен на стенде из тех же компонентов, запущенных нативными процессами: PostgreSQL 16, Redis 7, SeaweedFS 4.50 (тот же бинарник, те же флаги), HAProxy 2.8.30 с этим `haproxy.cfg` (статические серверы вместо DNS-шаблона) и 2 реплики API. Среда: Windows 10, Ryzen 5 7500f, 32 GB, Python 3.12, без cgroup-лимитов. Результаты — в `evidence/`. + +### Уровень 3 + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make test` | PASS: файл → сообщение с attachment_ids → download Бобом, SHA-256 совпал, Ева 404 | evidence/sandbox/smoke.txt; evidence/smoke.txt | +| Возможности lab3 сохранены | `run_all.py`, `in-network storage`, failover, persistence | 47 HTTP + 12 DB-тестов lab2–3 OK; failover/persistence PASS | evidence/sandbox/student-tests.txt | +| S3 в общем стенде; метаданные в PostgreSQL, байты в приватном бакете | `compose.yaml`, `ops.py api-fs`, `s3-check` | API без volumes, read-only rootfs; объекты только в `s3data` | evidence/s3_private.txt | +| 201 ready после сохранения; оригинал побайтно | `test_png_roundtrip…`, `test_exactly_max_size_is_accepted` | 201, `status=ready`, `variants=[]`; SHA-256 и Content-Length совпадают (вкл. ровно 10 485 760 байт) | evidence/sandbox/student-tests.txt | +| Message с attachment_ids, посторонний без доступа | smoke, `test_message_with_attachments…` | порядок id сохранён; Ева: 404 на метаданные и bytes | там же | +| PNG/JPEG ≤10 MiB, ≤40 Мп; MIME по содержимому; опасные имена; лимиты до распаковки | `test_too_large_*`, `test_too_many_pixels`, `test_mime_spoofing_is_415`, `test_dangerous_filenames_are_400` | 413/413/415/400; 48 Мп файл 50 КБ → 413 без распаковки | там же | +| Upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2 | PASS, 138–139 записей, 0 ошибок; файл скачан после остановки | evidence/sandbox/failover.txt | +| Restart стенда сохраняет метаданные и оригинал | `make test-persistence` | PASS; сменились PID PostgreSQL, SeaweedFS и обеих реплик; файл скачан со старым токеном | evidence/sandbox/persistence.txt | +| Недоступность S3, чужой чат, restart | `s3_outage.py`; `test_attachment_from_other_chat…`; persistence | 503 за 1.0–1.7 с, чат работает; 404; PASS | evidence/sandbox/s3_outage.json | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Тесты: большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `student-tests/test_attachments.py` (17 тестов) | 413 (Content-Length и chunked), 415 (обрезанный PNG/JPEG, неверный CRC, неверный объём IDAT, мусор, пустое тело, GIF), 415 (6 подмен MIME), 404 (чужое/неизвестное id, файл другого чата, Ева в своём чате), 400 (12 опасных имён); каждый негативный ответ < 500 в общем формате | evidence/sandbox/student-tests.txt | +| Обрыв загрузки и отказ между S3/БД не оставляют вечных объектов; компенсация/сверка с grace | `test_attachment_consistency.py` (10 тестов) в сети стенда | обрыв → 0 строк, 0 объектов; ошибка S3 → 503 + нет строки; ошибка БД после PUT → 503 + объект удалён; **падение процесса после PUT** → pending + объект → сборщик удаляет оба; dry-run ничего не меняет; повтор no-op; свежие/ready/используемые не удаляются | evidence/sandbox/consistency.txt; evidence/consistency.txt | +| Лимиты/таймауты/потоковая обработка; ≥10 одновременных загрузок с замером пика | `upload_memory.py --concurrency 10/20` | 10×9.5 MiB ×3 волны: 30/30 201, пик RSS 156–166 MiB на реплику (база 88–95); при 20 одновременных пик тот же (159–162); без лимитера 262–291 MiB | evidence/sandbox/upload_memory_*.json | +| Bootstrap идемпотентен | повторный `s3-init` | `already exists (idempotent no-op)`, все проверки ok | evidence/sandbox/s3_bootstrap.txt | +| Отдельные S3 credentials с ограниченными правами | `s3_bootstrap` проверки | API-ключи: put/get своего бакета — ok; CreateBucket/DeleteBucket — AccessDenied | evidence/sandbox/s3_private.txt | +| Админ-консоль не в общей сети | `ops.py s3-check` | 9333/8888/8080/19333/18888 закрыты, S3 :8333 — единственный | там же | +| Private bucket вне API | `ops.py s3-check` | анонимный GET реального объекта → 403, листинг → 403; с ключами API тот же объект → 200, SHA-256 совпал | там же | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Стенд замеров:** Windows 10, Ryzen 5 7500f, 32 GB; все процессы (2 реплики API, HAProxy, PostgreSQL, Redis, SeaweedFS, генератор нагрузки) на одном хосте. Реплика: 1 процесс Uvicorn, threadpool по умолчанию. Файлы — синтетический шум, PNG без сжатия, 9.5 MiB (несжимаемые, проверка проходит по полной программе). Память — VmRSS процесса реплики (опрос каждые 20 мс) и пик RSS, который считает ядро (VmHWM, обнуляется перед прогоном). Загрузки через LB (roundrobin по двум репликам), повторы — волнами. + +**Гипотеза:** пик памяти реплики ≈ база + `MAX_CONCURRENT_UPLOADS` × (~2 × размер файла) и не зависит от числа одновременных клиентов; без лимитера память растёт линейно с конкуренцией. + +| Прогон | Одновременно | Загрузок | 201 / SHA ok | p50 / p95, с | База → пик RSS, MiB (реплика 1 / 2) | +| --- | --- | --- | --- | --- | --- | +| до исправлений (zlib tail, bytes в botocore) | 10 | 30 | 30 / 30 | 2.01 / 2.29 | 124 → 237 / 124 → 244 | +| исправления кода, malloc по умолчанию | 10 | 30 | 30 / 30 | 1.20 / 1.34 | 124 → 187 / 124 → 187 | +| + MALLOC_ARENA_MAX=2, mmap ≥ 1 MiB (**итог**) | 10 | 30 | 30 / 30 | 1.32 / 2.07 | 95 → 163 / 95 → 156 (пик ядра VmHWM 166 / 165) | +| итог, вдвое больше клиентов | 20 | 40 | 40 / 40 | 2.16 / 3.08 | 115 → 162 / 97 → 159 | +| лимитер выключен (`MAX_CONCURRENT_UPLOADS=64`) | 20 | 40 | 40 / 40 | 2.90 / 3.03 | 88 → 262 / 89 → 291 | + +Вывод: гипотеза подтверждена. Прирост ~61–72 MiB ≈ 4 слота × 2 × 9.5 MiB. При удвоении клиентов пик не растёт (лишние ждут слот, их тело ещё не прочитано), растёт только задержка. Без лимитера пик +170–200 MiB при той же пропускной способности: при лимите 256 MiB контейнер был бы убит OOM. Поэтому лимит API — 320M, запас ~155 MiB. Микрозамеры (tracemalloc, 9.5 MiB PNG): проверка 19 MiB → 1.5 MiB и 0.17 → 0.06 с; `put_object(bytes)` 25 MiB → `BytesIO` 0.1 MiB. Ограничения: один хост и общий CPU для клиента и сервера; RSS не включает page cache; в Docker тот же скрипт (`--docker`) читает VmHWM процесса API внутри контейнера, `docker stats` и cgroup `memory.peak`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| SIGTERM обслужившей реплики (public failover ×2) | записи идут через другую, файл скачивается | 138/139 записей, 0 ошибок, файл совпал | первая запись после hook ≤ 4.5 с (hook ждёт drain 3 с); потерь нет | +| Каждая реплика по очереди под нагрузкой (`failover_series.py`) | 0 ошибок, LB DOWN < 2 с | DOWN 1.30 / 1.53 с, 0 ошибок, RPS 318 → 239 / 283 → 243 | возврат в ротацию 2.9 / 3.1 с; данные целы | +| Рестарт API + PostgreSQL + Redis + S3 + LB | данные, сессия и файл сохранены | PASS, PID всех процессов сменились | ~38 с hook (из них прогрев S3 16 с); потерь нет | +| Остановка S3 | файлы → 503 быстро, остальное работает | upload 503 за 1.0–1.7 с, download 503 за 1.0 с, readiness 200, сообщения 201 | после старта S3 прогрев 17 с; старый файл побайтно цел | +| S3 «висит» (принимает соединение, не отвечает) | 503 в пределах дедлайна, не 504 | upload 503 за 10.0 с, download 503 за 5.0 с | прогрев 16 с; старый файл цел | +| Ошибка S3 PUT (`api-fault-s3`) | 503, строки нет | 503, 0 строк | сразу | +| Ошибка БД после PUT (`api-fault-db`) | 503, объект удалён | 503, 0 строк, число объектов не изменилось | сразу | +| Падение процесса после PUT (`api-fault-crash`) | pending + объект, убирает сборщик | 1 pending + объект → после grace (сдвинутого в тесте) 0 / 0 | ≤ grace + интервал (900 + 300 с по умолчанию) | +| Обрыв клиентом на 0.2 из 5 MiB | ничего не записано | 0 строк, объектов столько же | сразу | +| Нагрузка lab3 (`tests/load.py`, 20 с, c=4) | без регрессии | 276 RPS, 0 ошибок, p50/p95/p99 13/24/33 мс | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host, один PostgreSQL, один процесс SeaweedFS без репликации: отказ host/БД/S3 не переживается. S3 недоступен → файловые операции 503 (остальное работает). Восстановление S3 занимает 15–30 с прогрева. +- Тело загрузки держится в памяти процесса (≤ 10 MiB × 4 слота). Это ограничено и измерено, но не «настоящий» стриминг в S3. Следующий шаг — multipart upload кусками по 5 MiB с проверкой по частям или presigned upload (трек 1). +- Progressive JPEG проверяется структурно (маркеры, EOI), без декодирования пикселей: экономия ~115 MiB на 40 Мп. Испорченные энтропийные данные внутри структурно целого progressive-файла не обнаруживаются (их увидит lab5 при создании thumbnail → `failed`). +- `missing_objects` (ready без байтов) сборщик только сообщает; решение о судьбе метаданных — за оператором. +- Ключи идемпотентности, созданные репликой lab3 до обновления, имели отпечаток без `attachment_ids`; повтор такого ключа на lab4 даст 409 (окно 24 ч после rolling update). +- SIGKILL SeaweedFS через 10 с при остановке: безопасно для подтверждённых записей при живом ядре ОС, не при потере питания. +- Docker-прогон (`collect_evidence.py`) выполняется на машине студента. Нативный прогон подтверждает код, HAProxy-конфиг и флаги SeaweedFS, но не compose-обвязку. + +## Вопросы защиты — кратко + +- **Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?** Volume привязан к одному контейнеру и host. Другая реплика его не видит, поэтому после остановки загрузившей реплики файл недоступен (тест failover это и проверяет). У volume нет общего API доступа, ключей с правами и проверки целостности (Content-MD5), он не масштабируется независимо от API, а с read-only rootfs его пришлось бы ещё и открывать на запись. S3 — общий сервис с собственными ACL, его читают все реплики. +- **Когда можно честно вернуть 201?** Только после того, как S3 подтвердил PUT (с проверкой MD5) **и** COMMIT перевёл строку в `ready`. Раньше — нельзя: «готовое» вложение могло бы не скачаться. +- **S3 PUT прошёл, COMMIT — нет?** Остаются невидимая pending-строка и объект. Ошибку БД API компенсирует сам (удаляет объект, 503). Если процесс умер, сборщик после grace сначала атомарно удаляет pending-строку, потом объект. Объект без строки убирает проход по бакету. Строка всегда создаётся раньше объекта, поэтому «вечных» объектов нет. +- **Почему случайный ключ и приватный бакет не заменяют ACL приложения?** Приватный бакет защищает от анонимного доступа к S3, но клиенты в него и не ходят: все запросы идут через API с ключами API, которые читают **любой** объект бакета. Кто из пользователей что может скачать, знает только приложение (членство в чате). Случайный ключ — защита «неугадываемостью»: он может утечь (лог, ссылка), а доступ после выхода из чата или для чужого пользователя он не ограничивает. Поэтому каждый запрос проверяет членство и отвечает 404 постороннему. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..6f6bd929 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,396 @@ +# Лабораторная 4: HAProxy -> API x N (по умолчанию 2) -> PostgreSQL (метаданные) + Redis (кэш сессий) +# + SeaweedFS S3 (приватный бакет с байтами вложений). +# Сети: backend — lb, api, postgres, redis; storage (internal) — api, s3, s3-init. +# LB не видит S3, S3 не видит PostgreSQL; наружу (127.0.0.1) опубликован только LB. +name: messenger-lab4 + +x-api-env: &api-env + APP_ENV: ${APP_ENV:-lab} + APP_VERSION: ${APP_VERSION:-lab4} + SESSION_STORE: postgres + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.25} + CACHE_RETRY_AFTER_SECONDS: ${CACHE_RETRY_AFTER_SECONDS:-5} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + READY_DB_TIMEOUT_SECONDS: ${READY_DB_TIMEOUT_SECONDS:-1} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + GRACEFUL_TIMEOUT_SECONDS: ${GRACEFUL_TIMEOUT_SECONDS:-10} + KEEP_ALIVE_SECONDS: ${KEEP_ALIVE_SECONDS:-30} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + # Реплики НЕ применяют миграции — это делает сервис migrate. + RUN_MIGRATIONS_ON_START: "false" + # lab4: S3 (credentials API — отдельные, только Read/Write/List одного бакета) + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10} + S3_MAX_ATTEMPTS: ${S3_MAX_ATTEMPTS:-2} + # Общий бюджет вызова S3 (вкл. DNS) и circuit breaker — см. src/main.py s3_call + S3_DOWNLOAD_DEADLINE_SECONDS: ${S3_DOWNLOAD_DEADLINE_SECONDS:-5} + S3_UPLOAD_DEADLINE_SECONDS: ${S3_UPLOAD_DEADLINE_SECONDS:-10} + S3_CIRCUIT_OPEN_SECONDS: ${S3_CIRCUIT_OPEN_SECONDS:-5} + # Лимиты загрузки (не выше контракта: 10 MiB, 40 Мп) + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-4} + UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} + UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-60} + IMAGE_DECODE_CONCURRENCY: ${IMAGE_DECODE_CONCURRENCY:-2} + # Сборщик незавершённых загрузок и объектов-сирот (src/reconcile.py) + ATTACHMENT_GC_INTERVAL_SECONDS: ${ATTACHMENT_GC_INTERVAL_SECONDS:-300} + ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900} + ORPHAN_SWEEP_INTERVAL_SECONDS: ${ORPHAN_SWEEP_INTERVAL_SECONDS:-3600} + +x-api-healthcheck: &api-healthcheck + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 3s + timeout: 2s + retries: 2 + start_period: 20s + +x-api-limits: &api-limits + limits: + cpus: "0.50" + # lab4: 4 одновременные загрузки по 10 MiB; замер пика — evidence/upload_memory*.json + memory: 320M + +x-api-secrets: &api-secrets + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + +services: + postgres: + image: postgres:16.4-alpine + restart: unless-stopped + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + # (API_REPLICAS + 1 при rolling update) x DB_POOL_MAX + migrate + запас < 100 + command: ["postgres", "-c", "max_connections=100"] + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [backend] + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + redis: + image: redis:7.4.1-alpine + restart: unless-stopped + user: redis + # Только кэш проверок сессии: без RDB/AOF, ограничение памяти и LRU. + command: + - sh + - -c + - exec redis-server --requirepass "$$(cat /run/secrets/redis_password)" --save "" --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru + secrets: + - redis_password + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: + cpus: "0.25" + memory: 96M + + # lab4: S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 gateway в + # одном процессе `weed server`). Наружу НЕ публикуется. Административные HTTP-порты + # (master 9333, filer 8888, volume 8080) слушают только 127.0.0.1 ВНУТРИ контейнера + # (-ip.bind), в сеть storage выставлен только S3 API (8333). Анонимного доступа нет: + # identities из secrets/s3_config (bootstrap-admin и messenger-api с правами + # Read/Write/List только на свой бакет). Bucket создаёт s3-init. + s3: + image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.50} + restart: unless-stopped + entrypoint: ["/usr/bin/weed"] + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + - -s3.port=8333 + - -s3.ip.bind=0.0.0.0 + - -s3.config=/run/secrets/s3_config + - -s3.iam=false + - -s3.port.iceberg=0 + - -s3.port.lance=0 + - -s3.autoCreateBucket=false + - -master.volumeSizeLimitMB=256 + - -master.electionTimeout=2s + - -master.telemetry=false + - -volume.max=0 + secrets: + - s3_config + volumes: + - s3data:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [storage] + # Штатная остановка SeaweedFS длится ~20 с; Docker ждёт 10 с и шлёт SIGKILL. + # Подтверждённые записи уже в page cache ОС, том консистентен (см. REPORT.md). + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + # Идемпотентный bootstrap бакета + проверка приватности и прав API-ключей. + # Повторный запуск: "already exists (idempotent no-op)". Admin-ключи есть только здесь. + s3-init: + build: . + command: ["python", "-m", "src.s3_bootstrap"] + restart: "no" + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: + condition: service_healthy + networks: [storage] + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.50" + memory: 192M + + # Однократные миграции до старта реплик. Повтор — no-op (schema_migrations), + # параллельный запуск сериализуется pg_advisory_lock. + migrate: + build: . + command: ["python", "-m", "src.migrate"] + restart: "no" + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + # DDL может идти дольше обычного запроса API + DB_STATEMENT_TIMEOUT_MS: "60000" + MIGRATE_WAIT_SECONDS: "60" + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + networks: [backend] + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + api: + build: . + image: messenger-api:${APP_VERSION:-lab4} + restart: unless-stopped + environment: *api-env + secrets: *api-secrets + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + # Порт только внутри сети backend — в обход LB API недоступен. + expose: + - "8080" + healthcheck: *api-healthcheck + # SIGTERM -> осушение DRAIN_SECONDS (3 с) -> graceful shutdown (<= 10 с). + stop_signal: SIGTERM + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=16m + networks: [backend, storage] + # Остановленный s3 исчезает из Docker DNS; короткий таймаут резолвера. + dns_opt: ["timeout:1", "attempts:1"] + deploy: + replicas: ${API_REPLICAS:-2} + resources: *api-limits + + # Экземпляр с коротким TTL — только для автотеста истечения сессии (профиль ttl). + # Не входит в LB (другое DNS-имя), портов наружу не публикует: тест запускается + # внутри сети командой `make test-expiry` + api-short-ttl: + image: messenger-api:${APP_VERSION:-lab4} + profiles: [ttl] + restart: "no" + environment: + <<: *api-env + SESSION_TTL_SECONDS: ${SHORT_SESSION_TTL_SECONDS:-2} + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: + - "8080" + healthcheck: *api-healthcheck + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=16m + networks: [backend, storage] + deploy: + resources: *api-limits + + # lab4: экземпляры с ВНЕДРЁННЫМИ отказами пути загрузки (профиль faults), только для + # student-tests/test_attachment_consistency.py. Не входят в LB (другие DNS-имена), + # порты не публикуют. Запуск: python scripts/ops.py in-network consistency + api-fault-s3: + image: messenger-api:${APP_VERSION:-lab4} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: s3_error_on_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage] + deploy: + resources: *api-limits + + api-fault-db: + image: messenger-api:${APP_VERSION:-lab4} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: db_error_after_s3_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage] + deploy: + resources: *api-limits + + # Процесс завершается (os._exit) между S3 PUT и COMMIT — «падение» реплики. + api-fault-crash: + image: messenger-api:${APP_VERSION:-lab4} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: crash_after_s3_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage] + deploy: + resources: *api-limits + + lb: + image: haproxy:2.8.30-alpine + restart: unless-stopped + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + stop_grace_period: 5s + networks: [backend] + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + +volumes: + pgdata: + s3data: + +networks: + backend: + # Внутренняя сеть хранилища объектов: без выхода наружу и без LB. + storage: + internal: true + +secrets: + postgres_password: + file: ./secrets/postgres_password + redis_password: + file: ./secrets/redis_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/docs/lab3/REPORT.md b/docs/lab3/REPORT.md new file mode 100644 index 00000000..641d0224 --- /dev/null +++ b/docs/lab3/REPORT.md @@ -0,0 +1,231 @@ +# Отчёт — лабораторная 3 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0; PostgreSQL 16, Redis 7, HAProxy 2.8 LTS. Документация: Docker Compose (services, deploy, depends_on, healthcheck), HAProxy 2.8 Configuration Manual (server-template, resolvers, http-check, retries/redispatch, http-error), PostgreSQL (INSERT … ON CONFLICT, deferrable constraints, advisory locks), Uvicorn (graceful shutdown). Помощь ИИ (Claude) использовалась при проектировании схемы идемпотентности и drain-фазы, написании скриптов стенда, тестов и формулировке отчёта; решения проверены тестами, приведёнными ниже. + +## Запуск из чистого clone + +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`. Зависимости Python закреплены в `requirements.txt`. Нужны Docker Desktop / Docker Engine с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Вариант оркестрации — **Compose + HAProxy** (Swarm не используется). + +```powershell +$env:PYTHONUTF8 = "1" +# 1. Локальные секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py +copy .env.example .env # Linux/macOS: cp .env.example .env + +# 2. Проверка, сборка и запуск: postgres+redis -> migrate (однократно) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Ожидание готовности: 2 реплики healthy, обе UP в HAProxy, /health/ready через LB = 200 +python scripts/ops.py wait +python scripts/ops.py ps # X-Instance-Id <-> контейнер <-> статус в LB +``` + +Время прогрева: первая сборка образа 1–2 мин; дальше стенд готов примерно через 20–30 с (healthcheck PostgreSQL, миграции, `start_period` API, `rise 2` в HAProxy). + +```powershell +# Публичные проверки (Linux/macOS: make check / make test / make test-persistence ... ) +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd # Linux/macOS: scripts/restart.sh +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd # Linux/macOS: scripts/stop-one.sh +python scripts/ops.py restore # вернуть остановленную реплику; затем повторить failover для другой + +# Собственные проверки уровня 4 +python student-tests/run_all.py --base-url http://localhost:8080 # API, auth, идемпотентность через реплики, race +python scripts/ops.py in-network storage # тесты с прямым доступом к БД (внутри compose-сети) +python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl, TTL 2 с) +python student-tests/failover_series.py --output evidence/failover_series.json # каждая реплика по очереди, SIGTERM +python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json # то же, SIGKILL +python student-tests/rolling_update_load.py --output evidence/rolling_update.json # rolling update под нагрузкой +python tests/load.py --duration 30 --concurrency 8 --output evidence/load.json + +# Всё перечисленное одной командой с записью результатов в evidence/ (~8–10 мин): +python scripts/collect_evidence.py +``` + +**Rolling update.** Задать новый тег `APP_VERSION` (например, `lab3.1`) в `.env` или в окружении и выполнить `python scripts/ops.py rolling-update`: сборка образа, однократные миграции (`docker compose run --rm migrate`), затем для каждой старой реплики: +1 новая → healthy и UP в HAProxy → graceful stop старой → удаление. + +**Остановка.** Безопасная, данные сохраняются: `docker compose --env-file .env --profile ttl down` (volume `pgdata` остаётся). Намеренное удаление всех учебных данных — только вручную: `docker compose --env-file .env --profile ttl down -v`. Hooks и тесты никогда не удаляют volumes. + +**Действия оператора:** создать секреты до первого запуска; пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` не перезаписывает существующие файлы. После `stop-one`/тестов отказа вернуть реплику `python scripts/ops.py restore`. + +## Архитектура и решения + +``` + 127.0.0.1:8080 (единственная публичная точка входа API) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки GET /health/ready + │ 127.0.0.1:8404 — страница состояния (UP/DOWN реплик), только чтение + │ обнаружение реплик: Docker DNS имени `api` -> server-template api1..api6 + ▼ + ┌───────── api (deploy.replicas = 2, без ports, без container_name) ─────────┐ + │ api-1: X-Instance-Id = hostname = короткий ID контейнера 0.5 CPU / 256M │ + │ api-2: ... 0.5 CPU / 256M │ + └──────┬──────────────────────────────────────────────────────────┬──────────┘ + ▼ ▼ + postgres 16 (1 CPU / 512M, volume pgdata) redis 7 (0.25 CPU / 96M) + источник истины: users, chats, messages, общий кэш проверок токена, + sessions, idempotency_keys, schema_migrations без persistence + ▲ + migrate (one-shot): python -m src.migrate — до старта реплик, один раз +``` + +Порядок старта задаётся `depends_on`: `postgres`, `redis` (healthy) → `migrate` (`service_completed_successfully`) → `api` (healthy) → `lb`. API-реплики не публикуют порты: `expose: 8080` только внутри сети `backend`. Наружу (на 127.0.0.1) опубликован только `lb`. + +### Почему HAProxy, а не Nginx + +В open-source Nginx нет активных health checks: он узнаёт об отказе только по ошибкам живых запросов (`max_fails`). HAProxy сам опрашивает `/health/ready` с заданным периодом. Поэтому время обнаружения отказа можно указать числом, а фаза осушения при SIGTERM действительно выводит реплику из ротации до остановки. `server-template` + `resolvers` синхронизируют список реплик с DNS-записями сервиса `api` каждые 2 с. Масштабирование и rolling update поэтому не требуют перезапуска или перечитывания конфига LB. Sticky sessions не используются (`balance roundrobin`, без cookie). + +### Что перенесено из lab2 и что изменено + +Перенесены без изменения семантики: модели и валидация, формат ошибок, keyset-пагинация, Argon2id, Basic → Bearer, сессии в PostgreSQL + кэш в Redis с надёжной инвалидацией (`session_cache_invalidations`), собственные тесты lab2. Схема lab2 уже рассчитана на несколько реплик: сессии лежат в общей БД, кэш — в общем Redis, а долг инвалидации перечитывается всеми репликами не реже раза в секунду. + +Изменено в lab3: + +1. **Переход 2 → 3 по контракту:** `X-Instance-Id` во всех ответах API, включая ошибки и health (middleware + `error_response`). Обязательный `Idempotency-Key` для `POST /chats/{id}/messages`. +2. **Общая идемпотентность** — таблица `idempotency_keys` (миграция `003`), подробности ниже. +3. **Миграции вынесены из реплик** в one-shot сервис `migrate` (`RUN_MIGRATIONS_ON_START=false`). Реплика готова только когда в `schema_migrations` есть последняя версия, которую знает её образ. +4. **Graceful shutdown с осушением** (`src/server.py`): по SIGTERM реплика сразу отвечает 503 на readiness, но ещё 3 с обслуживает запросы, затем штатно останавливается. +5. **Лимит одновременных хешей Argon2id** (`PASSWORD_HASH_CONCURRENCY=4`). У реплики лимит 256M, а threadpool FastAPI — до 40 потоков; без лимита всплеск входов мог бы занять 40 × 19 MiB и привести к OOM-kill. +6. **Readiness** использует отдельный короткий таймаут БД (`READY_DB_TIMEOUT_SECONDS=1`) меньше `timeout check` HAProxy. +7. Тесты lab2 адаптированы к переходу: при отправке сообщения добавляется `Idempotency-Key`, имя проекта сменилось на `messenger-lab3`. Тест истечения сессии и тесты хранилища теперь идут внутри compose-сети, без публикации портов в обход LB. + +### Идемпотентность отправки сообщения (`src/storage.py`, `migrations/003_idempotency_keys.sql`) + +- Ключ: `PRIMARY KEY (user_id, method, chat_id, idem_key)` — ровно область из контракта «пользователь + метод + путь чата + ключ». В строке хранятся SHA-256 канонического JSON тела (`sort_keys`, без пробелов: порядок ключей и форматирование не важны), `message_id`, `expires_at = now + IDEMPOTENCY_TTL_SECONDS (86400)`. +- **Одна транзакция READ COMMITTED:** + 1. `INSERT INTO idempotency_keys … ON CONFLICT DO UPDATE … WHERE expires_at <= now() RETURNING`. Конкурентная транзакция с тем же ключом **ждёт на уникальном индексе**, пока первая не сделает COMMIT/ROLLBACK. Это и даёт «ровно одно сообщение» при одновременных повторах через разные реплики: арбитр — PostgreSQL, а не память процесса. + 2. Ключ наш → `INSERT messages … WHERE EXISTS (членство)` с заранее выбранным id. FK `message_id → messages` объявлен `DEFERRABLE INITIALLY DEFERRED` и проверяется при COMMIT, поэтому `message_id NOT NULL` с первого шага. + 3. Ключ уже подтверждён → снова проверяется членство (и сессия — dependency `current_user`). Hash совпал → 201 и то же сообщение (+ информационный заголовок `Idempotency-Replayed: true`); не совпал → 409. +- Любая ошибка или обрыв до COMMIT откатывает и ключ, и сообщение: транзиентный 5xx не фиксируется. 404 и 400 ключ не «сжигают». Истёкший, но ещё не удалённый ключ можно занять заново. Фоновая очистка раз в 5 мин удаляет истёкшие ключи пачками под `pg_try_advisory_xact_lock`, то есть одной репликой за раз. +- Порядок проверок: аутентификация → ключ (нет/неверный → 400) → тело → доступ к чату (404). + +### Readiness, таймауты и остановка — согласованные значения + +| Параметр | Значение | Почему | +| --- | --- | --- | +| HAProxy `inter` / `fastinter` / `fall` / `rise` | 1 с / 0.5 с / 2 / 2 | DOWN после 2 неудачных проверок подряд, вторая — через 0.5 с | +| HAProxy `timeout check` | 1.5 с | больше `READY_DB_TIMEOUT_SECONDS` = 1 с | +| HAProxy `timeout connect`, `retries`, `option redispatch 1` | 1 с, 2, да | несостоявшееся соединение повторяется на другой реплике; запрос ещё не отправлен — безопасно и для POST | +| HAProxy `timeout server` | 10 с | > максимум запроса API: пул 3 с + `statement_timeout` 5 с | +| `resolvers` hold valid/obsolete | 2 с / 2 с | новые/исчезнувшие контейнеры попадают в LB за ≤ ~2–3 с | +| API `DRAIN_SECONDS` | 3 с | > худшее обнаружение DOWN при ответах 503 (≈ 1 + 0.5 с + задержка проверок) | +| API `GRACEFUL_TIMEOUT_SECONDS` | 10 с | дождаться запросов в обработке (≤ 8 с) | +| compose `stop_grace_period` (api) | 20 с | > 3 + 10 — SIGKILL не прилетает во время graceful stop | +| Docker healthcheck api | 3 с / timeout 2 с / retries 2 / start_period 20 с | для `depends_on` и `ops.py` (rolling update ждёт healthy) | +| Uvicorn keep-alive | 30 с | дольше, чем LB держит простаивающие соединения к серверу | + +**Максимальное время обнаружения отказа балансировщиком:** graceful stop (сразу 503) и падение процесса (соединение отклонено) — до ≈ 1.5–2 с; зависшая реплика без ответа — до 1 + 1.5 + 0.5 + 1.5 = **4.5 с**. Пока реплика не помечена DOWN, запрос к ней, не сумевший соединиться за 1 с, уходит на другую. **Измерено** (`evidence/failover_series*.json`, `evidence/shutdown_timeline.txt`): DOWN через 0.96–1.41 с после SIGTERM и через 1.10–1.28 с после SIGKILL. + +**Временная шкала graceful stop** (`evidence/shutdown_timeline.txt`): SIGTERM → +0.01 с readiness 503 → +1.30 с HAProxy DOWN (новые запросы больше не приходят) → +3.02 с закрыт listening-сокет → +3.12 с процесс завершён. LB выводит реплику из ротации на ~1.7 с раньше, чем она перестаёт принимать соединения, поэтому graceful stop не даёт ошибок клиенту. + +**Критичные зависимости readiness:** PostgreSQL (и наличие мигрированной схемы). Redis не критичен — это кэш с обходом (circuit breaker 5 с), как в lab2. Если ни одна реплика не готова, LB сам отвечает `503` в общем JSON-формате ошибок с тем же `X-Request-Id` (`http-error`, `evidence/lb_errors.txt`). `X-Instance-Id` в таком ответе нет — контракт это допускает. + +### Миграции и совместимость с работающей версией + +- `migrate` выполняется **один раз** до старта реплик. Повтор — no-op (`schema_migrations`); две одновременные копии сериализуются `pg_advisory_lock` (`evidence/migrations.txt`: копия B применила 3 миграции, копия A — `nothing applied`). Реплики миграции не запускают и не готовы, пока схема не мигрирована: на пустой БД readiness = 503 `Database schema is not migrated`, после `migrate` — 200 без рестарта. +- Rolling update применяет миграции **до** замены реплик, поэтому допускаются только **расширяющие** (expand) миграции, совместимые со старой версией. `003` только добавляет таблицу. Проверено: код lab2 (старая версия) проходит smoke lab2 на схеме lab3 одновременно с репликами lab3 (`evidence/migrations.txt`). +- Ограничение: смешанный парк «lab2 + lab3» во время обновления формально совместим по схеме, но реплики lab2 не требуют и не записывают `Idempotency-Key`. Поэтому гарантия идемпотентности действует только когда все реплики уже lab3. Удаляющие/переименовывающие миграции нужно делить на expand → переключение кода → contract отдельным релизом. + +### Соответствие `.env.example` и реальной конфигурации + +| Переменная | Где используется | +| --- | --- | +| `APP_HOST`, `APP_PORT` | внутри контейнера Uvicorn слушает `0.0.0.0:8080`; `APP_PORT` — публикуемый порт LB `127.0.0.1:${APP_PORT}` | +| `LB_STATS_PORT` | страница состояния HAProxy `127.0.0.1:8404` (`ops.py`, тесты отказов) | +| `APP_ENV`, `APP_VERSION` | `APP_VERSION` — тег образа `messenger-api:${APP_VERSION}` (rolling update) и строка в логе старта | +| `SESSION_TTL_SECONDS`, `SHORT_SESSION_TTL_SECONDS` | TTL сессий; второй — для `api-short-ttl` (профиль `ttl`) | +| `DATABASE_URL_FILE`, `CACHE_URL_FILE` | Compose secrets `/run/secrets/database_url`, `/run/secrets/cache_url` | +| `SESSION_STORE` | `postgres` — источник истины сессий; Redis — общий кэш | +| `SESSION_CACHE_TTL_SECONDS` | TTL записи кэша и «надгробия» logout | +| `API_REPLICAS` | `deploy.replicas` сервиса `api` (2); ожидаемое число реплик в `ops.py` | +| `IDEMPOTENCY_TTL_SECONDS` | срок хранения результата ключа (≥ 86400, иначе API не стартует) | +| `DB_POOL_*`, `DB_*_TIMEOUT*` | пул (1..10 на реплику), ожидание соединения 3 с, connect 3 с, statement 5 с | +| `CACHE_TIMEOUT_SECONDS`, `CACHE_RETRY_AFTER_SECONDS` | таймаут Redis 250 мс и обход кэша на 5 с после ошибки | +| `READY_DB_TIMEOUT_SECONDS`, `DRAIN_SECONDS`, `GRACEFUL_TIMEOUT_SECONDS`, `KEEP_ALIVE_SECONDS` | readiness и остановка — таблица выше | +| `PASSWORD_HASH_CONCURRENCY` | одновременных хешей Argon2id на реплику | + +`max_connections=100` у PostgreSQL ≥ (2 реплики + 1 во время rolling update) × 10 + migrate + api-short-ttl + запас. + +### Инфраструктура + +| Роль | Service | Версия | Health/readiness | Volume | Сеть/порт | Credentials | Лимиты | +| --- | --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | `haproxy:2.8.30-alpine` | `wget /lb-health` (monitor-uri) | — (конфиг ro bind) | `backend`; **127.0.0.1:8080**, 127.0.0.1:8404 | — | 0.5 CPU / 128M | +| api × 2 | `api` | сборка `Dockerfile`, `python:3.12.7-slim-bookworm` | `/health/ready` (БД + схема + не в drain) | нет, `read_only` + tmpfs `/tmp` | `backend`, `expose 8080` (наружу нет) | secrets `database_url`, `cache_url` | 0.5 CPU / 256M | +| миграции | `migrate` | тот же Dockerfile | one-shot, exit 0 | — | `backend` | secret `database_url` | 0.5 CPU / 256M | +| postgres | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `pgdata` | `backend`, наружу нет | secret `postgres_password` | 1 CPU / 512M | +| redis | `redis` | `redis:7.4.1-alpine`, user `redis` | `redis-cli ping` | нет (кэш) | `backend`, наружу нет | secret `redis_password` | 0.25 CPU / 96M | +| тест TTL | `api-short-ttl` (профиль `ttl`) | образ api | `/health/ready` | — | `backend`, не в LB, портов нет | как api | 0.5 CPU / 256M | + +`container_name` нигде не задан. Секреты общие только там, где нужны: пароль PostgreSQL есть только у `postgres`, пароль Redis — у `redis`, строки подключения — у `api`/`migrate`. Bootstrap: схема создаётся сервисом `migrate` на пустом volume; повтор безопасен. Hooks: `scripts/restart.sh|.cmd` → `ops.py restart` (stop lb+api → restart postgres+redis → start api → healthy → start lb); `scripts/stop-one.sh|.cmd` → `ops.py stop-one` (одна реплика по `TARGET_INSTANCE_ID`, остаётся остановленной). Каждое действие пишется в `evidence/ops.log`. + +## Доказательства критериев + +**Стенд, на котором сняты приложенные результаты.** Windows 10, Ryzen 5 7500f, 32 GB RAM. PostgreSQL 16, Redis 7 и HAProxy **2.8.30**, Docker Desktop. Код приложения, миграции и **тот же `infra/haproxy/haproxy.cfg`** использованы как есть. Только `server-template` + DNS заменены на 6 статических слотов, реплики запущены как `python -m src.server` (2 процесса), CPU/RAM-лимитов нет. Compose-файл проверен `docker compose config`, конфиг HAProxy — `haproxy -c` той же версии 2.8.30, что в образе. `scripts/ops.py` прогнан против заглушки `docker`. + +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab3 public API scenario` | `evidence/smoke.txt` | +| Контракт | `make check`, `make validate`, `docker compose config --quiet`, `haproxy -c` | все PASS / valid | `evidence/check.txt` | +| Стенд в `compose.yaml`, ≥ 2 реплики, одна точка входа, общая БД и хранилище сессий | `compose.yaml`; `python scripts/ops.py ps` | `api` × 2 без `ports`; публикуется только `lb`; одна `postgres`, один `redis` | `compose.yaml`, `evidence/stand.txt` (после `collect_evidence.py`) | +| Разные запросы одного пользователя — разными репликами без повторного входа | 20 × `GET /users/me` с одним токеном через LB | чередование двух `X-Instance-Id`, все 200 | `evidence/instances.txt` | +| Остановка любой одной реплики: ≤ 10 с, чтение/запись продолжаются, подтверждённые доступны | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` × 2 (вторая реплика после восстановления первой) + SIGKILL | PASS ×3; `failed_attempts: 0`; запись не прерывалась (первая запись после возврата hook — 3.3 с = длительность graceful hook; при SIGKILL — 0.11 с); остановлены разные реплики | `evidence/failover.txt` | +| Какой процесс реально остановлен | журнал hook | `SIGTERM nat-api-4` в прогоне 1, `SIGTERM nat-api-3` в прогоне 2 (в compose — `evidence/ops.log`: имя контейнера, IP, состояние после) | `evidence/failover.txt` | +| Повтор POST с тем же ключом не создаёт дубль, в т.ч. конкурентно и при смене реплики | smoke; `test_idempotency.py` (24 параллельных POST, Barrier); failover-тест повторяет неопределённые POST тем же ключом | 24 × 201, один `id`, ответы через **обе** реплики, в истории 1 сообщение; дублей нет ни в одном сценарии отказа | `evidence/student-tests.txt`, `evidence/failover*.txt` | +| Persistence: рестарт стенда с volumes и тем же токеном | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; PID до/после: postgres 1469→2901, redis 1479→2911, api 2846→2917 и 2866→2925, HAProxy перезапущен | `evidence/persistence.txt` | +| Миграции отдельно и согласованно | сервис `migrate`; readiness проверяет схему | реплики не мигрируют; повтор no-op; гонка двух `migrate` безопасна | `evidence/migrations.txt` | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Health/readiness, timeouts и SIGTERM согласованы; LB исключает недоступный backend; период проверок и макс. время обнаружения | `infra/haproxy/haproxy.cfg`, `src/server.py`; замер шкалы остановки | проверки каждые 1 с (0.5 с после первой неудачи), fall 2; обнаружение ≤ 1.5–2 с (503/RST), ≤ 4.5 с (зависание). Замер: 503 на readiness через 0.01 с, DOWN через 1.30 с, приём прекращён через 3.02 с | таблица выше, `evidence/shutdown_timeline.txt`, `evidence/lb_log_sample.txt` | +| Последовательно остановить каждую реплику (восстанавливая предыдущую); RPS, ошибки, время восстановления | `failover_series.py` (4 писателя + 2 читателя, timeout клиента 2 с) | **SIGTERM:** 0 ошибок; DOWN в LB через 1.41 / 0.96 с; RPS 407 → 324 (одна реплика) → 354 после restore; p95 30–31 мс; реплика снова в ротации через 3.0 / 2.6 с после restore. **SIGKILL:** 6 / 3 оборванных запроса (502) в окне ≤ 0.21 с после kill, все записи повторены тем же ключом без дублей; DOWN через 1.28 / 1.10 с | `evidence/failover_series.json`, `evidence/failover_series_kill.json`, `evidence/failover_series.txt` | +| Общая идемпотентность параллельными запросами с одним ключом | `test_idempotency.py`, `test_storage_idempotency.py` | 24 параллельных → 1 сообщение через 2 реплики; 12 разных тел с одним ключом → один победитель, остальные 409; 16 потоков на уровне хранилища → одна вставка; сбой после вставки ключа и сообщения → ни ключа, ни сообщения, повтор успешен | `evidence/student-tests.txt` | +| CPU/RAM-лимиты; API без портов в обход LB; секреты/хранилища общие только где нужно; нет `container_name` | `compose.yaml` | лимиты у всех сервисов (api 0.5 CPU / 256M); у `api` только `expose`; `container_name` отсутствует; RSS реплики под нагрузкой 132 MB < 256M | `compose.yaml`, `evidence/load.txt` | +| Rolling update под нагрузкой | `rolling_update_load.py` → `ops.py rolling-update` (на нативном стенде — тот же алгоритм) | замена 2 реплик за 11.8 с, **0 ошибок**, RPS 388 → 381 во время → 385 после, p95 31 → 34 мс; после — только новые реплики; 5709 подтверждённых сообщений на месте, дублей нет | `evidence/rolling_update.json`, `evidence/rolling_update.txt` | +| Миграции однократно и совместимы с работающей версией | сервис `migrate`; код lab2 на схеме lab3 | см. раздел «Миграции»; ограничение смешанного парка описано | `evidence/migrations.txt` | +| Возможности lab2 сохранены | `run_all.py` (47 тестов + race), `test_expiry` на экземпляре с TTL 2 с | все PASS (1 skip — expiry в общем прогоне, запускается отдельно и проходит) | `evidence/student-tests.txt` | + +## Эксперименты и отказы + +**Гипотеза.** Если всё состояние, от которого зависит корректность (данные, сессии, ключи идемпотентности), лежит в общем PostgreSQL, а LB активно проверяет готовность и реплика по SIGTERM сначала выходит из ротации, то: (1) graceful-остановка любой реплики и rolling update проходят без ошибок для клиента; (2) падение реплики даёт только кратковременные ошибки запросов, которые были в обработке, и клиент безопасно повторяет их с тем же ключом; (3) дублей и потерь подтверждённых сообщений нет. Подтверждено измерениями ниже. + +**Методика.** Генератор `student-tests/loadgen.py`: 4 писателя (POST с `Idempotency-Key`; при обрыве, timeout или 5xx — повтор с тем же ключом и телом до 201) и 2 читателя (история, `/users/me`); timeout клиента 2 с; опрос страницы HAProxy каждые 100 мс. Каждый раунд: 5 с прогрева → действие → 15 с без реплики → restore → 3 с. После раунда — полный обход истории и сверка со всеми подтверждёнными ответами. Стенд — см. «Доказательства»: клиент на той же машине, без лимитов. Повторов для статистики нет: разброс RPS между раундами ±5–8 %. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| SIGTERM реплики 1 под нагрузкой | 0 ошибок, LB выводит до остановки | 0 ошибок; DOWN через 1.41 с; реплика отвечала ещё 1.34 с (до DOWN) | hook 3.3 с; RPS 407 → 324 → 354; возврат в ротацию 3.0 с после restore; потерь нет | +| SIGTERM реплики 2 | то же | 0 ошибок; DOWN через 0.96 с | hook 3.4 с; RPS 393 → 304 → 348; возврат 2.6 с; потерь нет | +| SIGKILL реплики 1 | ошибки только у запросов в обработке, повтор без дублей | 6 × 502 в окне 0.21 с, 4 повтора записи тем же ключом; DOWN через 1.28 с | новые запросы успешны сразу (redispatch); возврат 2.8 с; дублей нет | +| SIGKILL реплики 2 | то же | 3 × 502 в окне 0.14 с, 1 повтор записи; DOWN через 1.10 с | возврат 2.7 с; дублей нет | +| Публичный failover (×2 graceful, ×1 kill) | PASS, ≤ 10 с | PASS; 0 failed attempts | первая запись после hook: 3.3 с / 3.4 с / 0.11 с | +| Rolling update 2 реплик под нагрузкой | 0 ошибок, после — только новые | 0 ошибок; `[api1,api2] → [api1,api2,api3] → [api2,api3] → … → [api3,api4]` | 11.8 с; RPS не просел (388 / 381 / 385) | +| Рестарт API + PostgreSQL + Redis + LB | данные и сессия сохраняются | PASS, тот же токен | потерь нет | +| Все реплики недоступны | LB отвечает 503 в формате контракта | JSON 503 с `request_id` = `X-Request-Id` | — | + +**Нагрузка** (`tests/load.py`, closed-loop POST, без retry): 30 с, concurrency 8 → 10 955 успешных, 0 ошибок, **364 RPS**, p50 19.9 мс, p95 42.8 мс, p99 53.4 мс (только успешные); набор данных — один чат, ~11 тыс. новых сообщений. RSS каждой реплики 132 MB, 14 потоков. Во время отказа одной реплики RPS падает примерно на 20 %: при 2 vCPU на машину одна реплика (один процесс Uvicorn) упирается в CPU. Это пропускная способность HTTP-подтверждений в данном профиле, а не максимум системы; с лимитами 0.5 CPU на реплику в Docker цифры будут ниже. + +## Ограничения и следующие шаги + +- **Отказы, которые выдерживаются:** остановка или падение любой одной API-реплики, rolling update реплик, рестарт всего стенда с volumes. **Не выдерживаются:** отказ `lb`, `postgres` (единственная копия, без репликации) и Docker-host — это SPOF. Отказ `redis` не влияет на корректность (обход кэша, как в lab2). +- При падении (SIGKILL) запросы, которые реплика обрабатывала в этот момент, завершаются ошибкой 502. LB не повторяет уже отправленные POST — по контракту это делает клиент с тем же `Idempotency-Key`. Для `POST /users` и `POST /chats` повтор после неопределённого результата может создать второй объект (контракт этого не запрещает и прозрачных повторов не требует). +- HAProxy держит до 6 слотов реплик (`server-template api 1-6`). Больше — поменять число в конфиге. +- Страница состояния HAProxy на 127.0.0.1:8404 без аутентификации, только чтение и только localhost. +- Приложенные результаты сняты на нативном эквиваленте стенда (см. «Доказательства»), без CPU/RAM-лимитов и без Docker DNS. Перед защитой их нужно повторить `python scripts/collect_evidence.py` на Docker; конкретные цифры на ноутбуке будут другими. +- Время обнаружения на Docker дополнительно включает DNS-синхронизацию (до ~2 с на появление новой реплики в LB). На отказ это не влияет: остановленную реплику исключают активные проверки раньше, чем DNS. +- Идемпотентность гарантирована, только когда все реплики — lab3 (см. «Миграции»). +- HTTP без TLS — допустимо только на локальном учебном стенде. Пароль Redis передаётся `redis-server` аргументом внутри контейнера (как в lab2). + +## Вопросы защиты — кратко + +- **Отказ процесса vs отказ узла.** Процесс (контейнер реплики) падает, остальные реплики на том же host живы, общие PostgreSQL/Redis/LB целы — это и проверяет лаба. При отказе узла пропадает всё, что на нём: здесь это весь стенд, включая LB и единственную БД. Защита от него — несколько host, репликация БД, LB с переносом адреса (трек 2 уровня 5, не заявлен). +- **Почему depends_on/healthcheck — не балансировщик.** `depends_on` управляет только порядком старта, а Docker healthcheck лишь меняет статус контейнера. Ни один из них не перенаправляет трафик и не выводит больную реплику из ротации. Это делает LB своими проверками (здесь — HAProxy `http-check` каждые 1 с). +- **Какие POST можно повторять после timeout.** Только те, что защищены ключом идемпотентности: здесь `POST /chats/{id}/messages` с тем же `Idempotency-Key` и тем же телом. Ключ защищает от дубля, если ответ потерялся после COMMIT: повтор вернёт исходное сообщение, а не создаст второе. `POST /users`, `POST /chats`, `POST /auth/sessions` после неопределённого результата повторять «вслепую» нельзя — создастся новый объект или новая сессия. +- **Сессии в локальной памяти при росте числа реплик.** Токен, выданный репликой A, неизвестен реплике B: каждый второй запрос получит 401, logout на одной реплике не отзовёт сессию на другой, рестарт реплики разлогинит её пользователей. «Чинить» это sticky sessions — значит терять сессии при отказе реплики. Поэтому сессии лежат в общем PostgreSQL (кэш — общий Redis), а ключи идемпотентности — в общей БД. diff --git a/docs/lab3/evidence/README.md b/docs/lab3/evidence/README.md new file mode 100644 index 00000000..d074e56d --- /dev/null +++ b/docs/lab3/evidence/README.md @@ -0,0 +1,25 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Файлы этой работы (lab3) + +`python scripts/collect_evidence.py` на compose-стенде перезаписывает их результатами Docker; `SUMMARY.txt` — итог по шагам. + +| Файл | Что доказывает | +| --- | --- | +| `check.txt` | make check / validate, `docker compose config`, `haproxy -c` | +| `smoke.txt` | публичный smoke lab3 через LB | +| `instances.txt` | одна сессия обслуживается разными репликами (X-Instance-Id) | +| `student-tests.txt` | собственные тесты: lab2 + идемпотентность через реплики + хранилище + expiry | +| `failover.txt` | публичный failover для обеих реплик и SIGKILL; какой процесс/контейнер остановлен | +| `failover_series*.json/.txt` | уровень 4: каждая реплика по очереди под нагрузкой — RPS, ошибки, обнаружение, восстановление | +| `rolling_update*.json/.txt` | уровень 4: rolling update под нагрузкой | +| `persistence.txt` | рестарт API + PostgreSQL + Redis + LB с тем же токеном | +| `migrations.txt` | миграции однократно, readiness ждёт схему, совместимость со старой версией | +| `shutdown_timeline.txt` | SIGTERM → readiness 503 → DOWN в LB → выход процесса | +| `lb_errors.txt`, `lb_log_sample.txt` | ответы и журнал балансировщика | +| `load.json`, `load.txt` | нагрузка 30 с, RSS реплик | +| `ops.log` | журнал scripts/ops.py (реальные контейнеры — после прогона на Docker) | diff --git a/docs/lab3/evidence/SUMMARY.txt b/docs/lab3/evidence/SUMMARY.txt new file mode 100644 index 00000000..0a23b4e2 --- /dev/null +++ b/docs/lab3/evidence/SUMMARY.txt @@ -0,0 +1,34 @@ +# 2026-10-26 19:35:57 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Rolling update lab3 -> lab3.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab3/evidence/check.txt b/docs/lab3/evidence/check.txt new file mode 100644 index 00000000..a01561ad --- /dev/null +++ b/docs/lab3/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-10-26 19:29:57 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-10-26 19:29:57 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab3/evidence/failover.txt b/docs/lab3/evidence/failover.txt new file mode 100644 index 00000000..6375f767 --- /dev/null +++ b/docs/lab3/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-10-26 19:30:28 UTC | Failover #1: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP +# 2026-10-26 19:30:44 UTC | Failover #1: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 4.672} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:30:44 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:30:51 UTC | Failover #1: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:30:51.183Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 6.3s + +# 2026-10-26 19:30:51 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:07 UTC | Failover #2: make test-failover | exit 0 | 16.2s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.094} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:07 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:31:13 UTC | Failover #2: restore | exit 0 | 6.0s +$ python scripts/ops.py restore +2026-10-26T19:31:13.832Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 5.9s + +# 2026-10-26 19:31:14 UTC | Failover #3: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:30 UTC | Failover #3: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.766} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:30 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 exited unhealthy messenger-api:lab3 - - +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:37 UTC | Failover #3: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:31:37.160Z restore: started ['messenger-lab3-api-1']; 2 replicas healthy and UP in LB after 6.3s + +# остановленные публичным тестом реплики: ['a21409fffeec', 'a21409fffeec', 'c4c5ae5b0949'] diff --git a/docs/lab3/evidence/failover_series.json b/docs/lab3/evidence/failover_series.json new file mode 100644 index 00000000..571c5c34 --- /dev/null +++ b/docs/lab3/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.09, + "lb_detect_down_seconds": 1.77, + "target_last_served_after_stop_seconds": 1.67, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.7, + "lb_back_up_seconds": 1.39, + "target_back_in_rotation_seconds": 6.69, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1322, + "ok": 1322, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 264.4, + "write_ok_rps": 159.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1059, + "ok": 1059, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 105.9, + "write_ok_rps": 64.0, + "latency_ms_ok": { + "p50": 47.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 53.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.0, + "requests": 1618, + "ok": 1618, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 161.8, + "write_ok_rps": 96.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.77, + [ + "api1" + ] + ], + [ + 21.39, + [ + "api1", + "api2" + ] + ], + [ + 21.77, + [ + "api1" + ] + ], + [ + 24.28, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2674, + "stored_messages": 2674, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.03, + "lb_detect_down_seconds": 0.97, + "target_last_served_after_stop_seconds": 0.91, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.56, + "lb_back_up_seconds": 0.67, + "target_back_in_rotation_seconds": 6.56, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1353, + "ok": 1353, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 269.7, + "write_ok_rps": 158.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 990, + "ok": 990, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 99.0, + "write_ok_rps": 59.2, + "latency_ms_ok": { + "p50": 62.0, + "p95": 156.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.03, + "requests": 410, + "ok": 410, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 81.8, + "write_ok_rps": 49.1, + "latency_ms_ok": { + "p50": 93.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.03, + "to_s": 29.86, + "requests": 1602, + "ok": 1601, + "errors": 1, + "error_ratio": 0.0006, + "ok_rps": 162.9, + "write_ok_rps": 98.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 5.98, + [ + "api2" + ] + ], + [ + 20.7, + [ + "api1", + "api2" + ] + ], + [ + 20.94, + [ + "api2" + ] + ], + [ + 23.47, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2604, + "stored_messages": 2604, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/failover_series.txt b/docs/lab3/evidence/failover_series.txt new file mode 100644 index 00000000..e4f0e742 --- /dev/null +++ b/docs/lab3/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-10-26 19:32:39 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 61.9s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=stop) under load + hook 4.09s | LB DOWN after 1.77 s | errors 0 (-) | RPS before 264.4, outage 105.9, after restore 161.8 | p95 outage 110.0 ms | back in rotation 6.69 s | data ok=True +-- stopping c4c5ae5b0949 (mode=stop) under load + hook 4.03s | LB DOWN after 0.97 s | errors 0 (-) | RPS before 269.7, outage 99.0, after restore 162.9 | p95 outage 156.0 ms | back in rotation 6.56 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-10-26 19:33:42 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 63.1s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=kill) under load + hook 1.01s | LB DOWN after 2.53 s | errors 0 (-) | RPS before 235.6, outage 76.1, after restore 146.5 | p95 outage 187.0 ms | back in rotation 7.06 s | data ok=True +-- stopping c4c5ae5b0949 (mode=kill) under load + hook 0.92s | LB DOWN after 2.38 s | errors 0 (-) | RPS before 270.1, outage 78.3, after restore 149.7 | p95 outage 187.0 ms | back in rotation 7.19 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab3/evidence/failover_series_kill.json b/docs/lab3/evidence/failover_series_kill.json new file mode 100644 index 00000000..f54d5b8d --- /dev/null +++ b/docs/lab3/evidence/failover_series_kill.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.01, + "lb_detect_down_seconds": 2.53, + "target_last_served_after_stop_seconds": 0.5, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.09, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1178, + "ok": 1178, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 235.6, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 761, + "ok": 761, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 76.1, + "write_ok_rps": 46.0, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.01, + "requests": 454, + "ok": 454, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 90.5, + "write_ok_rps": 53.8, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 30.44, + "requests": 1527, + "ok": 1527, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 146.5, + "write_ok_rps": 85.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.53, + [ + "api1" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.94, + [ + "api1" + ] + ], + [ + 24.48, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2318, + "stored_messages": 2318, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 0.92, + "lb_detect_down_seconds": 2.38, + "target_last_served_after_stop_seconds": 0.44, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.25, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.19, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1355, + "ok": 1355, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 270.1, + "write_ok_rps": 156.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 783, + "ok": 783, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 78.3, + "write_ok_rps": 47.5, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1016.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 52.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 125.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 30.56, + "requests": 1579, + "ok": 1579, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 149.7, + "write_ok_rps": 89.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.39, + [ + "api2" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.58, + [ + "api2" + ] + ], + [ + 24.11, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2474, + "stored_messages": 2474, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/instances.txt b/docs/lab3/evidence/instances.txt new file mode 100644 index 00000000..788d114d --- /dev/null +++ b/docs/lab3/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-10-26 19:29:59 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C08C-1F90-6ADFAA37167-088A-0008 + 2 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C09C-1F90-6ADFAA37189-088B-0008 + 3 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A0-1F90-6ADFAA371A4-088C-0008 + 4 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0A4-1F90-6ADFAA371B1-088D-0008 + 5 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A6-1F90-6ADFAA371B5-088E-0008 + 6 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0AA-1F90-6ADFAA371B8-088F-0008 + 7 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0B2-1F90-6ADFAA371D0-0890-0008 + 8 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0BE-1F90-6ADFAA371E0-0891-0008 + 9 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0C6-1F90-6ADFAA371E3-0892-0008 +10 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0CE-1F90-6ADFAA371FE-0893-0008 +11 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0D6-1F90-6ADFAA37202-0894-0008 +12 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0E4-1F90-6ADFAA3721D-0895-0008 +13 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0EC-1F90-6ADFAA37220-0896-0008 +14 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0FC-1F90-6ADFAA3723C-0897-0008 +15 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C106-1F90-6ADFAA3724B-0898-0008 +16 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C10E-1F90-6ADFAA3724D-0899-0008 +17 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C11A-1F90-6ADFAA37268-089A-0008 +18 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C126-1F90-6ADFAA3726B-089C-0008 +19 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C130-1F90-6ADFAA37286-089D-0008 +20 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C140-1F90-6ADFAA3728A-089E-0008 +разных реплик: 2 diff --git a/docs/lab3/evidence/lb_errors.txt b/docs/lab3/evidence/lb_errors.txt new file mode 100644 index 00000000..4a1f8683 --- /dev/null +++ b/docs/lab3/evidence/lb_errors.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker) +# все реплики остановлены -> ответ формирует сам HAProxy (http-error): общий JSON-формат ошибок, X-Request-Id совпадает с request_id; X-Instance-Id нет (контракт это допускает) +$ curl -si http://127.0.0.1:8080/api/v1/users/me +HTTP/1.1 503 Service Unavailable +x-request-id: 7F000001CD38-1F90-6ADCA7141E5-336F-0B75 +content-length: 128 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"7F000001CD38-1F90-6ADCA7141E5-336F-0B75"}} + +$ curl -si -H "X-Request-Id: demo-123" http://127.0.0.1:8080/health/ready +HTTP/1.1 503 Service Unavailable +x-request-id: demo-123 +content-length: 97 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"demo-123"}} diff --git a/docs/lab3/evidence/lb_log_sample.txt b/docs/lab3/evidence/lb_log_sample.txt new file mode 100644 index 00000000..a05fe969 --- /dev/null +++ b/docs/lab3/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-10-26 19:35:57 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.2s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:34156 [26/Oct/2026:19:35:54.564] public api/api1 0/0/0/7/7 201 401 ---- 8/8/7/1/0 0/0 rid=AC120001856C-1F90-6ADFAB9A234-1128-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34116 [26/Oct/2026:19:35:54.563] public api/api1 0/0/0/10/10 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018544-1F90-6ADFAB9A233-1126-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34168 [26/Oct/2026:19:35:54.578] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018578-1F90-6ADFAB9A242-1129-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34064 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/58/59 201 401 ---- 8/8/7/7/0 0/0 rid=AC1200018510-1F90-6ADFAB9A218-111F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34040 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/62/62 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000184F8-1F90-6ADFAB9A218-111D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34098 [26/Oct/2026:19:35:54.537] public api/api2 0/0/0/61/62 201 401 ---- 6/6/5/5/0 0/0 rid=AC1200018532-1F90-6ADFAB9A219-1121-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34100 [26/Oct/2026:19:35:54.538] public api/api2 0/0/0/63/63 201 401 ---- 6/6/5/4/0 0/0 rid=AC1200018534-1F90-6ADFAB9A21A-1123-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34162 [26/Oct/2026:19:35:54.578] public api/api2 0/0/0/27/28 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018572-1F90-6ADFAB9A242-112A-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34186 [26/Oct/2026:19:35:54.600] public api/api1 0/0/0/7/7 201 401 ---- 6/6/3/0/0 0/0 rid=AC120001858A-1F90-6ADFAB9A258-112C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34128 [26/Oct/2026:19:35:54.563] public api/api2 0/0/0/44/44 201 401 ---- 5/5/2/2/0 0/0 rid=AC1200018550-1F90-6ADFAB9A233-1125-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34182 [26/Oct/2026:19:35:54.587] public api/api2 0/0/0/21/21 201 401 ---- 4/4/1/1/0 0/0 rid=AC1200018586-1F90-6ADFAB9A24B-112B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34142 [26/Oct/2026:19:35:54.564] public api/api2 0/0/0/45/45 201 401 ---- 3/3/0/0/0 0/0 rid=AC120001855E-1F90-6ADFAB9A234-1127-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34198 [26/Oct/2026:19:35:54.614] public api/api1 0/0/0/14/14 201 401 ---- 8/8/6/2/0 0/0 rid=AC1200018596-1F90-6ADFAB9A25F-112E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34230 [26/Oct/2026:19:35:54.621] public api/api1 0/0/0/10/11 201 401 ---- 7/7/5/1/0 0/0 rid=AC12000185B6-1F90-6ADFAB9A26A-1130-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34242 [26/Oct/2026:19:35:54.622] public api/api1 0/0/0/9/9 201 401 ---- 6/6/4/0/0 0/0 rid=AC12000185C2-1F90-6ADFAB9A26E-1132-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34270 [26/Oct/2026:19:35:54.638] public api/api1 0/0/0/6/7 201 401 ---- 8/8/6/1/0 0/0 rid=AC12000185DE-1F90-6ADFAB9A27D-1134-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34266 [26/Oct/2026:19:35:54.640] public api/api1 0/0/0/6/6 201 401 ---- 7/7/5/0/0 0/0 rid=AC12000185DA-1F90-6ADFAB9A26E-1136-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34278 [26/Oct/2026:19:35:54.656] public api/api1 0/0/0/5/5 201 401 ---- 8/8/6/0/0 0/0 rid=AC12000185E6-1F90-6ADFAB9A27F-1138-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34302 [26/Oct/2026:19:35:54.686] public api/api1 0/0/0/6/6 201 401 ---- 8/8/7/0/0 0/0 rid=AC12000185FE-1F90-6ADFAB9A299-113A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34282 [26/Oct/2026:19:35:54.639] public api/api2 0/0/0/66/66 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185EA-1F90-6ADFAB9A27F-1135-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34254 [26/Oct/2026:19:35:54.622] public api/api2 0/0/0/83/84 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000185CE-1F90-6ADFAB9A26E-1133-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34294 [26/Oct/2026:19:35:54.671] public api/api2 0/0/0/36/36 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000185F6-1F90-6ADFAB9A28C-1139-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34338 [26/Oct/2026:19:35:54.715] public api/api1 0/0/0/5/5 201 401 ---- 8/8/5/0/0 0/0 rid=AC1200018622-1F90-6ADFAB9A2CB-113C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34346 [26/Oct/2026:19:35:54.734] public api/api1 0/0/1/6/7 201 401 ---- 8/8/7/0/0 0/0 rid=AC120001862A-1F90-6ADFAB9A2CC-113E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34360 [26/Oct/2026:19:35:54.745] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018638-1F90-6ADFAB9A2E9-1140-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34290 [26/Oct/2026:19:35:54.652] public api/api2 0/0/0/56/123 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185F2-1F90-6ADFAB9A28C-1137-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34192 [26/Oct/2026:19:35:54.609] public api/api2 0/0/0/171/172 201 401 ---- 8/8/7/6/0 0/0 rid=AC1200018590-1F90-6ADFAB9A25E-112D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34240 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/159/160 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000185C0-1F90-6ADFAB9A26C-1131-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34214 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/160/161 201 401 ---- 6/6/5/4/0 0/0 rid=AC12000185A6-1F90-6ADFAB9A269-112F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34312 [26/Oct/2026:19:35:54.698] public api/api2 0/0/0/87/87 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018608-1F90-6ADFAB9A2BA-113B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34374 [26/Oct/2026:19:35:54.779] public api/api1 0/0/0/8/8 201 401 ---- 4/4/3/0/0 0/0 rid=AC1200018646-1F90-6ADFAB9A30B-1142-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34364 [26/Oct/2026:19:35:54.755] public api/api2 0/0/0/35/35 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001863C-1F90-6ADFAB9A2F3-1141-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34326 [26/Oct/2026:19:35:54.734] public api/api2 0/0/0/59/59 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018616-1F90-6ADFAB9A2CB-113F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34354 [26/Oct/2026:19:35:54.725] public api/api2 0/0/0/68/68 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018632-1F90-6ADFAB9A2D5-113D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34384 [26/Oct/2026:19:35:54.801] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC1200018650-1F90-6ADFAB9A31B-1144-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34426 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/15/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC120001867A-1F90-6ADFAB9A322-1148-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34392 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018658-1F90-6ADFAB9A31D-1146-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34440 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 5/5/4/0/0 0/0 rid=AC1200018688-1F90-6ADFAB9A322-114A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34390 [26/Oct/2026:19:35:54.799] public api/api2 0/0/1/18/20 201 401 ---- 4/4/3/3/0 0/0 rid=AC1200018656-1F90-6ADFAB9A31C-1143-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34382 [26/Oct/2026:19:35:54.801] public api/api2 0/0/0/18/18 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001864E-1F90-6ADFAB9A31B-1145-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34408 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/17/17 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018668-1F90-6ADFAB9A322-1147-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34422 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/18/18 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018676-1F90-6ADFAB9A322-1149-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34462 [26/Oct/2026:19:35:54.837] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC120001869E-1F90-6ADFAB9A344-114E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34508 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC12000186CC-1F90-6ADFAB9A346-1152-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34450 [26/Oct/2026:19:35:54.834] public api/api1 0/0/0/20/20 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018692-1F90-6ADFAB9A33F-114C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34490 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/17 201 401 ---- 5/5/4/0/0 0/0 rid=AC12000186BA-1F90-6ADFAB9A346-1150-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34534 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/10/10 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186E6-1F90-6ADFAB9A361-1156-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34530 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/12/12 201 401 ---- 7/7/6/0/0 0/0 rid=AC12000186E2-1F90-6ADFAB9A361-1155-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34446 [26/Oct/2026:19:35:54.834] public api/api2 0/0/0/59/59 201 401 ---- 8/8/6/6/0 0/0 rid=AC120001868E-1F90-6ADFAB9A337-114B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34458 [26/Oct/2026:19:35:54.836] public api/api2 0/0/0/62/62 201 401 ---- 8/8/6/5/0 0/0 rid=AC120001869A-1F90-6ADFAB9A343-114D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34538 [26/Oct/2026:19:35:54.897] public api/api1 0/0/1/8/9 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186EA-1F90-6ADFAB9A372-1158-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34478 [26/Oct/2026:19:35:54.837] public api/api2 0/0/0/71/71 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000186AE-1F90-6ADFAB9A345-114F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34570 [26/Oct/2026:19:35:54.904] public api/api1 0/0/0/9/9 201 401 ---- 8/8/5/0/0 0/0 rid=AC120001870A-1F90-6ADFAB9A388-115A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34594 [26/Oct/2026:19:35:54.934] public api/api1 0/0/1/5/6 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018722-1F90-6ADFAB9A391-115D-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34498 [26/Oct/2026:19:35:54.838] public api/api2 0/0/0/71/136 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000186C2-1F90-6ADFAB9A346-1151-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34522 [26/Oct/2026:19:35:54.856] public api/api2 0/0/0/121/121 201 401 ---- 5/5/4/4/0 0/0 rid=AC12000186DA-1F90-6ADFAB9A358-1153-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34536 [26/Oct/2026:19:35:54.882] public api/api2 0/0/0/96/96 201 401 ---- 4/4/3/3/0 0/0 rid=AC12000186E8-1F90-6ADFAB9A372-1157-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34528 [26/Oct/2026:19:35:54.865] public api/api2 0/0/0/114/115 201 401 ---- 3/3/2/2/0 0/0 rid=AC12000186E0-1F90-6ADFAB9A361-1154-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34556 [26/Oct/2026:19:35:54.899] public api/api2 0/0/0/86/86 201 401 ---- 2/2/1/1/0 0/0 rid=AC12000186FC-1F90-6ADFAB9A383-1159-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34586 [26/Oct/2026:19:35:54.934] public api/api2 0/0/1/51/53 201 401 ---- 1/1/0/0/0 0/0 rid=AC120001871A-1F90-6ADFAB9A391-115C-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" diff --git a/docs/lab3/evidence/load.json b/docs/lab3/evidence/load.json new file mode 100644 index 00000000..87063baf --- /dev/null +++ b/docs/lab3/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab3/evidence/load.txt b/docs/lab3/evidence/load.txt new file mode 100644 index 00000000..c7c17fbe --- /dev/null +++ b/docs/lab3/evidence/load.txt @@ -0,0 +1,28 @@ +# 2026-10-26 19:35:55 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.4s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-10-26 19:35:57 UTC | CPU/RAM после нагрузки | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-api-6 19.92% 89.26MiB / 256MiB 34.87% +messenger-lab3-api-5 0.67% 89.33MiB / 256MiB 34.89% +messenger-lab3-lb-1 0.14% 14.34MiB / 128MiB 11.20% +messenger-lab3-postgres-1 11.42% 68.96MiB / 512MiB 13.47% +messenger-lab3-redis-1 2.36% 4.148MiB / 96MiB 4.32% + diff --git a/docs/lab3/evidence/migrations.txt b/docs/lab3/evidence/migrations.txt new file mode 100644 index 00000000..b07edae5 --- /dev/null +++ b/docs/lab3/evidence/migrations.txt @@ -0,0 +1,26 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +## 1. Реплика не готова, пока схема не мигрирована; migrate однократный, повтор — no-op; две копии одновременно — advisory lock +$ # реплика против ПУСТОЙ БД, миграции ещё не выполнялись (RUN_MIGRATIONS_ON_START=false) +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready +{"error":{"code":"unavailable","message":"Database schema is not migrated","request_id":"1f3109e3-a934-4c1b-bfbd-22350e35e05d"}} 503 +$ psql -c "select to_regclass('schema_migrations')" +NULL +$ python -m src.migrate # = сервис migrate (one-shot) +applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ python -m src.migrate # повтор: no-op +schema is up to date; nothing applied +$ # две копии migrate одновременно (advisory lock) на ещё одной пустой БД +NOTICE: database "messenger_race" does not exist, skipping +copy A: schema is up to date; nothing applied +copy B: applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +schema_migrations rows: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready # та же реплика, без рестарта +{"status":"ok"} 200 + +## 2. Совместимость миграции 003 со старой версией (rolling update): код lab2 работает на схеме lab3 одновременно с репликами lab3 +$ # старая версия: код lab2 (без Idempotency-Key/X-Instance-Id) против БД, мигрированной до 003 +migrations applied at startup: none +$ cd lab2 && python tests/smoke.py --base-url http://127.0.0.1:18098 # контракт lab2 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +$ cd lab3 && python tests/smoke.py --base-url http://127.0.0.1:8080 # одновременно реплики lab3 за LB +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/persistence.txt b/docs/lab3/evidence/persistence.txt new file mode 100644 index 00000000..61135587 --- /dev/null +++ b/docs/lab3/evidence/persistence.txt @@ -0,0 +1,24 @@ +# 2026-10-26 19:35:10 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 31 seconds (healthy) 32 seconds ago +messenger-lab3-api-6 Up 21 seconds (healthy) 22 seconds ago +messenger-lab3-lb-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 4 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 18 minutes (healthy) 44 minutes ago +# 2026-10-26 19:35:24 UTC | Persistence: make test-persistence | exit 0 | 14.1s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:35:24 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 7 seconds (healthy) 46 seconds ago +messenger-lab3-api-6 Up 7 seconds (healthy) 36 seconds ago +messenger-lab3-lb-1 Up Less than a second (health: starting) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 5 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 7 seconds (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 7 seconds (healthy) 44 minutes ago + diff --git a/docs/lab3/evidence/rolling_back.json b/docs/lab3/evidence/rolling_back.json new file mode 100644 index 00000000..e79934d3 --- /dev/null +++ b/docs/lab3/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 27.69, + "replicas_before": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "replicas_after": [ + "c38982836a1d", + "ecaa2acc2a0a" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1342, + "ok": 1342, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 267.5, + "write_ok_rps": 159.5, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 94.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + }, + "during_update": { + "from_s": 5.02, + "to_s": 32.7, + "requests": 6993, + "ok": 6993, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 252.6, + "write_ok_rps": 149.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 140.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3", + "c38982836a1d", + "ecaa2acc2a0a" + ] + }, + "after": { + "from_s": 32.7, + "to_s": 40.77, + "requests": 2045, + "ok": 2045, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 253.6, + "write_ok_rps": 152.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 94.0 + }, + "instances": [ + "c38982836a1d", + "ecaa2acc2a0a" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api1", + "api3" + ] + ], + [ + 11.42, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.23, + [ + "api1", + "api3" + ] + ], + [ + 14.77, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.16, + [ + "api1", + "api2" + ] + ], + [ + 25.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.66, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6166, + "stored_messages": 6166, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.json b/docs/lab3/evidence/rolling_update.json new file mode 100644 index 00000000..e4d09f8e --- /dev/null +++ b/docs/lab3/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.38, + "replicas_before": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "replicas_after": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1161, + "ok": 1161, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 232.2, + "write_ok_rps": 136.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.38, + "requests": 6950, + "ok": 6950, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.9, + "write_ok_rps": 145.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 156.0 + }, + "instances": [ + "6e2f0c643449", + "a21409fffeec", + "b6e350f043f3", + "c4c5ae5b0949" + ] + }, + "after": { + "from_s": 33.38, + "to_s": 41.41, + "requests": 2008, + "ok": 2008, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 250.0, + "write_ok_rps": 147.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 12.5, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.73, + [ + "api1", + "api2" + ] + ], + [ + 15.25, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.28, + [ + "api2", + "api3" + ] + ], + [ + 25.73, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.34, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5981, + "stored_messages": 5981, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.txt b/docs/lab3/evidence/rolling_update.txt new file mode 100644 index 00000000..1fef4643 --- /dev/null +++ b/docs/lab3/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-10-26 19:34:26 UTC | Rolling update lab3 -> lab3.1 под нагрузкой | exit 0 | 44.1s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['a21409fffeec', 'c4c5ae5b0949'] +2026-10-26T19:33:48.176Z rolling-update: start; replicas=2 old=['c4c5ae5b0949', 'a21409fffeec'] +2026-10-26T19:33:51.770Z rolling-update: images built in 3.6s (tag lab3.1) +2026-10-26T19:33:53.990Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-10-26T19:34:04.788Z rolling-update: step 1/2 new=6e2f0c643449 (messenger-api:lab3.1, ready+UP in 6.8s) replaced old=c4c5ae5b0949 (graceful stop 3.8s) +2026-10-26T19:34:15.884Z rolling-update: step 2/2 new=b6e350f043f3 (messenger-api:lab3.1, ready+UP in 6.7s) replaced old=a21409fffeec (graceful stop 4.3s) +2026-10-26T19:34:16.330Z rolling-update: done in 28.1s; replicas now ['6e2f0c643449', 'b6e350f043f3'] +update 28.4s rc=0 | replicas ['a21409fffeec', 'c4c5ae5b0949'] -> ['6e2f0c643449', 'b6e350f043f3'] | errors during update 0 (-) | RPS before 232.2, during 244.9, after 250.0 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-10-26 19:35:09 UTC | Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой | exit 0 | 43.4s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:32.642Z rolling-update: start; replicas=2 old=['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:35.497Z rolling-update: images built in 2.9s (tag lab3) +2026-10-26T19:34:37.818Z rolling-update: migrations once in 2.3s: schema is up to date; nothing applied +2026-10-26T19:34:48.605Z rolling-update: step 1/2 new=c38982836a1d (messenger-api:lab3, ready+UP in 7.0s) replaced old=6e2f0c643449 (graceful stop 3.7s) +2026-10-26T19:34:59.414Z rolling-update: step 2/2 new=ecaa2acc2a0a (messenger-api:lab3, ready+UP in 6.8s) replaced old=b6e350f043f3 (graceful stop 3.9s) +2026-10-26T19:34:59.817Z rolling-update: done in 27.4s; replicas now ['c38982836a1d', 'ecaa2acc2a0a'] +update 27.7s rc=0 | replicas ['6e2f0c643449', 'b6e350f043f3'] -> ['c38982836a1d', 'ecaa2acc2a0a'] | errors during update 0 (-) | RPS before 267.5, during 252.6, after 253.6 | p95 during 78.0 ms | data ok=True +PASS: rolling update under load + +# 2026-10-26 19:35:09 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c38982836a1d messenger-lab3-api-5 running healthy messenger-api:lab3 api2 UP +ecaa2acc2a0a messenger-lab3-api-6 running healthy messenger-api:lab3 api3 UP + diff --git a/docs/lab3/evidence/shutdown_timeline.txt b/docs/lab3/evidence/shutdown_timeline.txt new file mode 100644 index 00000000..7c3bddb7 --- /dev/null +++ b/docs/lab3/evidence/shutdown_timeline.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +# Настройки: HAProxy inter 1s / fastinter 500ms / fall 2 / rise 2 / timeout check 1.5s; API DRAIN_SECONDS=3, GRACEFUL_TIMEOUT_SECONDS=10 +# graceful stop timeline, t=0 — начало наблюдения; интервал опроса 50 мс + +0.02s (от SIGTERM -0.48s) LB api3: UP + +0.02s (от SIGTERM -0.48s) nat-api-3 /health/ready -> 200 + +0.50s (от SIGTERM -0.00s) SIGTERM -> nat-api-3 (pid 3196) + +0.51s (от SIGTERM +0.01s) nat-api-3 /health/ready -> 503 + +1.26s (от SIGTERM +0.76s) LB api3: UP 1/2 + +1.80s (от SIGTERM +1.30s) LB api3: DOWN + +3.52s (от SIGTERM +3.02s) nat-api-3 /health/ready -> connection refused + +3.62s (от SIGTERM +3.12s) nat-api-3 process exited + +# лог реплики: +2026-10-24 15:39:19,576 INFO messenger.server signal 15: instance nat-api-3 draining for 3.0s (readiness=503), then graceful stop +2026-10-24 15:39:22,580 INFO uvicorn.error Shutting down +2026-10-24 15:39:22,681 INFO messenger.api shutdown: instance=nat-api-3 closing pool +2026-10-24 15:39:22,682 INFO uvicorn.error Finished server process [3196] diff --git a/docs/lab3/evidence/smoke.txt b/docs/lab3/evidence/smoke.txt new file mode 100644 index 00000000..574b73d4 --- /dev/null +++ b/docs/lab3/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-10-26 19:30:00 UTC | Публичный smoke | exit 0 | 0.9s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/stand.txt b/docs/lab3/evidence/stand.txt new file mode 100644 index 00000000..20ae6e53 --- /dev/null +++ b/docs/lab3/evidence/stand.txt @@ -0,0 +1,33 @@ +# 2026-10-26 19:29:57 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab3-api-1 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 12 minutes (healthy) 8080/tcp +messenger-lab3-api-2 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 13 minutes (healthy) 8080/tcp +messenger-lab3-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 39 minutes ago Up 13 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab3-migrate-1 messenger-lab3-migrate "python -m src.migra…" migrate 39 minutes ago Exited (0) 13 minutes ago +messenger-lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 39 minutes ago Up 13 minutes (healthy) 5432/tcp +messenger-lab3-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 39 minutes ago Up 13 minutes (healthy) 6379/tcp +# 2026-10-26 19:29:57 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:29:59 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-lb-1 1.87% 12.87MiB / 128MiB 10.06% +messenger-lab3-api-2 16.79% 204.4MiB / 256MiB 79.86% +messenger-lab3-api-1 0.31% 222.9MiB / 256MiB 87.07% +messenger-lab3-postgres-1 2.23% 58.11MiB / 512MiB 11.35% +messenger-lab3-redis-1 0.22% 4.559MiB / 96MiB 4.75% + +# 2026-10-26 19:29:59 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.3s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp + diff --git a/docs/lab3/evidence/student-tests.txt b/docs/lab3/evidence/student-tests.txt new file mode 100644 index 00000000..2489d48d --- /dev/null +++ b/docs/lab3/evidence/student-tests.txt @@ -0,0 +1,134 @@ +# 2026-10-26 19:30:06 UTC | Собственные тесты (HTTP через LB) | exit 0 | 6.2s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests) ... ok +test_password_never_returned (test_api.ApiBehaviorTests) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests) ... ok +test_login_with_body_rejected (test_auth.AuthTests) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests) ... ok +test_malformed_basic_headers (test_auth.AuthTests) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 47 tests in 4.640s + +OK (skipped=13) + + 24 parallel POSTs, one key -> 1 message; replicas: ['a21409fffeec', 'c4c5ae5b0949'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-10-26 19:30:14 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.5s +$ python scripts/ops.py in-network storage + Container messenger-lab3-api-run-bdc116a8fd34 Creating + Container messenger-lab3-api-run-bdc116a8fd34 Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.223s + +OK + Container messenger-lab3-api-run-630b208e9588 Creating + Container messenger-lab3-api-run-630b208e9588 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.864s + +OK + +# 2026-10-26 19:30:27 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 13.7s +$ python scripts/ops.py in-network expiry + Container messenger-lab3-redis-1 Running + Container messenger-lab3-postgres-1 Running + Container messenger-lab3-api-1 Running + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-short-ttl-1 Creating + Container messenger-lab3-api-short-ttl-1 Created + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Starting + Container messenger-lab3-migrate-1 Started + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-redis-1 Waiting + Container messenger-lab3-redis-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-short-ttl-1 Starting + Container messenger-lab3-api-short-ttl-1 Started + Container messenger-lab3-api-run-a36baf72714e Creating + Container messenger-lab3-api-run-a36baf72714e Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab3-api-run-c445a7ef00cd Creating + Container messenger-lab3-api-run-c445a7ef00cd Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.620s + +OK + diff --git a/evidence/README.md b/evidence/README.md index 8df70015..59eb2597 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,9 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Что лежит здесь (lab4) + +- `sandbox/` — результаты, полученные при разработке на стенде из тех же компонентов, запущенных нативными процессами (PostgreSQL 16, Redis 7, SeaweedFS 4.50, HAProxy 2.8.30, 2 реплики API; без Docker). Как получены — REPORT.md, раздел «Доказательства критериев». +- Остальные файлы создаёт `python scripts/collect_evidence.py` на compose-стенде (сводка — `SUMMARY.txt`). Токены и пароли в файлы не пишутся. +- Доказательства lab3 сохранены в `docs/lab3/evidence/`. diff --git a/evidence/SUMMARY.txt b/evidence/SUMMARY.txt new file mode 100644 index 00000000..c28d9f48 --- /dev/null +++ b/evidence/SUMMARY.txt @@ -0,0 +1,46 @@ +# 2026-11-04 14:12:25 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Образ SeaweedFS (закреплённая версия) evidence/stand.txt +OK S3 bootstrap: лог первого запуска s3-init evidence/s3_bootstrap.txt +OK S3 bootstrap: повторный запуск (идемпотентность) evidence/s3_bootstrap.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB, включая test_attachments.py) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) evidence/consistency.txt +OK Сверка БД <-> S3 (dry-run) evidence/reconcile.txt +OK Приватный бакет вне API, права ключей, закрытые админ-порты evidence/s3_private.txt +OK На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes evidence/s3_private.txt +OK API: файлы в /tmp (tmpfs) после загрузок evidence/s3_private.txt +OK Пик памяти: 10 одновременных загрузок по 9.5 MiB evidence/upload_memory.txt +OK Пик памяти: 20 одновременных загрузок evidence/upload_memory.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Недоступность S3 и восстановление evidence/s3_outage.txt +OK Rolling update lab4 -> lab4.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Persistence: сверка БД <-> S3 после рестарта evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..6abe7718 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-11-04 14:03:55 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-04 14:03:55 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 00000000..fd9c005d --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,75 @@ +# 2026-11-04 14:04:49 UTC | Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) | exit 0 | 15.8s +$ python scripts/ops.py in-network consistency + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-fault-s3-1 Creating + Container messenger-lab4-api-fault-db-1 Creating + Container messenger-lab4-api-fault-crash-1 Creating + Container messenger-lab4-api-fault-db-1 Created + Container messenger-lab4-api-fault-s3-1 Created + Container messenger-lab4-api-fault-crash-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-crash-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-s3-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-db-1 Starting + Container messenger-lab4-api-fault-crash-1 Started + Container messenger-lab4-api-fault-s3-1 Started + Container messenger-lab4-api-fault-db-1 Started + Container messenger-lab4-api-run-46322fbc844a Creating + Container messenger-lab4-api-run-46322fbc844a Created +ready: http://api-fault-s3:8080/health/ready + Container messenger-lab4-api-run-ff8a88e550db Creating + Container messenger-lab4-api-run-ff8a88e550db Created +ready: http://api-fault-db:8080/health/ready + Container messenger-lab4-api-run-476fc515a8f0 Creating + Container messenger-lab4-api-run-476fc515a8f0 Created +ready: http://api-fault-crash:8080/health/ready + Container messenger-lab4-api-run-9b2dce8846fe Creating + Container messenger-lab4-api-run-9b2dce8846fe Created +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.224s + +OK diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..65ce6a95 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-11-04 14:05:30 UTC | Failover #1: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP +# 2026-11-04 14:05:46 UTC | Failover #1: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.187} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:05:46 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:05:52 UTC | Failover #1: restore | exit 0 | 6.3s +$ python scripts/ops.py restore +2026-11-04T14:05:52.981Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.2s + +# 2026-11-04 14:05:53 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:09 UTC | Failover #2: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:09 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:06:16 UTC | Failover #2: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-04T14:06:16.514Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-04 14:06:16 UTC | Failover #3: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:33 UTC | Failover #3: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:33 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 exited unhealthy messenger-api:lab4 - - +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:40 UTC | Failover #3: restore | exit 0 | 6.7s +$ python scripts/ops.py restore +2026-11-04T14:06:40.028Z restore: started ['messenger-lab4-api-1']; 2 replicas healthy and UP in LB after 6.6s + +# остановленные публичным тестом реплики: ['4fc3ad11e268', '4fc3ad11e268', 'c9fab9c9e2c1'] diff --git a/evidence/failover_series.json b/evidence/failover_series.json new file mode 100644 index 00000000..9aad2e20 --- /dev/null +++ b/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.12, + "lb_detect_down_seconds": 1.56, + "target_last_served_after_stop_seconds": 1.56, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.62, + "lb_back_up_seconds": 1.28, + "target_back_in_rotation_seconds": 7.62, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1155, + "ok": 1155, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 231.0, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1010, + "ok": 1010, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 101.0, + "write_ok_rps": 61.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 588, + "ok": 588, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 117.6, + "write_ok_rps": 43.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 219.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.92, + "requests": 1531, + "ok": 1531, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 140.2, + "write_ok_rps": 82.4, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.56, + [ + "api1" + ] + ], + [ + 21.28, + [ + "api1", + "api2" + ] + ], + [ + 21.61, + [ + "api1" + ] + ], + [ + 24.17, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2426, + "stored_messages": 2426, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.83, + "lb_detect_down_seconds": 1.55, + "target_last_served_after_stop_seconds": 1.39, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.0, + "lb_back_up_seconds": 1.62, + "target_back_in_rotation_seconds": 6.99, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1015, + "ok": 1015, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 203.0, + "write_ok_rps": 121.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 833, + "ok": 833, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 83.3, + "write_ok_rps": 49.3, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 371, + "ok": 371, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 74.2, + "write_ok_rps": 44.2, + "latency_ms_ok": { + "p50": 93.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.27, + "requests": 1496, + "ok": 1496, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 145.7, + "write_ok_rps": 86.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.55, + [ + "api2" + ] + ], + [ + 21.62, + [ + "api1", + "api2" + ] + ], + [ + 22.58, + [ + "api2" + ] + ], + [ + 25.03, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2215, + "stored_messages": 2215, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/evidence/failover_series.txt b/evidence/failover_series.txt new file mode 100644 index 00000000..1a84b38d --- /dev/null +++ b/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-11-04 14:07:43 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 63.4s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=stop) under load + hook 4.12s | LB DOWN after 1.56 s | errors 0 (-) | RPS before 231.0, outage 101.0, after restore 140.2 | p95 outage 110.0 ms | back in rotation 7.62 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=stop) under load + hook 4.83s | LB DOWN after 1.55 s | errors 0 (-) | RPS before 203.0, outage 83.3, after restore 145.7 | p95 outage 187.0 ms | back in rotation 6.99 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-11-04 14:08:46 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 62.7s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=kill) under load + hook 1.09s | LB DOWN after 1.97 s | errors 6 (502) | RPS before 244.2, outage 77.1, after restore 119.5 | p95 outage 187.0 ms | back in rotation 7.52 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=kill) under load + hook 1.14s | LB DOWN after 2.59 s | errors 6 (502) | RPS before 186.8, outage 82.9, after restore 134.8 | p95 outage 188.0 ms | back in rotation 6.83 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/evidence/failover_series_kill.json b/evidence/failover_series_kill.json new file mode 100644 index 00000000..96f1688f --- /dev/null +++ b/evidence/failover_series_kill.json @@ -0,0 +1,298 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.09, + "lb_detect_down_seconds": 1.97, + "target_last_served_after_stop_seconds": 0.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.56, + "restore_hook_seconds": 7.53, + "lb_back_up_seconds": 2.0, + "target_back_in_rotation_seconds": 7.52, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1221, + "ok": 1221, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.2, + "write_ok_rps": 145.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 777, + "ok": 771, + "errors": 6, + "error_ratio": 0.0077, + "ok_rps": 77.1, + "write_ok_rps": 40.7, + "latency_ms_ok": { + "p50": 63.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 565, + "ok": 565, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 113.0, + "write_ok_rps": 44.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 281.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.84, + "requests": 1296, + "ok": 1296, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 119.5, + "write_ok_rps": 68.1, + "latency_ms_ok": { + "p50": 31.0, + "p95": 172.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.97, + [ + "api1" + ] + ], + [ + 22.0, + [ + "api1", + "api2" + ] + ], + [ + 22.45, + [ + "api1" + ] + ], + [ + 24.97, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2100, + "stored_messages": 2100, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.14, + "lb_detect_down_seconds": 2.59, + "target_last_served_after_stop_seconds": 0.51, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.59, + "restore_hook_seconds": 6.83, + "lb_back_up_seconds": 1.61, + "target_back_in_rotation_seconds": 6.83, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 934, + "ok": 934, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 186.8, + "write_ok_rps": 111.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 835, + "ok": 829, + "errors": 6, + "error_ratio": 0.0072, + "ok_rps": 82.9, + "write_ok_rps": 40.8, + "latency_ms_ok": { + "p50": 32.0, + "p95": 188.0, + "p99": 1000.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 541, + "ok": 541, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 108.2, + "write_ok_rps": 37.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 203.0, + "p99": 282.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.16, + "requests": 1369, + "ok": 1369, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 134.8, + "write_ok_rps": 77.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.59, + [ + "api2" + ] + ], + [ + 21.61, + [ + "api1", + "api2" + ] + ], + [ + 21.95, + [ + "api2" + ] + ], + [ + 24.53, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 1941, + "stored_messages": 1941, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + } + ], + "pass": true +} diff --git a/evidence/instances.txt b/evidence/instances.txt new file mode 100644 index 00000000..df79b9b1 --- /dev/null +++ b/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-11-04 14:04:00 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001993A-1F90-6AEB3B50162-0571-0008 + 2 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019944-1F90-6AEB3B50169-0572-0008 + 3 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001994A-1F90-6AEB3B5017C-0573-0008 + 4 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019958-1F90-6AEB3B50199-0574-0008 + 5 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001995E-1F90-6AEB3B5019C-0575-0008 + 6 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019960-1F90-6AEB3B501B7-0576-0008 + 7 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019966-1F90-6AEB3B501C6-0577-0008 + 8 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019974-1F90-6AEB3B501C9-0578-0008 + 9 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019978-1F90-6AEB3B501E5-0579-0008 +10 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019980-1F90-6AEB3B501F4-057A-0008 +11 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019984-1F90-6AEB3B50205-057B-0008 +12 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019988-1F90-6AEB3B50209-057C-0008 +13 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019998-1F90-6AEB3B5020C-057D-0008 +14 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC120001999C-1F90-6AEB3B50223-057E-0008 +15 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001999E-1F90-6AEB3B50226-057F-0008 +16 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199AE-1F90-6AEB3B50241-0580-0008 +17 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199BA-1F90-6AEB3B50245-0581-0008 +18 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199CA-1F90-6AEB3B50260-0582-0008 +19 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199D2-1F90-6AEB3B50263-0584-0008 +20 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199DE-1F90-6AEB3B50266-0585-0008 +разных реплик: 2 diff --git a/evidence/lb_log_sample.txt b/evidence/lb_log_sample.txt new file mode 100644 index 00000000..234f9e0c --- /dev/null +++ b/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-11-04 14:12:25 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:56696 [04/Nov/2026:14:12:22.325] public api/api2 0/0/0/11/11 201 421 ---- 8/8/7/2/0 0/0 rid=AC120001DD78-1F90-6AEB3D46145-0F07-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56712 [04/Nov/2026:14:12:22.328] public api/api2 0/0/0/10/11 201 421 ---- 8/8/6/1/0 0/0 rid=AC120001DD88-1F90-6AEB3D46148-0F09-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56732 [04/Nov/2026:14:12:22.330] public api/api2 0/0/0/9/9 201 421 ---- 7/7/5/0/0 0/0 rid=AC120001DD9C-1F90-6AEB3D4614A-0F0B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56746 [04/Nov/2026:14:12:22.340] public api/api2 0/0/0/6/6 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DDAA-1F90-6AEB3D46154-0F0D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56734 [04/Nov/2026:14:12:22.349] public api/api2 0/0/1/6/7 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DD9E-1F90-6AEB3D46152-0F0F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56764 [04/Nov/2026:14:12:22.365] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDBC-1F90-6AEB3D4615C-0F11-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56666 [04/Nov/2026:14:12:22.308] public api/api1 0/0/0/88/89 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD5A-1F90-6AEB3D46134-0F04-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56788 [04/Nov/2026:14:12:22.398] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDD4-1F90-6AEB3D4618E-0F13-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56688 [04/Nov/2026:14:12:22.324] public api/api1 0/0/0/84/85 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD70-1F90-6AEB3D46144-0F06-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56802 [04/Nov/2026:14:12:22.410] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE2-1F90-6AEB3D4619A-0F15-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56698 [04/Nov/2026:14:12:22.325] public api/api1 0/0/0/99/100 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD7A-1F90-6AEB3D46145-0F08-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56808 [04/Nov/2026:14:12:22.427] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE8-1F90-6AEB3D461AB-0F17-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56748 [04/Nov/2026:14:12:22.341] public api/api1 0/0/0/156/157 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DDAC-1F90-6AEB3D46155-0F0E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56768 [04/Nov/2026:14:12:22.358] public api/api1 0/0/0/140/140 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DDC0-1F90-6AEB3D46166-0F10-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56784 [04/Nov/2026:14:12:22.395] public api/api1 0/0/0/103/104 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DDD0-1F90-6AEB3D46174-0F12-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56678 [04/Nov/2026:14:12:22.334] public api/api1 0/0/1/164/166 201 421 ---- 8/8/4/4/0 0/0 rid=AC120001DD66-1F90-6AEB3D4613D-0F0C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56728 [04/Nov/2026:14:12:22.328] public api/api1 0/0/0/171/172 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DD98-1F90-6AEB3D46148-0F0A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56800 [04/Nov/2026:14:12:22.406] public api/api1 0/0/0/96/96 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE0-1F90-6AEB3D46196-0F14-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56806 [04/Nov/2026:14:12:22.418] public api/api1 0/0/0/88/88 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE6-1F90-6AEB3D461A2-0F16-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56846 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE0E-1F90-6AEB3D461F5-0F1A-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56856 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/12/12 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE18-1F90-6AEB3D461F5-0F1C-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56872 [04/Nov/2026:14:12:22.502] public api/api1 0/0/0/12/12 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DE28-1F90-6AEB3D461F6-0F1D-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56824 [04/Nov/2026:14:12:22.434] public api/api1 0/0/0/84/84 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DDF8-1F90-6AEB3D461B2-0F18-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56840 [04/Nov/2026:14:12:22.501] public api/api1 0/0/0/17/18 201 421 ---- 7/7/2/1/0 0/0 rid=AC120001DE08-1F90-6AEB3D461F5-0F1B-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56898 [04/Nov/2026:14:12:22.512] public api/api2 0/0/0/6/6 201 421 ---- 6/6/1/0/0 0/0 rid=AC120001DE42-1F90-6AEB3D46200-0F1E-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56860 [04/Nov/2026:14:12:22.518] public api/api2 0/0/1/11/12 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DE1C-1F90-6AEB3D461F6-0F20-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56926 [04/Nov/2026:14:12:22.521] public api/api2 0/0/0/12/12 201 421 ---- 8/8/5/1/0 0/0 rid=AC120001DE5E-1F90-6AEB3D46209-0F24-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56888 [04/Nov/2026:14:12:22.519] public api/api2 0/0/0/14/14 201 421 ---- 7/7/4/0/0 0/0 rid=AC120001DE38-1F90-6AEB3D461FD-0F22-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56874 [04/Nov/2026:14:12:22.519] public api/api1 0/0/0/15/16 201 421 ---- 6/6/5/3/0 0/0 rid=AC120001DE2A-1F90-6AEB3D461F8-0F21-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56916 [04/Nov/2026:14:12:22.520] public api/api1 0/0/0/14/14 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DE54-1F90-6AEB3D46208-0F23-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56904 [04/Nov/2026:14:12:22.516] public api/api1 0/0/0/19/20 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DE48-1F90-6AEB3D46204-0F1F-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56956 [04/Nov/2026:14:12:22.536] public api/api2 0/0/0/12/12 201 421 ---- 8/8/7/3/0 0/0 rid=AC120001DE7C-1F90-6AEB3D46218-0F2B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56942 [04/Nov/2026:14:12:22.535] public api/api2 0/0/0/15/16 201 421 ---- 8/8/6/2/0 0/0 rid=AC120001DE6E-1F90-6AEB3D46217-0F29-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56930 [04/Nov/2026:14:12:22.534] public api/api2 0/0/0/17/17 201 421 ---- 7/7/5/1/0 0/0 rid=AC120001DE62-1F90-6AEB3D46209-0F27-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56984 [04/Nov/2026:14:12:22.538] public api/api2 0/0/0/15/15 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DE98-1F90-6AEB3D4621A-0F2D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57024 [04/Nov/2026:14:12:22.564] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/1/0 0/0 rid=AC120001DEC0-1F90-6AEB3D4622A-0F30-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56998 [04/Nov/2026:14:12:22.565] public api/api2 0/0/0/8/8 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEA6-1F90-6AEB3D46227-0F2F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57044 [04/Nov/2026:14:12:22.596] public api/api2 0/0/0/6/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DED4-1F90-6AEB3D4623E-0F33-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56934 [04/Nov/2026:14:12:22.533] public api/api1 0/0/0/80/80 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DE66-1F90-6AEB3D46215-0F26-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57066 [04/Nov/2026:14:12:22.615] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEEA-1F90-6AEB3D46267-0F35-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56952 [04/Nov/2026:14:12:22.535] public api/api1 0/0/0/88/88 201 421 ---- 8/8/6/6/0 0/0 rid=AC120001DE78-1F90-6AEB3D46217-0F2A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56900 [04/Nov/2026:14:12:22.534] public api/api1 0/0/1/88/90 201 421 ---- 7/7/5/5/0 0/0 rid=AC120001DE44-1F90-6AEB3D46202-0F28-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56970 [04/Nov/2026:14:12:22.537] public api/api1 0/0/0/87/87 201 421 ---- 6/6/4/4/0 0/0 rid=AC120001DE8A-1F90-6AEB3D46219-0F2C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57018 [04/Nov/2026:14:12:22.565] public api/api1 0/0/0/62/63 201 421 ---- 8/8/6/5/0 0/0 rid=AC120001DEBA-1F90-6AEB3D46229-0F31-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57036 [04/Nov/2026:14:12:22.572] public api/api1 0/0/0/55/55 201 421 ---- 7/7/5/4/0 0/0 rid=AC120001DECC-1F90-6AEB3D4623C-0F32-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57014 [04/Nov/2026:14:12:22.553] public api/api1 0/0/0/74/75 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DEB6-1F90-6AEB3D46229-0F2E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57052 [04/Nov/2026:14:12:22.604] public api/api1 0/0/0/23/24 201 421 ---- 5/5/3/2/0 0/0 rid=AC120001DEDC-1F90-6AEB3D4625C-0F34-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57114 [04/Nov/2026:14:12:22.626] public api/api2 0/0/0/7/7 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DF1A-1F90-6AEB3D46272-0F37-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57138 [04/Nov/2026:14:12:22.629] public api/api2 0/0/0/9/9 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DF32-1F90-6AEB3D46275-0F39-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57142 [04/Nov/2026:14:12:22.630] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/0/0 0/0 rid=AC120001DF36-1F90-6AEB3D46276-0F3B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57156 [04/Nov/2026:14:12:22.641] public api/api2 0/0/0/5/5 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF44-1F90-6AEB3D46276-0F3D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57170 [04/Nov/2026:14:12:22.648] public api/api2 0/0/0/4/4 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF52-1F90-6AEB3D46288-0F3F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57168 [04/Nov/2026:14:12:22.657] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DF50-1F90-6AEB3D46282-0F41-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57096 [04/Nov/2026:14:12:22.625] public api/api1 0/0/0/86/86 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DF08-1F90-6AEB3D46271-0F36-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57108 [04/Nov/2026:14:12:22.626] public api/api1 0/0/0/89/89 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DF14-1F90-6AEB3D46272-0F38-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57160 [04/Nov/2026:14:12:22.656] public api/api1 0/0/0/61/61 201 421 ---- 5/5/4/4/0 0/0 rid=AC120001DF48-1F90-6AEB3D4627B-0F42-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57122 [04/Nov/2026:14:12:22.630] public api/api1 0/0/0/91/91 201 421 ---- 4/4/3/3/0 0/0 rid=AC120001DF22-1F90-6AEB3D46276-0F3A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57082 [04/Nov/2026:14:12:22.642] public api/api1 0/0/0/81/81 201 421 ---- 3/3/2/2/0 0/0 rid=AC120001DEFA-1F90-6AEB3D4626F-0F3E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57164 [04/Nov/2026:14:12:22.640] public api/api1 0/0/0/83/83 201 421 ---- 2/2/1/1/0 0/0 rid=AC120001DF4C-1F90-6AEB3D46280-0F3C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57180 [04/Nov/2026:14:12:22.654] public api/api1 0/0/0/69/70 201 421 ---- 1/1/0/0/0 0/0 rid=AC120001DF5C-1F90-6AEB3D4628E-0F40-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..281ec48e --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..ef628563 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:12:22 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.6s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-11-04 14:12:24 UTC | CPU/RAM после нагрузки | exit 0 | 2.1s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-api-6 17.48% 88.79MiB / 320MiB 27.75% +messenger-lab4-api-5 17.67% 89.4MiB / 320MiB 27.94% +messenger-lab4-lb-1 2.18% 14.14MiB / 128MiB 11.05% +messenger-lab4-postgres-1 0.14% 68.93MiB / 512MiB 13.46% +messenger-lab4-redis-1 2.43% 4.82MiB / 96MiB 5.02% +messenger-lab4-s3-1 0.31% 97.91MiB / 512MiB 19.12% + diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..e453f74c --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,48 @@ +# 2026-11-04 14:11:03 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 32 seconds (healthy) 32 seconds ago +messenger-lab4-api-6 Up 20 seconds (healthy) 21 seconds ago +messenger-lab4-lb-1 Up 11 minutes (healthy) 13 minutes ago +messenger-lab4-migrate-1 Exited (0) 6 minutes ago 13 minutes ago +messenger-lab4-postgres-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-redis-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-s3-1 Up About a minute (healthy) 13 minutes ago +messenger-lab4-s3-init-1 Exited (0) 6 minutes ago 13 minutes ago +# 2026-11-04 14:11:49 UTC | Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) | exit 0 | 45.6s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:11:49 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.2s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 7 seconds (healthy) About a minute ago +messenger-lab4-api-6 Up 7 seconds (healthy) About a minute ago +messenger-lab4-lb-1 Up 1 second (health: starting) 14 minutes ago +messenger-lab4-migrate-1 Exited (0) 7 minutes ago 14 minutes ago +messenger-lab4-postgres-1 Up 38 seconds (healthy) 14 minutes ago +messenger-lab4-redis-1 Up 39 seconds (healthy) 14 minutes ago +messenger-lab4-s3-1 Up 28 seconds (healthy) 14 minutes ago +messenger-lab4-s3-init-1 Exited (0) 7 minutes ago 14 minutes ago + +# 2026-11-04 14:11:52 UTC | Persistence: сверка БД <-> S3 после рестарта | exit 0 | 2.4s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-e029350f1930 Creating + Container messenger-lab4-api-run-e029350f1930 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 182, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.062, + "ready_rows": 182, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.073 +} + diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt new file mode 100644 index 00000000..2147d088 --- /dev/null +++ b/evidence/reconcile.txt @@ -0,0 +1,19 @@ +# 2026-11-04 14:04:52 UTC | Сверка БД <-> S3 (dry-run) | exit 0 | 2.2s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-fb929425df7f Creating + Container messenger-lab4-api-run-fb929425df7f Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 107, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.014, + "ready_rows": 107, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.025 +} diff --git a/evidence/rolling_back.json b/evidence/rolling_back.json new file mode 100644 index 00000000..0f4c8bb0 --- /dev/null +++ b/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.28, + "replicas_before": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "replicas_after": [ + "4749f3b4b256", + "7d965c2245c9" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1026, + "ok": 1026, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.2, + "write_ok_rps": 123.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.28, + "requests": 5808, + "ok": 5808, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.4, + "write_ok_rps": 122.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "1c20a55eebeb", + "4749f3b4b256", + "7d965c2245c9", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 33.28, + "to_s": 41.34, + "requests": 1492, + "ok": 1492, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 185.1, + "write_ok_rps": 111.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4749f3b4b256", + "7d965c2245c9" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api3" + ] + ], + [ + 11.78, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.7, + [ + "api1", + "api3" + ] + ], + [ + 15.26, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 18.2, + [ + "api1", + "api2" + ] + ], + [ + 25.8, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.51, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 4970, + "stored_messages": 4970, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/evidence/rolling_update.json b/evidence/rolling_update.json new file mode 100644 index 00000000..7c74c98e --- /dev/null +++ b/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 29.38, + "replicas_before": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "replicas_after": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1180, + "ok": 1180, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 236.0, + "write_ok_rps": 140.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 34.38, + "requests": 6657, + "ok": 6657, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 226.6, + "write_ok_rps": 133.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "4fc3ad11e268", + "c9fab9c9e2c1", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 34.38, + "to_s": 42.45, + "requests": 1674, + "ok": 1674, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 207.2, + "write_ok_rps": 121.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 110.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 12.44, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 13.16, + [ + "api1", + "api2" + ] + ], + [ + 15.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.44, + [ + "api2", + "api3" + ] + ], + [ + 26.94, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 31.47, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5617, + "stored_messages": 5617, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/evidence/rolling_update.txt b/evidence/rolling_update.txt new file mode 100644 index 00000000..03b1ccc1 --- /dev/null +++ b/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:10:19 UTC | Rolling update lab4 -> lab4.1 под нагрузкой | exit 0 | 44.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['4fc3ad11e268', 'c9fab9c9e2c1'] +2026-11-04T14:09:41.527Z rolling-update: start; replicas=2 old=['c9fab9c9e2c1', '4fc3ad11e268'] +2026-11-04T14:09:45.307Z rolling-update: images built in 3.8s (tag lab4.1) +2026-11-04T14:09:47.489Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:09:59.096Z rolling-update: step 1/2 new=1c20a55eebeb (messenger-api:lab4.1, ready+UP in 7.0s) replaced old=c9fab9c9e2c1 (graceful stop 4.4s) +2026-11-04T14:10:09.967Z rolling-update: step 2/2 new=ee285aa5856f (messenger-api:lab4.1, ready+UP in 6.6s) replaced old=4fc3ad11e268 (graceful stop 4.0s) +2026-11-04T14:10:10.389Z rolling-update: done in 29.1s; replicas now ['1c20a55eebeb', 'ee285aa5856f'] +update 29.4s rc=0 | replicas ['4fc3ad11e268', 'c9fab9c9e2c1'] -> ['1c20a55eebeb', 'ee285aa5856f'] | errors during update 0 (-) | RPS before 236.0, during 226.6, after 207.2 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-11-04 14:11:03 UTC | Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой | exit 0 | 43.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:26.009Z rolling-update: start; replicas=2 old=['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:28.537Z rolling-update: images built in 2.5s (tag lab4) +2026-11-04T14:10:30.771Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:10:42.303Z rolling-update: step 1/2 new=4749f3b4b256 (messenger-api:lab4, ready+UP in 7.2s) replaced old=1c20a55eebeb (graceful stop 4.1s) +2026-11-04T14:10:53.630Z rolling-update: step 2/2 new=7d965c2245c9 (messenger-api:lab4, ready+UP in 7.0s) replaced old=ee285aa5856f (graceful stop 4.2s) +2026-11-04T14:10:54.052Z rolling-update: done in 28.0s; replicas now ['4749f3b4b256', '7d965c2245c9'] +update 28.3s rc=0 | replicas ['1c20a55eebeb', 'ee285aa5856f'] -> ['4749f3b4b256', '7d965c2245c9'] | errors during update 0 (-) | RPS before 205.2, during 205.4, after 185.1 | p95 during 94.0 ms | data ok=True +PASS: rolling update under load + +# 2026-11-04 14:11:03 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4749f3b4b256 messenger-lab4-api-5 running healthy messenger-api:lab4 api2 UP +7d965c2245c9 messenger-lab4-api-6 running healthy messenger-api:lab4 api3 UP + diff --git a/evidence/s3_bootstrap.txt b/evidence/s3_bootstrap.txt new file mode 100644 index 00000000..52e17d57 --- /dev/null +++ b/evidence/s3_bootstrap.txt @@ -0,0 +1,225 @@ +# 2026-11-04 14:03:57 UTC | S3 bootstrap: лог первого запуска s3-init | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color s3-init +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +# 2026-11-04 14:04:00 UTC | S3 bootstrap: повторный запуск (идемпотентность) | exit 0 | 2.3s +$ docker compose --env-file .env run --rm -T s3-init + Container messenger-lab4-s3-1 Running + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-s3-init-run-8fa89d92841c Creating + Container messenger-lab4-s3-init-run-8fa89d92841c Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} + diff --git a/evidence/s3_outage.json b/evidence/s3_outage.json new file mode 100644 index 00000000..465a6c63 --- /dev/null +++ b/evidence/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 20.34, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.53s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "6.53s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "6.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 3.66s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "3.66s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "20.34s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/evidence/s3_outage.txt b/evidence/s3_outage.txt new file mode 100644 index 00000000..bcb31b0d --- /dev/null +++ b/evidence/s3_outage.txt @@ -0,0 +1,24 @@ +# 2026-11-04 14:09:34 UTC | Недоступность S3 и восстановление | exit 0 | 48.7s +$ python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json +PASS baseline upload 201 — 201 +$ docker compose --env-file .env stop s3 +PASS readiness stays 200 while S3 is down (S3 is not a readiness dependency) — 200 +PASS text message 201 during S3 outage — 201 +PASS history 200 during S3 outage — 200 +PASS attachment metadata 200 (PostgreSQL only) — 200 +PASS message referencing existing attachment 201 — 201 +PASS upload #1 -> 503 unavailable — 503 unavailable in 6.53s +PASS upload #1 failed fast (<= 15s) — 6.53s +PASS upload #1 has Retry-After +PASS upload #2 -> 503 unavailable — 503 unavailable in 6.31s +PASS upload #2 failed fast (<= 15s) — 6.31s +PASS upload #2 has Retry-After +PASS download -> 503 during outage — 503 in 3.66s +PASS download failed fast — 3.66s +PASS session still valid — 200 +$ docker compose --env-file .env start s3 +$ C:\Users\mrend\AppData\Local\Programs\Python\Python312\python.exe scripts/ops.py wait-s3 +PASS S3 read/write warm-up after start — 20.34s +PASS old file downloads byte-for-byte after recovery — 200 0.02s +PASS new upload 201 after recovery (same token, no API restart) — 201 +PASS diff --git a/evidence/s3_private.txt b/evidence/s3_private.txt new file mode 100644 index 00000000..4a729d30 --- /dev/null +++ b/evidence/s3_private.txt @@ -0,0 +1,67 @@ +# 2026-11-04 14:04:54 UTC | Приватный бакет вне API, права ключей, закрытые админ-порты | exit 0 | 2.3s +$ python scripts/ops.py s3-check + Container messenger-lab4-api-run-33922d9bc56f Creating + Container messenger-lab4-api-run-33922d9bc56f Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} +# 2026-11-04 14:04:54 UTC | На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes | exit 0 | 0.5s +$ python scripts/ops.py api-fs +messenger-lab4-api-1 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] +messenger-lab4-api-2 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] + +# 2026-11-04 14:04:55 UTC | API: файлы в /tmp (tmpfs) после загрузок | exit 0 | 0.3s +$ docker compose --env-file .env exec -T api sh -c df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l) +Filesystem Size Used Avail Use% Mounted on +tmpfs 16M 0 16M 0% /tmp +files_in_tmp=0 + diff --git a/evidence/sandbox/check.txt b/evidence/sandbox/check.txt new file mode 100644 index 00000000..395bd0f8 --- /dev/null +++ b/evidence/sandbox/check.txt @@ -0,0 +1,6 @@ +# 2026-11-02T13:36:48Z | offline contract +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-02T13:36:48Z | full OpenAPI validation +$ python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/sandbox/consistency.txt b/evidence/sandbox/consistency.txt new file mode 100644 index 00000000..3ca0db12 --- /dev/null +++ b/evidence/sandbox/consistency.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:02Z | DB<->S3 consistency, aborted upload, injected faults (api-11 s3_error_on_put, api-12 db_error_after_s3_put, api-13 crash_after_s3_put) +$ BASE_URL=... FAULT_*_URL=... python -m unittest discover -s student-tests -p test_attachment_consistency.py -v +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.391s + +OK diff --git a/evidence/sandbox/failover.txt b/evidence/sandbox/failover.txt new file mode 100644 index 00000000..d68450f0 --- /dev/null +++ b/evidence/sandbox/failover.txt @@ -0,0 +1,12 @@ +# 2026-11-02T13:28:02Z failover run 1: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 4.468} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:27:47Z stop-one api-1 pid 7684 +restored; next run +# 2026-11-02T13:28:23Z failover run 2: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 0, "first_recovery_seconds": 4.475} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:28:08Z stop-one api-1 pid 7937 +restored; next run diff --git a/evidence/sandbox/failover_series.json b/evidence/sandbox/failover_series.json new file mode 100644 index 00000000..c69f3b37 --- /dev/null +++ b/evidence/sandbox/failover_series.json @@ -0,0 +1,268 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "api-1", + "api-2" + ], + "rounds": [ + { + "target_instance": "api-1", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.0, + "lb_detect_down_seconds": 1.3, + "target_last_served_after_stop_seconds": 1.31, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.86, + "lb_back_up_seconds": 2.91, + "target_back_in_rotation_seconds": 2.87, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1597, + "ok": 1597, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 318.1, + "write_ok_rps": 187.8, + "latency_ms_ok": { + "p50": 16.7, + "p95": 38.6, + "p99": 50.6 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 2392, + "ok": 2392, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 239.2, + "write_ok_rps": 140.9, + "latency_ms_ok": { + "p50": 24.4, + "p95": 39.5, + "p99": 48.8 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 1057, + "ok": 1057, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 211.4, + "write_ok_rps": 124.2, + "latency_ms_ok": { + "p50": 27.6, + "p95": 44.5, + "p99": 54.8 + }, + "instances": [ + "api-2" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 26.15, + "requests": 1671, + "ok": 1671, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 272.8, + "write_ok_rps": 160.7, + "latency_ms_ok": { + "p50": 21.2, + "p95": 39.0, + "p99": 49.4 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api-1", + "api-2" + ] + ], + [ + 6.32, + [ + "api-2" + ] + ], + [ + 22.93, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 3957, + "stored_messages": 3957, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "api-2", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.23, + "lb_detect_down_seconds": 1.53, + "target_last_served_after_stop_seconds": 1.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.45, + "lb_back_up_seconds": 3.05, + "target_back_in_rotation_seconds": 3.05, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.01, + "requests": 1416, + "ok": 1416, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 282.9, + "write_ok_rps": 164.8, + "latency_ms_ok": { + "p50": 19.4, + "p95": 41.3, + "p99": 53.4 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.01, + "to_s": 15.01, + "requests": 2431, + "ok": 2431, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 243.1, + "write_ok_rps": 142.8, + "latency_ms_ok": { + "p50": 24.8, + "p95": 39.6, + "p99": 46.3 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.01, + "to_s": 20.01, + "requests": 1154, + "ok": 1154, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 230.7, + "write_ok_rps": 136.0, + "latency_ms_ok": { + "p50": 25.4, + "p95": 39.9, + "p99": 46.5 + }, + "instances": [ + "api-1" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 26.3, + "requests": 1851, + "ok": 1851, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 293.9, + "write_ok_rps": 172.6, + "latency_ms_ok": { + "p50": 19.8, + "p95": 35.8, + "p99": 42.1 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.01, + [ + "api-1", + "api-2" + ] + ], + [ + 6.53, + [ + "api-1" + ] + ], + [ + 23.06, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 4020, + "stored_messages": 4020, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/evidence/sandbox/load.json b/evidence/sandbox/load.json new file mode 100644 index 00000000..f7d5a058 --- /dev/null +++ b/evidence/sandbox/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.055, + "concurrency": 4, + "successful_requests": 5544, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 276.435, + "successful_latency_ms": { + "p50": 13.241, + "p95": 23.756, + "p99": 32.552 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/sandbox/persistence.txt b/evidence/sandbox/persistence.txt new file mode 100644 index 00000000..707b5ef0 --- /dev/null +++ b/evidence/sandbox/persistence.txt @@ -0,0 +1,7 @@ +# 2026-11-02T13:24:54Z native sandbox: python3 tests/resilience.py --mode persistence --action-script +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# hook log: PIDs before/after (postgres, seaweedfs, api x2) + S3 warm-up +2026-11-02T13:24:15Z restart begin pg=6347 s3=6937 api=6405,6525 +{"s3_ready_after_s": 16.4, "last_cycle_s": 1.028, "existing_object_read": true} +2026-11-02T13:24:53Z restart end pg=7134 s3=7149 api=7195,7219 diff --git a/evidence/sandbox/reconcile.txt b/evidence/sandbox/reconcile.txt new file mode 100644 index 00000000..78029ae9 --- /dev/null +++ b/evidence/sandbox/reconcile.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:06Z | reconcile dry-run +$ python -m src.reconcile +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 238, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.036, + "ready_rows": 238, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.045 +} diff --git a/evidence/sandbox/s3_bootstrap.txt b/evidence/sandbox/s3_bootstrap.txt new file mode 100644 index 00000000..2a46dc37 --- /dev/null +++ b/evidence/sandbox/s3_bootstrap.txt @@ -0,0 +1,45 @@ +# 2026-11-02T13:36:49Z | S3 bootstrap, 2nd run on existing bucket (idempotent) +$ python -m src.s3_bootstrap +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} diff --git a/evidence/sandbox/s3_outage.json b/evidence/sandbox/s3_outage.json new file mode 100644 index 00000000..21afe8a8 --- /dev/null +++ b/evidence/sandbox/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.42, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 1.75s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "1.75s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 2.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "2.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 0.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "0.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.42s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.01s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/evidence/sandbox/s3_outage_hang.json b/evidence/sandbox/s3_outage_hang.json new file mode 100644 index 00000000..4f585820 --- /dev/null +++ b/evidence/sandbox/s3_outage_hang.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.09, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.02s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "10.02s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.01s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "10.01s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 5.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "5.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.09s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/evidence/sandbox/s3_private.txt b/evidence/sandbox/s3_private.txt new file mode 100644 index 00000000..73989707 --- /dev/null +++ b/evidence/sandbox/s3_private.txt @@ -0,0 +1,54 @@ +# 2026-11-02T13:37:07Z | private bucket outside API (anonymous S3 requests), key permissions, admin ports +$ python -m src.s3_bootstrap --check-only --sample-from-db +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} diff --git a/evidence/sandbox/smoke.txt b/evidence/sandbox/smoke.txt new file mode 100644 index 00000000..c0127ee4 --- /dev/null +++ b/evidence/sandbox/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-02T13:36:50Z | public smoke +$ python3 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/sandbox/student-tests.txt b/evidence/sandbox/student-tests.txt new file mode 100644 index 00000000..bd5e9660 --- /dev/null +++ b/evidence/sandbox/student-tests.txt @@ -0,0 +1,120 @@ +# 2026-11-02T13:37:16Z | student tests over HTTP via LB (lab2-4); DB-level tests skip here and run below with DB access +$ python3 student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... + 24 parallel POSTs, one key -> 1 message; replicas: ['api-1', 'api-2'] +ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 6.028s + +OK (skipped=23) + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# DB-level tests (lab2-3) with direct DB access +$ python -m unittest discover -s student-tests -p "test_storage_*.py"; ... -p test_session_cache.py +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.022s + +OK +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.564s + +OK diff --git a/evidence/sandbox/upload_memory_c10.json b/evidence/sandbox/upload_memory_c10.json new file mode 100644 index 00000000..15d8fa09 --- /dev/null +++ b/evidence/sandbox/upload_memory_c10.json @@ -0,0 +1,43 @@ +{ + "label": "final code; peak via VmHWM and 20ms sampling", + "date_utc": "2026-11-02T13:41:45Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 5.06, + "latency_201_s": { + "p50": 1.316, + "p95": 2.069, + "max": 2.088280546000078, + "mean": 1.407 + }, + "memory": { + "pid-9293": { + "baseline_mib": 95.5, + "peak_mib": 163.4, + "growth_mib": 67.9, + "samples": 356 + }, + "pid-9364": { + "baseline_mib": 95.4, + "peak_mib": 156.1, + "growth_mib": 60.6, + "samples": 356 + } + }, + "process_peak_rss_mib": { + "pid-9293": 165.8, + "pid-9364": 165.3 + } +} diff --git a/evidence/sandbox/upload_memory_c20_limit4.json b/evidence/sandbox/upload_memory_c20_limit4.json new file mode 100644 index 00000000..dc214741 --- /dev/null +++ b/evidence/sandbox/upload_memory_c20_limit4.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-02T13:11:09Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.14, + "latency_201_s": { + "p50": 2.159, + "p95": 3.077, + "max": 3.089563180000141, + "mean": 2.079 + }, + "memory": { + "pid-3762": { + "baseline_mib": 114.5, + "peak_mib": 161.7, + "growth_mib": 47.2, + "samples": 387 + }, + "pid-3834": { + "baseline_mib": 96.5, + "peak_mib": 159.1, + "growth_mib": 62.6, + "samples": 387 + } + } +} diff --git a/evidence/sandbox/upload_memory_c20_nolimit.json b/evidence/sandbox/upload_memory_c20_nolimit.json new file mode 100644 index 00000000..84924f38 --- /dev/null +++ b/evidence/sandbox/upload_memory_c20_nolimit.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=64 (limiter effectively off)", + "date_utc": "2026-11-02T13:11:34Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.06, + "latency_201_s": { + "p50": 2.901, + "p95": 3.031, + "max": 3.03438846999984, + "mean": 2.702 + }, + "memory": { + "pid-4062": { + "baseline_mib": 88.2, + "peak_mib": 261.6, + "growth_mib": 173.4, + "samples": 400 + }, + "pid-4133": { + "baseline_mib": 88.7, + "peak_mib": 291.4, + "growth_mib": 202.8, + "samples": 400 + } + } +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..8d23ad7d --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-04 14:04:01 UTC | Публичный smoke | exit 0 | 0.8s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stand.txt b/evidence/stand.txt new file mode 100644 index 00000000..d4cd20d1 --- /dev/null +++ b/evidence/stand.txt @@ -0,0 +1,42 @@ +# 2026-11-04 14:03:55 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab4-api-1 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-api-2 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab4-migrate-1 messenger-lab4-migrate "python -m src.migra…" migrate 6 minutes ago Exited (0) 2 minutes ago +messenger-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 6 minutes ago Up 5 minutes (healthy) 5432/tcp +messenger-lab4-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 6 minutes ago Up 5 minutes (healthy) 6379/tcp +messenger-lab4-s3-1 chrislusf/seaweedfs:4.50 "/usr/bin/weed serve…" s3 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +messenger-lab4-s3-init-1 messenger-lab4-s3-init "python -m src.s3_bo…" s3-init 6 minutes ago Exited (0) 2 minutes ago +# 2026-11-04 14:03:55 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:03:57 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.4s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-lb-1 0.14% 13.6MiB / 128MiB 10.63% +messenger-lab4-api-2 0.26% 90.05MiB / 320MiB 28.14% +messenger-lab4-api-1 0.30% 106MiB / 320MiB 33.13% +messenger-lab4-postgres-1 0.13% 71.34MiB / 512MiB 13.93% +messenger-lab4-redis-1 2.31% 5.629MiB / 96MiB 5.86% +messenger-lab4-s3-1 0.21% 508.3MiB / 512MiB 99.28% + +# 2026-11-04 14:03:57 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.2s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp +s3 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp + +# 2026-11-04 14:03:57 UTC | Образ SeaweedFS (закреплённая версия) | exit 0 | 0.3s +$ docker compose --env-file .env images s3 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +messenger-lab4-s3-1 chrislusf/seaweedfs 4.50 linux/amd64 c2658820e763 197MB 10 days ago + diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..edbc5879 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,168 @@ +# 2026-11-04 14:04:10 UTC | Собственные тесты (HTTP через LB, включая test_attachments.py) | exit 0 | 9.4s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 7.352s + +OK (skipped=23) + + 24 parallel POSTs, one key -> 1 message; replicas: ['4fc3ad11e268', 'c9fab9c9e2c1'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-11-04 14:04:18 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.9s +$ python scripts/ops.py in-network storage + Container messenger-lab4-api-run-7c01226dcaba Creating + Container messenger-lab4-api-run-7c01226dcaba Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.302s + +OK + Container messenger-lab4-api-run-b6f4e53090a5 Creating + Container messenger-lab4-api-run-b6f4e53090a5 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.860s + +OK + +# 2026-11-04 14:04:33 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 15.2s +$ python scripts/ops.py in-network expiry + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-short-ttl-1 Creating + Container messenger-lab4-api-short-ttl-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-short-ttl-1 Starting + Container messenger-lab4-api-short-ttl-1 Started + Container messenger-lab4-api-run-ae85ff205e89 Creating + Container messenger-lab4-api-run-ae85ff205e89 Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab4-api-run-87803f7efa19 Creating + Container messenger-lab4-api-run-87803f7efa19 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.644s + +OK + diff --git a/evidence/upload_memory.txt b/evidence/upload_memory.txt new file mode 100644 index 00000000..5eb2de44 --- /dev/null +++ b/evidence/upload_memory.txt @@ -0,0 +1,108 @@ +# 2026-11-04 14:05:09 UTC | Пик памяти: 10 одновременных загрузок по 9.5 MiB | exit 0 | 14.2s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 10 --rounds 3 --docker --label c=10, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c10.json +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} +# 2026-11-04 14:05:29 UTC | Пик памяти: 20 одновременных загрузок | exit 0 | 20.4s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 20 --rounds 2 --docker --label c=20, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c20.json +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} + diff --git a/evidence/upload_memory_c10.json b/evidence/upload_memory_c10.json new file mode 100644 index 00000000..05051469 --- /dev/null +++ b/evidence/upload_memory_c10.json @@ -0,0 +1,51 @@ +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/evidence/upload_memory_c20.json b/evidence/upload_memory_c20.json new file mode 100644 index 00000000..5093d110 --- /dev/null +++ b/evidence/upload_memory_c20.json @@ -0,0 +1,52 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/infra/README.md b/infra/README.md index 5a39e862..4c634241 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,10 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Решение (lab4) + +- `haproxy/haproxy.cfg` — балансировщик (из lab3) + длинный `timeout server` только для `POST …/attachments`. +- SeaweedFS настраивается флагами в `compose.yaml` (сервис `s3`) и файлом identities `secrets/s3_config.json`, который генерирует `scripts/init_secrets.py` (две identity: `bootstrap_admin` — только для `s3-init`; `messenger_api` — `Read/Write/List` одного бакета; анонимного доступа нет). +- Bootstrap бакета и проверки приватности: `src/s3_bootstrap.py` (сервис `s3-init`, повтор — no-op). +- Таблица «роль → сервис → версия → health → volume → сеть/порт → credentials» — в REPORT.md, раздел «Выбор S3-сервера». diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..62706b67 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,104 @@ +# HAProxy 2.8 LTS — единственная публичная точка входа (lab3, дополнено в lab4). +# +# Обнаружение реплик: Docker DNS (127.0.0.11) отдаёт по имени сервиса `api` +# A-записи ВСЕХ запущенных контейнеров сервиса. server-template держит до 6 слотов +# и каждые 2 с синхронизирует их с DNS: новый контейнер (scale / rolling update) +# появляется сам, остановленный исчезает. Перезапуск LB при масштабировании не нужен. +# +# Исключение недоступного backend (активные проверки): +# GET /health/ready каждые inter=1s; после первой неудачи — fastinter=500ms; +# fall=2 неудачи подряд -> DOWN; rise=2 удачи -> UP; timeout check 1500ms. +# Время обнаружения: +# * graceful stop (реплика сразу отвечает 503): <= 1 с + 0.5 с ~ 1.5 с +# * crash / kill (соединение отклонено/сброшено): ~1.5 с +# * зависание без ответа: <= 1 + 1.5 + 0.5 + 1.5 = 4.5 с +# Пока реплика ещё не помечена DOWN, запрос к ней, не сумевший СОЕДИНИТЬСЯ +# (timeout connect 1s), повторяется на другой реплике (retries 2 + redispatch). +# Повторяются только ошибки установления соединения (retry-on conn-failure по +# умолчанию): запрос ещё не был отправлен, поэтому это безопасно и для POST. +# Уже отправленные POST LB не повторяет — это делает клиент с тем же +# Idempotency-Key (контракт запрещает прозрачные повторы POST /users, /chats). +# +# Согласование таймаутов: +# API: пул БД 3 с + statement_timeout 5 с = максимум ~8 с на запрос +# -> timeout server 10s; graceful shutdown API 10 с; drain API 3 с > 1.5 с. +# lab4: загрузка вложения (POST .../attachments) — до UPLOAD_READ_TIMEOUT_SECONDS +# 60 с чтения тела + очередь 5 с + S3 PUT (2 попытки по ~12 с) -> для этого +# пути timeout server 90s (http-request set-timeout), остальные — прежние 10 с. +# Скачивание идёт потоком, timeout server — это пауза между пакетами, не общая длительность. +# Docker stop_grace_period 20 с > drain 3 с + graceful 10 с. + +global + log stdout format raw local0 info + maxconn 2000 + +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + # 8 КБ: в ответ DNS помещаются A-записи всех слотов. + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + option redispatch 1 + retries 2 + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1500ms + default-server inter 1s fastinter 500ms downinter 2s fall 2 rise 2 + +frontend public + bind :8080 + # Сквозной идентификатор запроса: HAProxy генерирует X-Request-Id, если клиент + # не прислал свой; API отражает его в ответе и в теле ошибок, LB — в логе. + unique-id-format %{+X}o\ %ci%cp-%fp-%Ts%ms-%rt-%pid + http-request set-header X-Request-Id %[unique-id] unless { req.hdr(X-Request-Id) -m found } + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[capture.req.hdr(0)] inst=%[capture.res.hdr(0)] \"%HM %HPO\"" + http-request capture req.hdr(X-Request-Id) len 64 + declare capture response len 32 + http-response capture res.hdr(X-Instance-Id) id 0 + default_backend api + +backend api + balance roundrobin + # lab4: только загрузка вложения получает длинный timeout server (см. выше). + acl is_upload method POST + acl is_upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 90s if is_upload is_upload_path + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # Ответы, которые формирует сам LB (нет ни одной готовой реплики, таймаут), + # — в общем JSON-формате ошибок контракта, с тем же X-Request-Id. + http-error status 503 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API instance is available\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API instance timeout\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 502 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Bad gateway\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + # До 6 реплик (в т.ч. N+1 во время rolling update). init-addr none: LB стартует, + # даже если реплик ещё нет; слоты без адреса находятся в MAINT. + server-template api 1-6 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +# Внутренняя страница состояния (UP/DOWN по репликам) — только чтение. +# Публикуется на 127.0.0.1:8404 хоста; используется scripts/ops.py. +frontend stats + bind :8404 + monitor-uri /lb-health + stats enable + stats uri /stats + stats refresh 2s + stats show-legends + http-request set-log-level silent diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..1bfb18ff --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,66 @@ +-- 001_init: базовая схема мессенджера lab2. +-- Применяется src/migrate.py ровно один раз (таблица schema_migrations), +-- внутри одной транзакции; повторный запуск миграций ничего не дублирует. + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Список чатов пользователя: WHERE user_id = ? ORDER BY (created_at, id) +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); + +-- "В каких чатах состоит пользователь" (GET /chats) +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- История чата с keyset-пагинацией: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT $4 +-- Обоснование и EXPLAIN (ANALYZE, BUFFERS): evidence/explain_history.txt +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +-- Серверные сессии. Источник истины для сессий — PostgreSQL; +-- Redis хранит только кэш проверок токена (см. REPORT.md). +-- Хранится SHA-256 от токена, сам токен в БД не попадает. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_user_id_idx ON sessions (user_id); +CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); diff --git a/migrations/002_session_cache_invalidations.sql b/migrations/002_session_cache_invalidations.sql new file mode 100644 index 00000000..1c26116b --- /dev/null +++ b/migrations/002_session_cache_invalidations.sql @@ -0,0 +1,13 @@ +-- 002: надёжная инвалидация кэша сессий. +-- Строка вставляется В ТОЙ ЖЕ транзакции, что и sessions.revoked_at, и удаляется +-- только после того, как в Redis записано «надгробие» для этого токена. +-- Пока в таблице есть строки, API не доверяет положительным записям кэша +-- и проверяет каждый токен в PostgreSQL (см. src/sessions.py). +CREATE TABLE session_cache_invalidations ( + token_hash bytea PRIMARY KEY, + revoked_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT session_cache_invalidations_hash_len CHECK (octet_length(token_hash) = 32) +); + +CREATE INDEX session_cache_invalidations_revoked_at_idx + ON session_cache_invalidations (revoked_at); diff --git a/migrations/003_idempotency_keys.sql b/migrations/003_idempotency_keys.sql new file mode 100644 index 00000000..af60c764 --- /dev/null +++ b/migrations/003_idempotency_keys.sql @@ -0,0 +1,40 @@ +-- 003 (lab3): общая идемпотентность POST /chats/{chat_id}/messages. +-- +-- Область уникальности по контракту: пользователь + HTTP-метод + путь чата + ключ. +-- Путь однозначно задаётся chat_id, метод сейчас всегда POST (колонка оставлена, +-- чтобы область ключа была записана явно и расширялась без смены PK). +-- +-- Атомарность: строка ключа и сообщение пишутся ОДНОЙ транзакцией. +-- 1. INSERT ключа (ON CONFLICT ...) — конкурентный повтор с тем же ключом +-- ждёт на уникальном индексе, пока первая транзакция не завершится; +-- 2. INSERT сообщения с заранее выбранным id; +-- 3. COMMIT. FK message_id -> messages проверяется в момент COMMIT +-- (DEFERRABLE INITIALLY DEFERRED), поэтому message_id NOT NULL с первого шага. +-- Ошибка/откат на любом шаге не оставляет ни сообщения, ни ключа: +-- транзиентный 5xx не фиксируется как окончательный результат. +-- +-- Совместимость с работающей версией (rolling update): миграция только +-- добавляет новую таблицу, старые таблицы и запросы не меняются. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + method text NOT NULL DEFAULT 'POST', + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL + REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idem_key), + CONSTRAINT idempotency_keys_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_method CHECK (method IN ('POST')), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_expiry_after_create CHECK (expires_at > created_at) +); + +-- Периодическая очистка истёкших ключей (src/idempotency.py). +CREATE INDEX idempotency_keys_expires_at_idx ON idempotency_keys (expires_at); +-- ON DELETE CASCADE от messages без полного скана. +CREATE INDEX idempotency_keys_message_id_idx ON idempotency_keys (message_id); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..20561c2d --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,68 @@ +-- 004 (lab4): вложения. Метаданные — в PostgreSQL, байты — в приватном S3-бакете. +-- +-- Жизненный цикл строки attachments (внутреннее поле state, в API не видно): +-- pending — намерение записано ДО S3 PUT (одна короткая транзакция); +-- ready — объект записан в S3, затем UPDATE ... WHERE state='pending'. +-- Наружу (GET, ссылки из сообщений) доступны только ready. Строки pending, +-- оставшиеся после сбоя между S3 и COMMIT, удаляет сборщик (src/reconcile.py) +-- после grace period; сначала удаляется строка (атомарный «захват»), потом объект. +-- Миграция только расширяющая (expand): реплики lab3 продолжают работать. + +-- Сообщение с вложениями может иметь пустой текст (text = ""). +-- Запрет «пустой текст без вложений» проверяет приложение. +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); +-- Цель составного внешнего ключа message_attachments (message_id, chat_id). +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE RESTRICT, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + -- Только отображаемое имя. Никогда не используется как путь или ключ S3. + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + -- Ключ объекта назначает сервер (случайный UUID), он не равен id вложения + -- и не раскрывается в API. + object_key text NOT NULL, + state text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_state CHECK (state IN ('pending', 'ready')) +); + +-- Сборщик незавершённых загрузок: WHERE state = 'pending' AND created_at < ... +CREATE INDEX attachments_pending_created_idx ON attachments (created_at) WHERE state = 'pending'; +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +-- Связь сообщение -> вложения (порядок attachment_ids сохраняется в position). +-- Составные внешние ключи гарантируют НА УРОВНЕ БД, что вложение и сообщение +-- принадлежат одному чату. ON DELETE RESTRICT: строку вложения, на которую +-- ссылается сообщение, нельзя удалить (сборщик удаляет только pending, а +-- pending прикрепить нельзя — двойная защита). +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..095b9d60 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic==2.10.4 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +# lab4: S3-клиент и независимый парсер изображений +boto3==1.35.99 +Pillow==11.1.0 diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py new file mode 100644 index 00000000..414546ae --- /dev/null +++ b/scripts/collect_evidence.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Прогон всех проверок lab4 на запущенном compose-стенде с записью в evidence/. + + python scripts/collect_evidence.py # ~12-15 минут + python scripts/collect_evidence.py --quick # ~7-8 минут: без kill-серии, rolling update и нагрузки + +Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`, +`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики, +перезапускает хранилища и S3 (volumes сохраняются), останавливает S3 и делает +rolling update lab4 -> lab4.1 -> lab4 (откат), поэтому запускайте его только на учебном стенде. +Токены и пароли в файлы не пишутся (тесты держат их в памяти). +""" +import argparse +import json +import os +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.request import Request, urlopen + +ROOT = Path(__file__).resolve().parents[1] +EV = ROOT / "evidence" +PY = sys.executable or "python" +WIN = os.name == "nt" +HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh")) +HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh")) +BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080") +RESULTS = [] + +try: + sys.stdout.reconfigure(errors="replace") +except AttributeError: + pass + + +def stamp(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def step(title, cmd, out_file, append=False, env=None, timeout=900): + """Выполнить команду, сохранить «$ команда + вывод» в evidence/.""" + print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True) + started = time.monotonic() + try: + proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + timeout=timeout, env=dict(os.environ, **(env or {})), + encoding="utf-8", errors="replace") + output, rc = proc.stdout, proc.returncode + except subprocess.TimeoutExpired as exc: + output, rc = (exc.stdout or "") + "\nTIMEOUT", 124 + elapsed = time.monotonic() - started + shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "") + text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip()) + path = EV / out_file + with path.open("a" if append else "w", encoding="utf-8") as f: + f.write(text + ("\n" if append else "")) + print(output.rstrip()[-1500:], flush=True) + RESULTS.append((title, rc, out_file)) + return rc, output + + +def instances_through_lb(n=20): + """Одна сессия, n запросов через LB -> какие реплики отвечали.""" + import base64 + import uuid + name = "inst_" + uuid.uuid4().hex[:12] + pw = "Inst-" + uuid.uuid4().hex + body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode() + urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10) + cred = base64.b64encode((name + ":" + pw).encode()).decode() + sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST", + headers={"Authorization": "Basic " + cred}), timeout=10).read()) + lines = [] + for i in range(n): + with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r: + lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % ( + i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id"))) + return lines + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--quick", action="store_true", help="без SIGKILL-серии, rolling update и нагрузки") + args = ap.parse_args() + EV.mkdir(exist_ok=True) + (EV / "ops.log").write_text("", encoding="utf-8") + dc = ["docker", "compose", "--env-file", ".env"] + + step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt") + step("Compose config", dc + ["config", "--quiet"], "check.txt", append=True) + step("Стенд: контейнеры", dc + ["ps", "-a"], "stand.txt") + step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True) + step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True) + step("Стенд: опубликованные порты (только lb)", dc + ["ps", "--format", "table {{.Service}}\t{{.Ports}}"], + "stand.txt", append=True) + step("Образ SeaweedFS (закреплённая версия)", dc + ["images", "s3"], "stand.txt", append=True) + + # Bucket bootstrap: первый запуск был при up; повтор — no-op. + step("S3 bootstrap: лог первого запуска s3-init", dc + ["logs", "--no-color", "s3-init"], "s3_bootstrap.txt") + step("S3 bootstrap: повторный запуск (идемпотентность)", dc + ["run", "--rm", "-T", "s3-init"], + "s3_bootstrap.txt", append=True) + + lines = instances_through_lb() + (EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n" + % (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})), + encoding="utf-8") + print("\n".join(lines)) + + step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt") + step("Собственные тесты (HTTP через LB, включая test_attachments.py)", + [PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt") + step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"], + "student-tests.txt", append=True) + step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"], + "student-tests.txt", append=True) + step("Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*)", + [PY, "scripts/ops.py", "in-network", "consistency"], "consistency.txt") + step("Сверка БД <-> S3 (dry-run)", [PY, "scripts/ops.py", "reconcile"], "reconcile.txt") + step("Приватный бакет вне API, права ключей, закрытые админ-порты", [PY, "scripts/ops.py", "s3-check"], + "s3_private.txt") + step("На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes", + [PY, "scripts/ops.py", "api-fs"], "s3_private.txt", append=True) + step("API: файлы в /tmp (tmpfs) после загрузок", + dc + ["exec", "-T", "api", "sh", "-c", "df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l)"], + "s3_private.txt", append=True) + + step("Пик памяти: 10 одновременных загрузок по 9.5 MiB", + [PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "10", "--rounds", "3", + "--docker", "--label", "c=10, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c10.json"], + "upload_memory.txt") + step("Пик памяти: 20 одновременных загрузок", + [PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "20", "--rounds", "2", + "--docker", "--label", "c=20, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c20.json"], + "upload_memory.txt", append=True) + + # Публичный тест останавливает реплику, обслужившую его первое сообщение. + # Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь + # лишними запросами, пока цель не окажется другой (README: «повторить для второй»). + targets = [] + for i in range(1, 5): + if len(set(targets)) >= 2: + break + for _ in range(i - 1): + urlopen(BASE + "/health/live", timeout=5).read() + step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1) + step("Failover #%d: make test-failover (вкл. скачивание файла через другую реплику)" % i, + [PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE], + "failover.txt", append=True) + step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"], + "failover.txt", append=True) + step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True) + stops = [l for l in (EV / "ops.log").read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l] + if stops: + targets.append(stops[-1].split("target=")[1].split()[0]) + with (EV / "failover.txt").open("a", encoding="utf-8") as f: + f.write("# остановленные публичным тестом реплики: %s\n" % targets) + + step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE, + "--output", "evidence/failover_series.json"], "failover_series.txt") + if not args.quick: + step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE, + "--mode", "kill", "--output", "evidence/failover_series_kill.json"], + "failover_series.txt", append=True) + + step("Недоступность S3 и восстановление", [PY, "student-tests/s3_outage.py", "--base-url", BASE, + "--output", "evidence/s3_outage.json"], "s3_outage.txt") + + if not args.quick: + step("Rolling update lab4 -> lab4.1 под нагрузкой", + [PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"], + "rolling_update.txt", env={"APP_VERSION": "lab4.1"}) + step("Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой", + [PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"], + "rolling_update.txt", append=True, env={"APP_VERSION": "lab4"}) + step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True) + + fmt = ["ps", "-a", "--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"] + step("Persistence: контейнеры до", dc + fmt, "persistence.txt") + step("Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB)", + [PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE], + "persistence.txt", append=True) + step("Persistence: контейнеры после (Status = Up N seconds)", dc + fmt, "persistence.txt", append=True) + step("Persistence: сверка БД <-> S3 после рестарта", [PY, "scripts/ops.py", "reconcile"], "persistence.txt", + append=True) + + if not args.quick: + step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30", + "--concurrency", "8", "--output", "evidence/load.json"], "load.txt") + step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True) + + step("Лог LB (последние строки, без секретов)", dc + ["logs", "--no-color", "--tail", "60", "lb"], "lb_log_sample.txt") + + summary = ["# %s" % stamp()] + ["%-4s %-70s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS] + (EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8") + print("\n" + "\n".join(summary)) + sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..aec4f962 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Создаёт локальные учебные секреты в ./secrets (каталог в .gitignore). + +Повторный запуск ничего не перезаписывает: пароль PostgreSQL задаётся только +при первой инициализации volume, и смена файла сломала бы подключение. +Значения секретов не печатаются. + +lab4: две пары S3-ключей — bootstrap-admin (только для one-shot s3-init) и +messenger-api (Read/Write/List одного бакета). s3_config.json — identities +SeaweedFS, собирается из этих ключей (пересоздаётся, если удалён: ключи S3 не +привязаны к данным в volume, их можно сменить: удалить 5 файлов s3_* и +перезапустить init_secrets.py + docker compose up -d --force-recreate s3 s3-init api). +""" +import json +import os +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIR = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + path = DIR / name + if path.exists(): + print("keep secrets/%s" % name) + return + path.write_text(value + "\n", encoding="utf-8") + # Каталог 0700, файлы 0644: внутри контейнера их читают непривилегированные + # пользователи (appuser, redis), а на хосте каталог закрыт. + os.chmod(path, 0o644) + print("create secrets/%s" % name) + + +def read(name: str) -> str: + return (DIR / name).read_text(encoding="utf-8").strip() + + +def main() -> None: + DIR.mkdir(exist_ok=True) + try: + os.chmod(DIR, 0o700) + except OSError: + pass + write("postgres_password", secrets.token_hex(24)) + write("redis_password", secrets.token_hex(24)) + write("database_url", "postgresql://messenger:%s@postgres:5432/messenger" % read("postgres_password")) + write("cache_url", "redis://:%s@redis:6379/0" % read("redis_password")) + # lab4: S3 (SeaweedFS). Access key — ASCII 20 символов, secret — 40 (как у AWS). + write("s3_admin_access_key_id", "ADM" + secrets.token_hex(8).upper() + "X") + write("s3_admin_secret_access_key", secrets.token_urlsafe(30)[:40]) + write("s3_access_key_id", "API" + secrets.token_hex(8).upper() + "X") + write("s3_secret_access_key", secrets.token_urlsafe(30)[:40]) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + config = { + "identities": [ + { + "name": "bootstrap_admin", + "credentials": [{"accessKey": read("s3_admin_access_key_id"), + "secretKey": read("s3_admin_secret_access_key")}], + "actions": ["Admin", "Read", "List", "Tagging", "Write"], + }, + { + # Только свой бакет: чтение, запись/удаление объектов, листинг. + # Создать/удалить бакет или прочитать чужой этими ключами нельзя. + "name": "messenger_api", + "credentials": [{"accessKey": read("s3_access_key_id"), + "secretKey": read("s3_secret_access_key")}], + "actions": ["Read:" + bucket, "Write:" + bucket, "List:" + bucket], + }, + ] + # Identity "anonymous" не объявлена -> анонимные запросы получают 403. + } + write("s3_config.json", json.dumps(config, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 00000000..72b2e41c --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Операции над compose-стендом lab4. Python 3.9+, только стандартная библиотека. + +Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает +`docker compose` и `docker` без shell-интерполяции. Секреты не печатает. + + python scripts/ops.py wait # дождаться N здоровых реплик и LB + python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB + python scripts/ops.py stop-one # TARGET_INSTANCE_ID=; оставляет остановленной + python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB + python scripts/ops.py restart # hook persistence: API + PostgreSQL + Redis + S3 + LB, volumes целы + python scripts/ops.py wait-s3 # дождаться, пока S3 реально читает/пишет (прогрев после рестарта) + python scripts/ops.py s3-check # приватность бакета вне API, права ключей, закрытые админ-порты + python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (по умолчанию dry-run) + python scripts/ops.py api-fs # у API read-only rootfs, только tmpfs, без volumes + python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса + python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети + python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с) + python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы + +Переменные окружения: + COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab4, как `name:` в compose.yaml) + ENV_FILE файл переменных (по умолчанию .env, если существует) + API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2) + APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404) + STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение») + +Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) — +это доказательство того, какой контейнер реально остановлен/заменён. +""" +import csv +import io +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OPS_LOG = ROOT / "evidence" / "ops.log" + +try: # консоль Windows может быть не в UTF-8 + sys.stdout.reconfigure(errors="replace") + sys.stderr.reconfigure(errors="replace") +except AttributeError: + pass + + +# ---------------------------------------------------------------- settings + +def _dotenv(): + path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env"))) + values = {} + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + values[k.strip()] = v.strip().strip('"').strip("'") + return path if path.is_file() else None, values + + +ENV_PATH, DOTENV = _dotenv() + + +def setting(name, default): + return os.environ.get(name) or DOTENV.get(name) or default + + +PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab4") +REPLICAS = int(setting("API_REPLICAS", "2")) +APP_PORT = int(setting("APP_PORT", "8080")) +STATS_PORT = int(setting("LB_STATS_PORT", "8404")) +LB_URL = "http://127.0.0.1:%d" % APP_PORT +STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT + + +def now_iso(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def log(msg): + line = "%s %s" % (now_iso(), msg) + print(line, flush=True) + try: + OPS_LOG.parent.mkdir(exist_ok=True) + with OPS_LOG.open("a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + pass + + +# ---------------------------------------------------------------- docker + +def compose_cmd(*args): + cmd = ["docker", "compose", "-p", PROJECT] + if ENV_PATH: + cmd += ["--env-file", str(ENV_PATH)] + return cmd + list(args) + + +def run(cmd, check=True, capture=False, timeout=300, show=False): + """capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо.""" + out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL) + result = subprocess.run( + cmd, cwd=ROOT, check=False, timeout=timeout, + stdout=out, stderr=out, text=True, + ) + if check and result.returncode != 0: + detail = (result.stderr or "").strip()[-400:] if capture else "" + raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail)) + return result + + +def compose(*args, timeout=300): + return run(compose_cmd(*args), timeout=timeout, capture=True) + + +def api_containers(include_stopped=True): + """[{id, short, name, state, health, image, ip}] — только сервис api, без one-off.""" + cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}", + "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=api", + "--filter", "label=com.docker.compose.oneoff=False"] + if include_stopped: + cmd.insert(2, "-a") + ids = [x for x in run(cmd, capture=True).stdout.split() if x] + if not ids: + return [] + raw = run(["docker", "inspect"] + ids, capture=True).stdout + result = [] + for item in json.loads(raw): + state = item["State"] + nets = item.get("NetworkSettings", {}).get("Networks") or {} + ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")] + result.append({ + "id": item["Id"], + "short": item["Id"][:12], + "hostname": item["Config"].get("Hostname", ""), + "name": item["Name"].lstrip("/"), + "state": state.get("Status"), + "health": (state.get("Health") or {}).get("Status", "-"), + "image": item["Config"].get("Image"), + "ip": ips[0] if ips else "", + "created": item.get("Created", "")[:19], + }) + result.sort(key=lambda c: c["name"]) + return result + + +def running(containers=None): + return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"] + + +# ---------------------------------------------------------------- load balancer + +def lb_servers(): + """[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом).""" + data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace") + rows = csv.DictReader(io.StringIO(data.lstrip("# "))) + servers = [] + for r in rows: + if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"): + continue + addr = r.get("addr") or "" + status = r.get("status", "") + if status.startswith("MAINT") and not addr: + continue # пустой слот server-template + servers.append({"name": r["svname"], "status": status, "addr": addr}) + return servers + + +def lb_up_count(): + return sum(1 for s in lb_servers() if s["status"].startswith("UP")) + + +def lb_ready(): + try: + with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r: + return r.status == 200 + except (urllib.error.URLError, OSError): + return False + + +def wait_until(predicate, timeout, what, interval=0.3): + deadline = time.monotonic() + timeout + last_exc = None + while time.monotonic() < deadline: + try: + if predicate(): + return True + except (urllib.error.URLError, OSError, ValueError, KeyError) as exc: + last_exc = exc + time.sleep(interval) + raise SystemExit("timeout after %ss waiting for %s%s" % ( + timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else "")) + + +def wait_healthy(ids, timeout=90): + def ok(): + states = {c["id"]: c for c in api_containers()} + return all(states.get(i, {}).get("health") == "healthy" for i in ids) + wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids)) + + +def wait_stack(replicas, timeout=120): + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas, + timeout, "%d healthy api replicas" % replicas) + wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas) + wait_until(lb_ready, 30, "LB /health/ready = 200") + + +# ---------------------------------------------------------------- commands + +def cmd_wait(_args): + t0 = time.monotonic() + wait_stack(REPLICAS) + print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0)) + + +def cmd_ps(_args): + containers = api_containers() + try: + by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()} + except (urllib.error.URLError, OSError): + by_addr = {} + print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status")) + for c in containers: + s = by_addr.get(c["ip"], {}) + print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ( + c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-"))) + + +def find_target(target): + for c in api_containers(): + # X-Instance-Id = hostname контейнера = первые 12 символов его ID. + if target and (c["hostname"] == target or c["id"].startswith(target)): + return c + return None + + +def cmd_stop_one(_args): + target = os.environ.get("TARGET_INSTANCE_ID", "").strip() + if not target: + raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)") + mode = os.environ.get("STOP_MODE", "stop").strip().lower() + c = find_target(target) + if c is None: + raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target)) + if c["state"] != "running": + raise SystemExit("container %s is already %s" % (c["name"], c["state"])) + t0 = time.monotonic() + log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"])) + if mode == "kill": + # «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял + # контейнер сам: тест требует, чтобы реплика оставалась остановленной. + run(["docker", "update", "--restart", "no", c["id"]]) + run(["docker", "kill", "--signal", "KILL", c["id"]]) + else: + # SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с. + run(["docker", "stop", "--time", "20", c["id"]], timeout=40) + after = find_target(target) or {} + log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)" + % (c["name"], after.get("state", "removed"), time.monotonic() - t0)) + + +def cmd_restore(_args): + t0 = time.monotonic() + stopped = [c for c in api_containers() if c["state"] != "running"] + for c in stopped: # вернуть политику, снятую в режиме kill + run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False) + # --no-recreate: не пересоздавать работающие; up запускает остановленные + # и добирает число контейнеров до deploy.replicas. + compose("up", "-d", "--no-deps", "--no-recreate", "api") + wait_stack(REPLICAS) + log("restore: started %s; %d replicas healthy and UP in LB after %.1fs" + % ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0)) + + +def cmd_restart(_args): + """Hook для make test-persistence: перезапуск API, ВСЕХ хранилищ (PostgreSQL, + Redis, S3) и LB. + + Volumes не удаляются, данные не пересоздаются, bucket не создаётся заново. + Порядок: сначала LB и API (новые записи не принимаются), затем хранилища, + затем ожидание реальной готовности S3 к чтению/записи (SeaweedFS отвечает + /healthz сразу, но первые операции после рестарта ждут 15–30 с), затем API и LB. + """ + t0 = time.monotonic() + before = {c["name"]: c["short"] for c in running()} + started_before = {svc: container_started(svc) for svc in ("postgres", "redis", "s3")} + compose("stop", "lb", "api", timeout=120) + compose("restart", "postgres", "redis", "s3", timeout=120) + wait_service_healthy("s3", 60) + t_warm = time.monotonic() + warm = s3_warmup(70) + log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm)) + compose("start", "api", timeout=120) + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS, + 80, "api healthy after restart") + compose("start", "lb", timeout=60) + log("restart: started_at postgres %s -> %s; redis %s -> %s; s3 %s -> %s; api %s restarted; lb restarted; %.1fs" + % (started_before["postgres"], container_started("postgres"), started_before["redis"], + container_started("redis"), started_before["s3"], container_started("s3"), + sorted(before), time.monotonic() - t0)) + + +def wait_service_healthy(service, timeout): + def ok(): + ids = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() + if not ids: + return False + out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip() + return out == "healthy" + wait_until(ok, timeout, service + " healthy") + + +def s3_warmup(timeout=70): + """One-off контейнер образа API в сетях стенда: put/get/delete пробы + чтение старого объекта.""" + out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--wait-ready", str(timeout)), capture=True, timeout=timeout + 60) + lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")] + return lines[-1] if lines else "" + + +def cmd_wait_s3(_args): + wait_service_healthy("s3", 60) + print(s3_warmup(90)) + + +def cmd_s3_check(_args): + """Проверка приватного бакета ВНЕ API: анонимные запросы к S3 из сети storage.""" + run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--check-only", "--sample-from-db"), show=True, timeout=180) + + +def cmd_api_fs(_args): + """Доказательство «на API нет локального хранилища оригиналов».""" + for c in api_containers(): + info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0] + mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])] + print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % ( + c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none")) + + +def cmd_reconcile(args): + run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.reconcile") + list(args), + show=True, timeout=600) + + +def container_started(service): + ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() + if not ids: + return "-" + return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23] + + +def cmd_rolling_update(args): + """Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой. + + В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только + расширяющие (expand): старая версия продолжает работать на новой схеме. + """ + t_all = time.monotonic() + if [c for c in api_containers() if c["state"] != "running"]: + raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first") + old = running() + n = len(old) + if n < 1: + raise SystemExit("no running api replicas") + log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old])) + if "--no-build" not in args: + t = time.monotonic() + compose("build", "api", "migrate", timeout=900) + log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab4"))) + t = time.monotonic() + out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines() + log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else "")) + for i, victim in enumerate(old, 1): + before = {c["id"] for c in api_containers()} + t = time.monotonic() + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180) + new = [c for c in api_containers() if c["id"] not in before] + if len(new) != 1: + raise SystemExit("expected exactly one new container, got %d" % len(new)) + fresh = new[0] + wait_healthy([fresh["id"]], timeout=90) + wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy") + t_ready = time.monotonic() - t + t = time.monotonic() + run(["docker", "stop", "--time", "20", victim["id"]], timeout=40) + run(["docker", "rm", victim["id"]]) + log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)" + % (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t)) + wait_stack(n) + log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()])) + + +def cmd_in_network(args): + """Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl. + + Запускаются в one-off контейнере образа API внутри сети backend: порты БД и + реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`, + поэтому LB его не видит. + """ + which = args[0] if args else "" + mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests") + base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount) + if which == "storage": + for pattern in ("test_storage_*.py", "test_session_cache.py"): + run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"], + show=True, timeout=600) + elif which == "consistency": + prof = compose_cmd("--profile", "faults") + faults = ["api-fault-s3", "api-fault-db", "api-fault-crash"] + run(prof + ["up", "-d", "--no-recreate"] + faults, show=True, timeout=300) + env = ["-e", "BASE_URL=http://lb:8080", + "-e", "FAULT_S3_URL=http://api-fault-s3:8080", + "-e", "FAULT_DB_URL=http://api-fault-db:8080", + "-e", "FAULT_CRASH_URL=http://api-fault-crash:8080"] + try: + for name in faults: + run(base + env + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name], + show=True, timeout=120) + run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_attachment_consistency.py", "-v"], show=True, timeout=600) + finally: + run(prof + ["rm", "-sf"] + faults, check=False, timeout=120) + elif which == "expiry": + prof = compose_cmd("--profile", "ttl") + run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300) + env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"] + try: + run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"], + show=True, timeout=120) + run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_expiry.py", "-v"], show=True, timeout=300) + finally: + run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120) + else: + raise SystemExit("usage: python scripts/ops.py in-network storage|expiry|consistency") + + +COMMANDS = { + "wait": cmd_wait, + "ps": cmd_ps, + "stop-one": cmd_stop_one, + "restore": cmd_restore, + "restart": cmd_restart, + "rolling-update": cmd_rolling_update, + "in-network": cmd_in_network, + "wait-s3": cmd_wait_s3, + "s3-check": cmd_s3_check, + "reconcile": cmd_reconcile, + "api-fs": cmd_api_fs, +} + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: + print(__doc__) + sys.exit(2) + COMMANDS[sys.argv[1]](sys.argv[2:]) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..5429f025 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант restart.sh: API + PostgreSQL + Redis + S3 + LB, volumes сохраняются. +rem python tests\resilience.py --mode persistence --action-script scripts\restart.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py restart +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..921ed9c5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Hook для make test-persistence: перезапускает API (все реплики), PostgreSQL, +# Redis, S3 (SeaweedFS) и LB, сохраняя volumes pgdata и s3data. Никаких down -v / +# flush / повторного seed / пересоздания бакета. После рестарта S3 ждёт, пока +# реально работают запись и чтение (прогрев SeaweedFS 15–30 с), затем поднимает API. +# Возвращает 0, когда API снова healthy; готовность LB тест ждёт сам. +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py restart diff --git a/scripts/sql/explain_history.sql b/scripts/sql/explain_history.sql new file mode 100644 index 00000000..d2ee4130 --- /dev/null +++ b/scripts/sql/explain_history.sql @@ -0,0 +1,93 @@ +-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на >= 10 000 сообщений. +-- Всё выполняется в ОДНОЙ транзакции и откатывается (ROLLBACK): синтетические +-- данные и временное удаление индекса не остаются в БД. +-- Запуск (compose): +-- docker compose exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 \ +-- < scripts/sql/explain_history.sql > evidence/explain_history.txt +-- Внимание: DROP INDEX внутри транзакции берёт эксклюзивную блокировку messages +-- до ROLLBACK (несколько секунд) — запускать на учебном стенде. + +\set ON_ERROR_STOP 1 +\timing off +SET TimeZone = 'UTC'; +-- Убираем мёртвые версии строк от прошлых запусков (VACUUM нельзя внутри транзакции). +VACUUM (ANALYZE) messages, chat_members, chats, users; +BEGIN; + +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-4000-8000-00000000e001', 'explain_alice', 'Explain', 'x'), + ('00000000-0000-4000-8000-00000000e002', 'explain_bob', 'Explain', 'x'); + +-- 1 большой чат (12 000 сообщений) + 40 чатов по 1 000 = 52 000 сообщений. +INSERT INTO chats (id, title, created_at) +SELECT CASE WHEN g = 0 THEN '00000000-0000-4000-8000-0000000c0000'::uuid ELSE gen_random_uuid() END, + 'explain-' || g, timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM generate_series(0, 40) AS g; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT c.id, u.id, u.pos +FROM chats c +CROSS JOIN (VALUES ('00000000-0000-4000-8000-00000000e001'::uuid, 0), + ('00000000-0000-4000-8000-00000000e002'::uuid, 1)) AS u(id, pos) +WHERE c.title LIKE 'explain-%'; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), c.id, + CASE WHEN g % 2 = 0 THEN '00000000-0000-4000-8000-00000000e001'::uuid + ELSE '00000000-0000-4000-8000-00000000e002'::uuid END, + 'synthetic message ' || g, + timestamptz '2026-01-02 00:00:00+00' + g * interval '1 millisecond' + (c.created_at - timestamptz '2026-01-01 00:00:00+00') +FROM chats c +CROSS JOIN LATERAL generate_series(1, CASE WHEN c.title = 'explain-0' THEN 12000 ELSE 1000 END) AS g +WHERE c.title LIKE 'explain-%'; + +ANALYZE users, chats, chat_members, messages; + +\echo '=== dataset' +SELECT count(*) AS total_messages, + count(*) FILTER (WHERE chat_id = '00000000-0000-4000-8000-0000000c0000') AS big_chat_messages +FROM messages; + +-- Позиция курсора после 10 000-го сообщения большого чата (как в next_cursor). +SELECT created_at AS cur_ts, id AS cur_id +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== 1. first page (no cursor), LIMIT 51 = limit+1' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 2. deep page: keyset cursor after message #10000 (query used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 3. for comparison: OFFSET pagination to the same page (NOT used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +OFFSET 10000 LIMIT 51; + +\echo '=== 4. same keyset query WITHOUT messages_chat_created_id_idx (index dropped inside the transaction)' +DROP INDEX messages_chat_created_id_idx; +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +ROLLBACK; +\echo '=== rolled back: synthetic data removed, index restored' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..1bd3049b --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант stop-one.sh: TARGET_INSTANCE_ID -> один контейнер api, остаётся остановленным. +rem python tests\resilience.py --mode failover --action-script scripts\stop-one.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py stop-one +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..a0b23e51 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Hook для make test-failover: останавливает ОДНУ API-реплику, которую тест передал +# в TARGET_INSTANCE_ID (= X-Instance-Id = hostname контейнера = короткий ID), и +# оставляет её остановленной. По умолчанию graceful (docker stop: SIGTERM -> +# осушение 3 с -> graceful shutdown); STOP_MODE=kill — «падение» (SIGKILL). +# БД, Redis, S3, LB и volumes не трогает. Восстановление: python3 scripts/ops.py restore +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py stop-one diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..120da440 --- /dev/null +++ b/src/config.py @@ -0,0 +1,187 @@ +"""Конфигурация из переменных окружения. + +Секреты (строки подключения с паролями) читаются из файлов *_FILE, +которые Compose монтирует как secrets в /run/secrets/. Для локального +запуска без Docker допускаются и прямые переменные DATABASE_URL / CACHE_URL. +""" +import os +from dataclasses import dataclass +from typing import Optional + + +def _read_secret(name: str) -> Optional[str]: + path = os.environ.get(name + "_FILE") + if path: + with open(path, "r", encoding="utf-8") as handle: + value = handle.read().strip() + return value or None + value = os.environ.get(name, "").strip() + return value or None + + +def _int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return int(raw) + + +def _float(name: str, default: float) -> float: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return float(raw) + + +def _bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class Settings: + app_env: str + app_version: str + instance_id: str + database_url: str + cache_url: Optional[str] + session_store: str + session_ttl_seconds: int + session_cache_ttl_seconds: int + db_pool_min: int + db_pool_max: int + db_pool_timeout: float + db_connect_timeout: int + db_statement_timeout_ms: int + cache_timeout_seconds: float + cache_retry_after_seconds: float + migrate_wait_seconds: int + run_migrations_on_start: bool + idempotency_ttl_seconds: int + idempotency_cleanup_interval_seconds: int + ready_db_timeout_seconds: float + drain_seconds: float + graceful_timeout_seconds: int + keep_alive_seconds: int + password_hash_concurrency: int + # lab4: S3 и вложения + s3_endpoint: Optional[str] + s3_region: str + s3_bucket: str + s3_access_key_id: Optional[str] + s3_secret_access_key: Optional[str] + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + s3_max_attempts: int + s3_download_deadline_seconds: float + s3_upload_deadline_seconds: float + s3_circuit_open_seconds: float + max_upload_bytes: int + max_image_pixels: int + max_concurrent_uploads: int + upload_queue_timeout_seconds: float + upload_read_timeout_seconds: float + image_decode_concurrency: int + attachment_gc_interval_seconds: int + attachment_gc_grace_seconds: int + orphan_sweep_interval_seconds: int + fault_injection: str + + +# Жёсткие пределы контракта lab4; конфигурация может их только ужесточить. +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 +# Grace period сборщика должен быть заметно больше самой долгой загрузки +# (чтение тела + S3 PUT с повтором), см. REPORT.md. +MIN_GC_GRACE_SECONDS = 300 + + +def load_settings() -> Settings: + database_url = _read_secret("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE must be set") + + ttl = _int("SESSION_TTL_SECONDS", 3600) + if ttl < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") + + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() + # Источник истины для сессий в этой реализации всегда PostgreSQL; + # Redis (CACHE_URL) — только кэш проверок токена. + if session_store != "postgres": + raise RuntimeError("only SESSION_STORE=postgres is supported") + + idem_ttl = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + if idem_ttl < 86400: + # Контракт: повтор в течение 24 часов возвращает тот же результат. + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be >= 86400") + + # X-Instance-Id: по умолчанию hostname контейнера (= короткий ID контейнера + # Docker), стабилен в пределах жизни процесса. INSTANCE_ID — для запуска без Docker. + from .runtime import INSTANCE_ID as instance_id + + max_upload = _int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES) + max_pixels = _int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS) + if not 1 <= max_upload <= CONTRACT_MAX_UPLOAD_BYTES: + raise RuntimeError("MAX_UPLOAD_BYTES must be within 1..10485760") + if not 1 <= max_pixels <= CONTRACT_MAX_IMAGE_PIXELS: + raise RuntimeError("MAX_IMAGE_PIXELS must be within 1..40000000") + gc_grace = _int("ATTACHMENT_GC_GRACE_SECONDS", 900) + # Меньше минимума — только для явных тестов сверки (ALLOW_SHORT_GC_GRACE=true). + if gc_grace < MIN_GC_GRACE_SECONDS and not _bool("ALLOW_SHORT_GC_GRACE", False): + raise RuntimeError("ATTACHMENT_GC_GRACE_SECONDS must be >= %d" % MIN_GC_GRACE_SECONDS) + + return Settings( + app_env=os.environ.get("APP_ENV", "lab"), + app_version=os.environ.get("APP_VERSION", "lab4"), + instance_id=instance_id, + database_url=database_url, + cache_url=_read_secret("CACHE_URL"), + session_store=session_store, + session_ttl_seconds=ttl, + session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60), + db_pool_min=_int("DB_POOL_MIN", 1), + db_pool_max=_int("DB_POOL_MAX", 10), + db_pool_timeout=_float("DB_POOL_TIMEOUT_SECONDS", 3.0), + db_connect_timeout=_int("DB_CONNECT_TIMEOUT_SECONDS", 3), + db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.25), + cache_retry_after_seconds=_float("CACHE_RETRY_AFTER_SECONDS", 5.0), + migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60), + # lab3: миграции выполняет отдельный one-shot сервис `migrate`, + # реплики API их не запускают (нет гонки N реплик за схему). + run_migrations_on_start=_bool("RUN_MIGRATIONS_ON_START", False), + idempotency_ttl_seconds=idem_ttl, + idempotency_cleanup_interval_seconds=_int("IDEMPOTENCY_CLEANUP_INTERVAL_SECONDS", 300), + ready_db_timeout_seconds=_float("READY_DB_TIMEOUT_SECONDS", 1.0), + drain_seconds=_float("DRAIN_SECONDS", 3.0), + graceful_timeout_seconds=_int("GRACEFUL_TIMEOUT_SECONDS", 10), + keep_alive_seconds=_int("KEEP_ALIVE_SECONDS", 30), + password_hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 4), + s3_endpoint=os.environ.get("S3_ENDPOINT", "").strip() or None, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments").strip(), + s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID"), + s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY"), + s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 2.0), + s3_read_timeout_seconds=_float("S3_READ_TIMEOUT_SECONDS", 10.0), + s3_max_attempts=_int("S3_MAX_ATTEMPTS", 2), + # Полный бюджет вызова S3 из запроса (DNS + connect + повторы). Скачивание + # обязано уложиться в timeout server HAProxy (10 с), загрузка — 10 с (PUT 10 MiB локально ~0.1 с). + s3_download_deadline_seconds=_float("S3_DOWNLOAD_DEADLINE_SECONDS", 5.0), + s3_upload_deadline_seconds=_float("S3_UPLOAD_DEADLINE_SECONDS", 10.0), + s3_circuit_open_seconds=_float("S3_CIRCUIT_OPEN_SECONDS", 5.0), + max_upload_bytes=max_upload, + max_image_pixels=max_pixels, + max_concurrent_uploads=max(1, _int("MAX_CONCURRENT_UPLOADS", 4)), + upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0), + upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 60.0), + image_decode_concurrency=max(1, _int("IMAGE_DECODE_CONCURRENCY", 2)), + attachment_gc_interval_seconds=_int("ATTACHMENT_GC_INTERVAL_SECONDS", 300), + attachment_gc_grace_seconds=gc_grace, + orphan_sweep_interval_seconds=_int("ORPHAN_SWEEP_INTERVAL_SECONDS", 3600), + # Только для тестового экземпляра api-faults (профиль faults), см. REPORT.md. + fault_injection=os.environ.get("FAULT_INJECTION", "").strip().lower(), + ) diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..d8d413b3 --- /dev/null +++ b/src/db.py @@ -0,0 +1,130 @@ +"""Пул соединений PostgreSQL и версионируемые миграции.""" +import logging +import time +from pathlib import Path +from typing import Optional + +import psycopg +from psycopg_pool import ConnectionPool + +from .config import Settings + +log = logging.getLogger("messenger.db") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + +# Произвольная константа: один процесс за раз применяет миграции, +# даже если одновременно стартуют несколько реплик API (lab3). +MIGRATION_LOCK_ID = 2026_0002 + + +def conninfo_kwargs(settings: Settings) -> dict: + return { + "connect_timeout": settings.db_connect_timeout, + # Все timestamptz приходят в UTC; любой запрос ограничен по времени. + "options": "-c TimeZone=UTC -c statement_timeout=%d" + % settings.db_statement_timeout_ms, + "application_name": "messenger-api", + } + + +def create_pool(settings: Settings) -> ConnectionPool: + pool = ConnectionPool( + conninfo=settings.database_url, + kwargs=conninfo_kwargs(settings), + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + # Сколько запрос ждёт свободное соединение, прежде чем получить 503. + timeout=settings.db_pool_timeout, + # Битые после рестарта БД соединения отбрасываются при выдаче из пула. + check=ConnectionPool.check_connection, + max_idle=300, + reconnect_timeout=3600, + open=False, + name="messenger", + ) + # wait=False: процесс поднимается, даже если БД ещё/уже недоступна; + # readiness это покажет, а запросы получат 503. + pool.open(wait=False) + return pool + + +def _migration_files(): + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + if not files: + raise RuntimeError("no migrations found in %s" % MIGRATIONS_DIR) + return files + + +def required_schema_version() -> str: + """Последняя миграция, которую знает этот образ. + + Реплика готова (readiness) только если она уже применена: миграции + запускает отдельный сервис `migrate`, а не сама реплика. + """ + return _migration_files()[-1].stem + + +def schema_has_version(pool: ConnectionPool, version: str, timeout: float) -> bool: + with pool.connection(timeout=timeout) as conn: + row = conn.execute( + "SELECT to_regclass('schema_migrations') IS NOT NULL" + ).fetchone() + if not row[0]: + return False + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = %s)", + (version,), + ).fetchone()[0] + + +def apply_migrations(settings: Settings) -> list: + """Применяет недостающие миграции; возвращает список применённых версий.""" + applied_now = [] + with psycopg.connect( + settings.database_url, autocommit=True, **conninfo_kwargs(settings) + ) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + done = { + row[0] + for row in conn.execute("SELECT version FROM schema_migrations") + } + for path in _migration_files(): + version = path.stem + if version in done: + continue + sql = path.read_text(encoding="utf-8") + # DDL и отметка о версии — одна транзакция: либо всё, либо ничего. + with conn.transaction(): + conn.execute(sql) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + applied_now.append(version) + log.info("applied migration %s", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,)) + return applied_now + + +def wait_and_migrate(settings: Settings) -> list: + deadline = time.monotonic() + settings.migrate_wait_seconds + last_error: Optional[Exception] = None + while time.monotonic() < deadline: + try: + return apply_migrations(settings) + except psycopg.OperationalError as exc: + last_error = exc + log.warning("database not ready for migrations, retrying") + time.sleep(1) + raise RuntimeError("database unavailable for migrations") from last_error diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..569a6555 --- /dev/null +++ b/src/images.py @@ -0,0 +1,284 @@ +"""Проверка загружаемых изображений по содержимому, с ограниченной памятью. + +Порядок для каждого файла (лимит байт уже применён при чтении тела): +1. Сигнатура: PNG (\\x89PNG\\r\\n\\x1a\\n) или JPEG (FF D8 FF). Иное -> 415. +2. Сигнатура должна совпасть с заявленным Content-Type (подмена MIME -> 415). +3. Размеры читаются из заголовка (PNG IHDR / JPEG SOFn) ДО распаковки пикселей; + width*height > MAX_IMAGE_PIXELS -> 413. Распаковка не начинается. +4. Целостность без полного декодирования в память: + * PNG: обход всех чанков с проверкой CRC, порядка IHDR..IDAT..IEND и + потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»: + объём распакованных данных должен в точности равняться ожидаемому + (height*(1+rowbytes), с учётом Adam7). Память — O(64 KiB) при любом + размере картинки; «zip-бомба» обрывается, как только вывод превысил ожидание. + * JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI). + Baseline дополнительно декодируется Pillow в режиме draft (масштаб 1/8, + ~5 MiB даже для 40 Мп). Progressive Pillow не декодируется: libjpeg держит + буфер коэффициентов всего кадра (~120 MiB на 40 Мп, замер в REPORT.md), + для него — только структурная проверка. +5. Pillow открывает файл независимым парсером: формат и размеры должны совпасть. +""" +import io +import struct +import zlib +from typing import Tuple + +from PIL import Image, ImageFile + +# Свою проверку пикселей делаем сами (шаг 3) — до вызова Pillow. +Image.MAX_IMAGE_PIXELS = None +ImageFile.LOAD_TRUNCATED_IMAGES = False + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +JPEG_SIG = b"\xff\xd8\xff" +INFLATE_CHUNK = 64 * 1024 + +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# SOF0..SOF15 без DHT(C4), JPG(C8), DAC(CC) +SOF_MARKERS = set(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} +PROGRESSIVE_SOF = {0xC2, 0xC6, 0xCA, 0xCE} + + +class ImageRejected(Exception): + def __init__(self, status: int, code: str, message: str) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def unsupported(message: str = "Unsupported or corrupted image") -> ImageRejected: + return ImageRejected(415, "unsupported_media_type", message) + + +def too_many_pixels() -> ImageRejected: + return ImageRejected(413, "payload_too_large", "Image has too many pixels") + + +def sniff(data) -> str: + head = bytes(data[:8]) + if head.startswith(PNG_SIG): + return "image/png" + if head.startswith(JPEG_SIG): + return "image/jpeg" + return "" + + +# ------------------------------------------------------------------- PNG + +def _png_expected_raw(width: int, height: int, depth: int, color: int, interlace: int) -> int: + bpp = depth * PNG_CHANNELS[color] + + def rows(w: int, h: int) -> int: + if w == 0 or h == 0: + return 0 + return h * (1 + (w * bpp + 7) // 8) + + if interlace == 0: + return rows(width, height) + total = 0 + for x0, y0, dx, dy in ADAM7: + pw = (width - x0 + dx - 1) // dx if width > x0 else 0 + ph = (height - y0 + dy - 1) // dy if height > y0 else 0 + total += rows(pw, ph) + return total + + +def inspect_png(data, max_pixels: int) -> Tuple[int, int, bool]: + view = memoryview(data) + size = len(view) + pos = 8 + width = height = 0 + expected = 0 + inflater = None + produced = 0 + seen_ihdr = seen_plte = idat_done = False + idat_started = False + color = 0 + + while True: + if pos + 12 > size: + raise unsupported("Truncated PNG") + length, ctype = struct.unpack(">I4s", view[pos:pos + 8]) + if length > 0x7FFFFFFF or pos + 12 + length > size: + raise unsupported("Truncated PNG") + if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype): + raise unsupported("Invalid PNG chunk") + body = view[pos + 8:pos + 8 + length] + (crc,) = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length]) + if zlib.crc32(body, zlib.crc32(ctype)) & 0xFFFFFFFF != crc: + raise unsupported("PNG CRC mismatch") + pos += 12 + length + + if not seen_ihdr: + if ctype != b"IHDR" or length != 13: + raise unsupported("PNG must start with IHDR") + width, height, depth, color, comp, flt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF): + raise unsupported("Invalid PNG dimensions") + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or comp or flt or interlace > 1: + raise unsupported("Invalid PNG header") + # Лимит пикселей — до любой распаковки. + if width * height > max_pixels: + raise too_many_pixels() + expected = _png_expected_raw(width, height, depth, color, interlace) + seen_ihdr = True + continue + + if ctype == b"IHDR": + raise unsupported("Duplicate IHDR") + if ctype == b"PLTE": + if idat_started: + raise unsupported("PLTE after IDAT") + seen_plte = True + elif ctype == b"IDAT": + if idat_done: + raise unsupported("Non-contiguous IDAT") + if color == 3 and not seen_plte: + raise unsupported("Missing PLTE") + if not idat_started: + idat_started = True + inflater = zlib.decompressobj() + produced += _inflate(inflater, body, expected - produced) + elif ctype == b"IEND": + if length != 0 or not idat_started: + raise unsupported("Invalid PNG end") + produced += _inflate(inflater, b"", expected - produced, final=True) + if not inflater.eof or produced != expected: + raise unsupported("Corrupted PNG image data") + # Данные после IEND не используются декодерами; допускаем. + return width, height, bool(interlace) + else: + if idat_started: + idat_done = True + if ctype[0] & 0x20 == 0: + # Неизвестный КРИТИЧЕСКИЙ чанк (заглавная первая буква): декодер обязан + # отказаться от такого файла. Вспомогательные (строчная буква) пропускаем. + raise unsupported("Unknown critical PNG chunk") + + +def _inflate(inflater, data, budget: int, final: bool = False) -> int: + """Распаковать data, не держа в памяти больше ~2 x INFLATE_CHUNK; вернуть объём вывода. + + Вход подаётся срезами memoryview по INFLATE_CHUNK: unconsumed_tail — копия + остатка входа, поэтому подача всего IDAT целиком давала бы копию размером с + файл на каждом шаге (замер: пик 2x размера файла и квадратичное копирование). + """ + produced = 0 + view = memoryview(data) + try: + for start in range(0, len(view), INFLATE_CHUNK): + buf = view[start:start + INFLATE_CHUNK] + while True: + out = inflater.decompress(buf, INFLATE_CHUNK) + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + buf = inflater.unconsumed_tail + if not buf: + break + if final: + while True: + out = inflater.flush(INFLATE_CHUNK) if not inflater.eof else b"" + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + if not out: + break + except zlib.error: + raise unsupported("Corrupted PNG image data") + return produced + + +# ------------------------------------------------------------------ JPEG + +def inspect_jpeg(data, max_pixels: int) -> Tuple[int, int, bool]: + raw = data if isinstance(data, (bytes, bytearray)) else bytes(data) + size = len(raw) + pos = 2 + width = height = 0 + progressive = False + seen_sof = False + while True: + # Маркер: один или несколько 0xFF, затем код. + if pos >= size or raw[pos] != 0xFF: + raise unsupported("Corrupted JPEG structure") + while pos < size and raw[pos] == 0xFF: + pos += 1 + if pos >= size: + raise unsupported("Truncated JPEG") + marker = raw[pos] + pos += 1 + if marker == 0xD9: # EOI + if not seen_sof: + raise unsupported("JPEG without frame") + return width, height, progressive + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if marker == 0xD8 or marker == 0x00: + raise unsupported("Corrupted JPEG structure") + if pos + 2 > size: + raise unsupported("Truncated JPEG") + (length,) = struct.unpack(">H", raw[pos:pos + 2]) + if length < 2 or pos + length > size: + raise unsupported("Truncated JPEG") + segment = raw[pos + 2:pos + length] + pos += length + if marker in SOF_MARKERS: + if seen_sof or len(segment) < 6: + raise unsupported("Invalid JPEG frame") + _precision, height, width, ncomp = struct.unpack(">BHHB", segment[:6]) + if width == 0 or height == 0 or ncomp not in (1, 3, 4) or len(segment) < 6 + 3 * ncomp: + raise unsupported("Invalid JPEG frame") + if width * height > max_pixels: + raise too_many_pixels() + progressive = marker in PROGRESSIVE_SOF + seen_sof = True + elif marker == 0xDA: # SOS: дальше энтропийные данные до следующего маркера + if not seen_sof: + raise unsupported("JPEG scan before frame") + while True: + nxt = raw.find(b"\xff", pos) + if nxt < 0 or nxt + 1 >= size: + raise unsupported("Truncated JPEG") + code = raw[nxt + 1] + if code == 0x00 or 0xD0 <= code <= 0xD7 or code == 0xFF: + pos = nxt + 1 if code == 0xFF else nxt + 2 + continue + pos = nxt + break + + +# -------------------------------------------------------------- common + +def inspect(data, declared_type: str, max_pixels: int) -> Tuple[str, int, int]: + """Вернуть (content_type, width, height) или бросить ImageRejected.""" + detected = sniff(data) + if not detected: + raise unsupported("Only PNG and JPEG images are accepted") + if detected != declared_type: + raise unsupported("Content-Type does not match image content") + if detected == "image/png": + width, height, _ = inspect_png(data, max_pixels) + pil_format = "PNG" + decode = False # целостность IDAT уже проверена потоковой распаковкой + else: + width, height, progressive = inspect_jpeg(data, max_pixels) + pil_format = "JPEG" + decode = not progressive + # Независимый парсер Pillow: только заголовок (open ленивый), затем при + # необходимости декодирование уменьшенной копии. + try: + with Image.open(io.BytesIO(data)) as img: + if img.format != pil_format or img.size != (width, height): + raise unsupported("Inconsistent image header") + if decode: + img.draft(None, (max(1, width // 8), max(1, height // 8))) + img.load() + except ImageRejected: + raise + except (OSError, SyntaxError, ValueError, struct.error, Image.DecompressionBombError, EOFError): + raise unsupported("Corrupted image") + return detected, width, height diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..9f98591c --- /dev/null +++ b/src/main.py @@ -0,0 +1,812 @@ +import asyncio +import base64 +import hashlib +import json +import logging +import os +import re +import threading +import unicodedata +import uuid as uuid_mod +from contextlib import asynccontextmanager +from datetime import datetime +from typing import Callable, Dict, Optional, Tuple +from uuid import UUID, uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import JSONResponse, StreamingResponse +from psycopg_pool import PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect +from starlette.middleware.base import BaseHTTPMiddleware + +from .config import load_settings +from .db import create_pool, required_schema_version, schema_has_version, wait_and_migrate +from .images import ImageRejected, inspect as inspect_image +from .models import ( + Attachment, + Chat, + ChatCreate, + ChatPage, + Message, + MessageCreate, + MessagePage, + Session, + User, + UserCreate, +) +from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable +from .reconcile import OBJECT_PREFIX, gc_loop +from .runtime import DRAIN, INSTANCE_ID +from .security import ( + configure_kdf_concurrency, + hash_password, + parse_basic, + parse_bearer, + verify_password, +) +from .sessions import SessionCache, SessionService +from .storage import ( + AttachmentNotFound, + IdempotencyConflict, + IdempotencyRace, + NotChatMember, + Storage, + UnknownMember, + UsernameTaken, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.api") + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +# Контракт lab3: Idempotency-Key — ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def _idempotency_cleanup_loop(storage: Storage, interval: float, stop: threading.Event) -> None: + """Фоновая очистка истёкших ключей. Advisory lock — одна реплика за раз.""" + while not stop.wait(interval): + try: + deleted = storage.delete_expired_idempotency_keys() + if deleted: + log.info("deleted %d expired idempotency keys", deleted) + except Exception as exc: # БД недоступна — попробуем в следующий раз + log.warning("idempotency cleanup skipped: %s", type(exc).__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = load_settings() + if settings.run_migrations_on_start: + # Только для локальной разработки. В compose миграции выполняет + # отдельный one-shot сервис `migrate` до старта реплик. + applied = wait_and_migrate(settings) + log.info("migrations applied at startup: %s", applied or "none") + configure_kdf_concurrency(settings.password_hash_concurrency) + pool = create_pool(settings) + storage = Storage(pool) + cache = SessionCache(settings) + app.state.settings = settings + app.state.storage = storage + app.state.cache = cache + app.state.sessions = SessionService(storage, cache, settings) + app.state.required_schema = required_schema_version() + app.state.schema_ok = False + stop_cleanup = threading.Event() + cleanup = threading.Thread( + target=_idempotency_cleanup_loop, + args=(storage, settings.idempotency_cleanup_interval_seconds, stop_cleanup), + name="idempotency-cleanup", + daemon=True, + ) + cleanup.start() + # lab4: S3 и ограничители ресурсов загрузок (на реплику). + app.state.objects = ObjectStore(settings) if ObjectStore.configured(settings) else None + app.state.upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads) + app.state.decode_slots = threading.BoundedSemaphore(settings.image_decode_concurrency) + if app.state.objects is not None: + threading.Thread( + target=gc_loop, + args=(pool, app.state.objects, settings, stop_cleanup), + name="attachment-gc", + daemon=True, + ).start() + else: + log.warning("S3 is not configured: attachment endpoints return 503") + log.info( + "attachments: s3=%s bucket=%s max_bytes=%d max_pixels=%d uploads<=%d decode<=%d gc_grace=%ss fault=%s", + "on" if app.state.objects is not None else "off", + settings.s3_bucket, + settings.max_upload_bytes, + settings.max_image_pixels, + settings.max_concurrent_uploads, + settings.image_decode_concurrency, + settings.attachment_gc_grace_seconds, + settings.fault_injection or "none", + ) + log.info( + "started: instance=%s version=%s session_ttl=%ss cache=%s pool=%s..%s schema>=%s", + INSTANCE_ID, + settings.app_version, + settings.session_ttl_seconds, + "redis" if cache.client is not None else "disabled", + settings.db_pool_min, + settings.db_pool_max, + app.state.required_schema, + ) + try: + yield + finally: + log.info("shutdown: instance=%s closing pool", INSTANCE_ID) + stop_cleanup.set() + pool.close() + if cache.client is not None: + cache.client.close() + + +app = FastAPI( + title="MTUSI Messenger (lab4: S3 attachments)", + version="4.0.0", + lifespan=lifespan, +) + + +# Request ID + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable): + incoming = request.headers.get("X-Request-Id", "") + request_id = incoming if REQUEST_ID_RE.match(incoming) else str(uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + if DRAIN.active: + # Во время осушения не держим keep-alive: следующий запрос клиент/LB + # откроет новым соединением — уже к другой реплике. + response.headers["Connection"] = "close" + return response + + +app.add_middleware(RequestIdMiddleware) + + +# errors + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", "Authentication required", {"WWW-Authenticate": challenge}) + + +def bad_request(message: str = "Invalid request") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Not found") + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, + headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) or str(uuid4()) + all_headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID} + all_headers.update(headers or {}) + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=all_headers, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError) -> JSONResponse: + return error_response(request, exc.status_code, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Invalid request") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + status_to_code = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + } + status_code = exc.status_code if exc.status_code in status_to_code else 500 + code = status_to_code.get(status_code, "internal_error") + message = str(exc.detail) if status_code != 500 else "Internal server error" + return error_response(request, status_code, code, message) + + +async def dependency_unavailable(request: Request, exc: Exception) -> JSONResponse: + # Наружу — только общий 503, без SQL, адресов и текста драйвера. + log.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "Service temporarily unavailable") + + +app.add_exception_handler(psycopg.OperationalError, dependency_unavailable) +app.add_exception_handler(psycopg.InterfaceError, dependency_unavailable) +app.add_exception_handler(PoolTimeout, dependency_unavailable) + + +@app.exception_handler(Exception) +async def unexpected_exception_handler(request: Request, exc: Exception) -> JSONResponse: + log.exception("unexpected error") + return error_response(request, 500, "internal_error", "Internal server error") + + +# helpers + +def require_json_content_type(request: Request) -> None: + content_type = request.headers.get("content-type", "") + if not content_type.lower().startswith("application/json"): + raise ApiError(415, "unsupported_media_type", "Unsupported media type") + + +def ensure_not_blank(value: str) -> None: + if not value.strip(): + raise bad_request() + + +def storage_of(request: Request) -> Storage: + return request.app.state.storage + + +def current_user(request: Request) -> User: + """Только Authorization: Bearer. Basic и X-User-Id здесь не дают доступа.""" + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise unauthorized() + user = request.app.state.sessions.resolve(token) + if user is None: + raise unauthorized() + request.state.token = token + return user + + +def encode_cursor(resource_id: UUID, created_at: datetime, item_id: UUID) -> str: + payload = { + "resource_id": str(resource_id), + "created_at": created_at.isoformat(), + "item_id": str(item_id), + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: Optional[str], expected_resource_id: UUID) -> Optional[Tuple[datetime, UUID]]: + if cursor is None: + return None + try: + padding = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + padding) + payload = json.loads(raw.decode("utf-8")) + resource_id = UUID(payload["resource_id"]) + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("invalid_cursor") + except (ValueError, KeyError, TypeError, AttributeError, json.JSONDecodeError): + raise bad_request("Invalid cursor") + if resource_id != expected_resource_id: + raise bad_request("Invalid cursor") + return created_at, item_id + + +# health + +@app.get("/health/live", response_model=Dict[str, str]) +def liveness() -> Dict[str, str]: + return {"status": "ok"} + + +@app.get("/health/ready", response_model=Dict[str, str]) +def readiness(request: Request) -> Dict[str, str]: + """Готовность к обязательному синхронному пути. + + * SIGTERM получен (фаза осушения) -> 503: LB выводит реплику из ротации; + * PostgreSQL недоступна за READY_DB_TIMEOUT_SECONDS -> 503; + * схема ещё не мигрирована до версии, которую знает образ -> 503 + (миграции выполняет отдельный сервис `migrate`). + Redis не критичен: это кэш с обходом (см. REPORT.md). + """ + if DRAIN.active: + raise ApiError(503, "unavailable", "Instance is shutting down") + state = request.app.state + timeout = state.settings.ready_db_timeout_seconds + try: + if state.schema_ok: + with state.storage.pool.connection(timeout=timeout) as conn: + conn.execute("SELECT 1") + elif schema_has_version(state.storage.pool, state.required_schema, timeout): + state.schema_ok = True + else: + raise ApiError(503, "unavailable", "Database schema is not migrated") + except (psycopg.Error, PoolTimeout): + raise ApiError(503, "unavailable", "Database is not available") + return {"status": "ok"} + + +# users + +@app.post("/api/v1/users", response_model=User, status_code=201) +def register_user(request: Request, data: UserCreate) -> User: + require_json_content_type(request) + ensure_not_blank(data.display_name) + ensure_not_blank(data.password) + password_hash = hash_password(data.password) + try: + return storage_of(request).create_user(data.username, data.display_name, password_hash) + except UsernameTaken: + raise ApiError(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(user: User = Depends(current_user)) -> User: + return user + + +# sessions + +async def require_empty_body(request: Request) -> None: + """Вход — только заголовок Authorization: Basic, без тела (контракт). + + Любые байты в теле (JSON или нет) — 400. Проверка выполняется до + дорогой проверки пароля и до создания сессии. + """ + if await request.body(): + raise bad_request("Request body is not allowed") + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201, + dependencies=[Depends(require_empty_body)]) +def create_session(request: Request) -> Session: + credentials = parse_basic(request.headers.get("Authorization")) + if credentials is None: + raise unauthorized(BASIC_CHALLENGE) + username, password = credentials + found = storage_of(request).get_credentials(username) + user, password_hash = found if found else (None, None) + # verify_password выполняется и для несуществующего username (с хешем-пустышкой). + if not verify_password(password_hash, password) or user is None: + raise unauthorized(BASIC_CHALLENGE) + token, expires_at = request.app.state.sessions.create(user) + return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_current_session(request: Request, user: User = Depends(current_user)) -> Response: + if not request.app.state.sessions.revoke(request.state.token): + raise unauthorized() + return Response(status_code=204) + + +# chats + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, data: ChatCreate, user: User = Depends(current_user)) -> Chat: + require_json_content_type(request) + ensure_not_blank(data.title) + if len(data.member_ids) != len(set(data.member_ids)): + raise bad_request("Duplicate member ids") + if user.id not in data.member_ids: + raise bad_request("Caller must be a chat member") + try: + return storage_of(request).create_chat(data.title, data.member_ids) + except UnknownMember: + raise bad_request("Unknown member") + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> ChatPage: + after = decode_cursor(cursor, user.id) + items, has_more = storage_of(request).list_user_chats(user.id, limit, after) + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(user.id, items[-1].created_at, items[-1].id) + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID, user: User = Depends(current_user)) -> Chat: + chat = storage_of(request).get_chat_for_member(chat_id, user.id) + if chat is None: + raise not_found() + return chat + + +# messages + +def request_fingerprint(data: MessageCreate) -> bytes: + """SHA-256 канонического JSON тела. + + Канонизация: проверенная модель -> JSON с сортировкой ключей и без пробелов. + Поэтому порядок ключей и форматирование исходного JSON не влияют на hash. + lab4: attachment_ids имеет значение по умолчанию [], поэтому отсутствующее + поле и явный [] дают одинаковый hash (контракт). Порядок id значим. + """ + canonical = json.dumps( + data.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +def create_message( + request: Request, + response: Response, + chat_id: UUID, + data: MessageCreate, + user: User = Depends(current_user), +) -> Message: + # Порядок: аутентификация (dependency) -> ключ -> тело -> доступ к чату. + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad_request("Missing or invalid Idempotency-Key") + require_json_content_type(request) + # lab4: пустой/пробельный текст допустим только вместе с вложениями. + if not data.attachment_ids: + ensure_not_blank(data.text) + if len(set(data.attachment_ids)) != len(data.attachment_ids): + raise bad_request("Duplicate attachment ids") + settings = request.app.state.settings + try: + message, replayed = storage_of(request).create_message( + chat_id, + user.id, + data.text, + key, + request_fingerprint(data), + settings.idempotency_ttl_seconds, + data.attachment_ids, + ) + except (NotChatMember, AttachmentNotFound): + # Чужое, неизвестное или ещё не готовое вложение — как и чужой чат: 404. + raise not_found() + except IdempotencyConflict: + raise ApiError(409, "conflict", "Idempotency-Key was used with a different request") + except IdempotencyRace: + raise ApiError(503, "unavailable", "Please retry with the same Idempotency-Key") + if replayed: + # Информационный заголовок (не часть контракта): ответ — повтор. + response.headers["Idempotency-Replayed"] = "true" + return message + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages( + request: Request, + chat_id: UUID, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> MessagePage: + after = decode_cursor(cursor, chat_id) + result = storage_of(request).list_messages(chat_id, user.id, limit, after) + if result is None: + raise not_found() + items, has_more = result + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(chat_id, items[-1].created_at, items[-1].id) + return MessagePage(items=items, next_cursor=next_cursor) + + +# attachments (lab4) + +ALLOWED_UPLOAD_TYPES = ("image/png", "image/jpeg") +UPLOAD_CHUNK = 64 * 1024 +KNOWN_VARIANTS = ("thumbnail", "crop", "optimized") + + +async def s3_call(store: ObjectStore, deadline: float, fn, *args): + """Вызов S3 из запроса с жёстким общим дедлайном. + + connect/read timeout boto3 не покрывают DNS: в Docker остановленный + контейнер s3 пропадает из DNS, и getaddrinfo висит секундами (замер на + Docker Desktop: 10–13 с, у скачивания это давало 504 от HAProxy). Дедлайн + ограничивает всё вместе; по истечении — 503 и открытый circuit breaker. + Поток с зависшим вызовом доработает в фоне; для PUT это может дать + объект-сироту, его уберёт сборщик (src/reconcile.py, проход B). + """ + try: + return await asyncio.wait_for(run_in_threadpool(fn, *args), timeout=deadline) + except asyncio.TimeoutError: + store.breaker.failure() + log.warning("s3 call exceeded %.1fs deadline", deadline) + raise ObjectStoreUnavailable() + + +def objects_of(request: Request) -> ObjectStore: + store = request.app.state.objects + if store is None: + raise ApiError(503, "unavailable", "Attachment storage is not available") + return store + + +def s3_unavailable() -> ApiError: + return ApiError(503, "unavailable", "Attachment storage is temporarily unavailable", + {"Retry-After": "2"}) + + +def validate_filename(name: Optional[str]) -> str: + """Отображаемое имя: 1–255 code points, без /, \\, NUL и управляющих символов. + + Имя никогда не становится путём или ключом S3; ограничения защищают + клиентов, которые покажут/сохранят файл под этим именем. + """ + if name is None or not name.strip() or len(name) > 255: + raise bad_request("Invalid filename") + if name in (".", ".."): + raise bad_request("Invalid filename") + for ch in name: + if ch in "/\\\x00" or unicodedata.category(ch) in ("Cc", "Cs") \ + or ch in "

‮‭‪‫‬⁦⁧⁨⁩": + raise bad_request("Invalid filename") + return name + + +def upload_media_type(request: Request) -> str: + raw = request.headers.get("content-type", "") + media = raw.split(";", 1)[0].strip().lower() + if media not in ALLOWED_UPLOAD_TYPES: + raise ApiError(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted") + return media + + +def declared_length(request: Request) -> Optional[int]: + raw = request.headers.get("content-length") + if raw is None: + return None + if not raw.strip().isdigit(): + raise bad_request("Invalid Content-Length") + return int(raw) + + +async def drain(request: Request, limit: int) -> None: + """Дочитать и выбросить тело (не больше limit байт), чтобы клиент увидел 413, + а не обрыв соединения посреди отправки. Память — O(chunk).""" + seen = 0 + try: + async for chunk in request.stream(): + seen += len(chunk) + if seen > limit: + break + except ClientDisconnect: + pass + + +async def read_limited_body(request: Request, limit: int): + """Потоковое чтение тела кусками; не больше limit байт в памяти. + + Возвращает (bytes, md5, sha256). Превышение лимита -> 413 сразу, как только + счётчик перешёл limit (остаток тела отбрасывается, не накапливается). + """ + chunks = [] + size = 0 + md5 = hashlib.md5(usedforsecurity=False) + sha = hashlib.sha256() + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > limit: + chunks.clear() + await drain(request, limit) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % limit) + chunks.append(chunk) + md5.update(chunk) + sha.update(chunk) + # Одна склейка в bytes точного размера; список кусков сразу освобождается. + # Дальше bytes не копируются: BytesIO(bytes) и memoryview разделяют буфер. + data = b"".join(chunks) + chunks.clear() + return data, md5.digest(), sha.hexdigest() + + +def _validate_image(request: Request, data: bytes, declared: str): + settings = request.app.state.settings + with request.app.state.decode_slots: # потолок одновременных разборов на реплику + return inspect_image(data, declared, settings.max_image_pixels) + + +def _inject(settings, point: str) -> None: + """Внедрение отказов — ТОЛЬКО для экземпляра api-faults (профиль compose faults).""" + fault = settings.fault_injection + if not fault: + return + if fault == "crash_after_s3_put" and point == "after_put": + log.error("FAULT_INJECTION: process exit between S3 PUT and DB COMMIT") + os._exit(70) + if fault == "db_error_after_s3_put" and point == "after_put": + raise psycopg.OperationalError("injected: database failure after S3 PUT") + if fault == "s3_error_on_put" and point == "before_put": + raise ObjectStoreUnavailable() + + +@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: Optional[str] = Query(default=None), + user: User = Depends(current_user), +) -> Attachment: + """Синхронная загрузка оригинала: 201 только после S3 PUT и COMMIT ready. + + Порядок проверок: аутентификация -> filename (400) -> Content-Type (415) -> + Content-Length (413 без чтения тела) -> членство в чате (404) -> слот + загрузки (503) -> потоковое чтение с лимитом (413) -> содержимое (415/413) + -> [1] pending в БД -> [2] S3 PUT -> [3] ready в БД. См. src/reconcile.py. + """ + settings = request.app.state.settings + storage = storage_of(request) + name = validate_filename(filename) + declared = upload_media_type(request) + length = declared_length(request) + if length is not None and length > settings.max_upload_bytes: + if length <= 2 * settings.max_upload_bytes: + await drain(request, length) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % settings.max_upload_bytes, + {"Connection": "close"} if length > 2 * settings.max_upload_bytes else None) + if not await run_in_threadpool(storage.is_chat_member, chat_id, user.id): + raise not_found() + store = objects_of(request) + + slots: asyncio.Semaphore = request.app.state.upload_slots + try: + await asyncio.wait_for(slots.acquire(), timeout=settings.upload_queue_timeout_seconds) + except asyncio.TimeoutError: + raise ApiError(503, "unavailable", "Too many concurrent uploads", {"Retry-After": "1"}) + try: + try: + data, md5_digest, sha256_hex = await asyncio.wait_for( + read_limited_body(request, settings.max_upload_bytes), + timeout=settings.upload_read_timeout_seconds, + ) + except asyncio.TimeoutError: + raise bad_request("Upload was not completed in time") + except ClientDisconnect: + # Клиент оборвал загрузку: в БД и S3 ничего не записано. + log.info("upload aborted by client before completion") + raise bad_request("Upload was interrupted") + + try: + content_type, width, height = await run_in_threadpool(_validate_image, request, data, declared) + except ImageRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) + + attachment_id = uuid_mod.uuid4() + # Ключ назначает сервер: случайный, не связан ни с filename, ни с id вложения. + object_key = "%s%s" % (OBJECT_PREFIX, uuid_mod.uuid4().hex) + # [1] намерение + created = await run_in_threadpool( + storage.create_pending_attachment, attachment_id, chat_id, user.id, name, + content_type, len(data), sha256_hex, width, height, object_key, + ) + if not created: + raise not_found() + # [2] байты + try: + _inject(settings, "before_put") + await s3_call(store, settings.s3_upload_deadline_seconds, + store.put, object_key, data, content_type, md5_digest, sha256_hex) + except ObjectStoreUnavailable: + await _compensate(storage, store, attachment_id, None) + raise s3_unavailable() + finally: + del data + # [3] фиксация + try: + _inject(settings, "after_put") + attachment = await run_in_threadpool(storage.mark_attachment_ready, attachment_id) + except (psycopg.Error, PoolTimeout): + await _compensate(storage, store, attachment_id, object_key) + raise + if attachment is None: + # Строку уже забрал сборщик (загрузка шла дольше grace) — объект лишний. + await _compensate(storage, store, attachment_id, object_key) + raise ApiError(503, "unavailable", "Upload expired, please retry") + return attachment + finally: + slots.release() + + +async def _compensate(storage: Storage, store: ObjectStore, attachment_id: UUID, object_key: Optional[str]) -> None: + """Best-effort откат в рамках запроса. Если не удалось — уберёт сборщик.""" + if object_key is not None: + try: + await s3_call(store, 5.0, store.delete, object_key) + except Exception as exc: + log.warning("compensation: object delete deferred to gc (%s)", type(exc).__name__) + try: + await run_in_threadpool(storage.delete_pending_attachment, attachment_id) + except Exception as exc: + log.warning("compensation: pending row left for gc (%s)", type(exc).__name__) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID, user: User = Depends(current_user)) -> Attachment: + found = storage_of(request).get_attachment_for_member(attachment_id, user.id) + if found is None: + raise not_found() + return found[0] + + +def content_disposition(filename: str) -> str: + from urllib.parse import quote + ascii_name = "".join(c if 32 <= ord(c) < 127 and c not in ('"', "\\", ";") else "_" for c in filename) + return "inline; filename=\"%s\"; filename*=UTF-8''%s" % (ascii_name, quote(filename, safe="")) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + request: Request, + attachment_id: UUID, + variant: Optional[str] = Query(default=None), + user: User = Depends(current_user), +): + """Байты оригинала через авторизованный API (без redirect), потоком из S3.""" + if variant is not None and variant != "original" and variant not in KNOWN_VARIANTS: + raise bad_request("Invalid variant") + found = await run_in_threadpool(storage_of(request).get_attachment_for_member, attachment_id, user.id) + if found is None: + raise not_found() + attachment, object_key = found + if variant in KNOWN_VARIANTS: + # lab4 не создаёт вариантов (variants=[]): у ready-вложения их нет. + raise not_found() + store = objects_of(request) + try: + chunks, length = await s3_call(store, request.app.state.settings.s3_download_deadline_seconds, + store.open, object_key) + except ObjectStoreUnavailable: + raise s3_unavailable() + except ObjectMissing: + log.error("ready attachment %s has no object in S3 (see python -m src.reconcile)", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + if length != attachment.size_bytes: + chunks.close() # закрыть поток S3 + log.error("attachment %s: object size differs from metadata", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + return StreamingResponse( + chunks, + media_type=attachment.content_type, + headers={ + "Content-Length": str(attachment.size_bytes), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": content_disposition(attachment.filename), + "ETag": '"%s"' % attachment.sha256, + }, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..0963b229 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,25 @@ +"""Однократный запуск миграций: python -m src.migrate + +lab3: это команда one-shot сервиса `migrate` в compose.yaml. Реплики API +миграции не применяют (RUN_MIGRATIONS_ON_START=false) и стартуют только после +успешного завершения `migrate` (depends_on: service_completed_successfully). +Повторный запуск ничего не меняет; одновременный запуск нескольких копий +сериализуется pg_advisory_lock (см. src/db.py). +""" +import logging + +from .config import load_settings +from .db import wait_and_migrate + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + applied = wait_and_migrate(load_settings()) + if applied: + print("applied migrations: " + ", ".join(applied)) + else: + print("schema is up to date; nothing applied") + + +if __name__ == "__main__": + main() diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..556d8d1e --- /dev/null +++ b/src/models.py @@ -0,0 +1,104 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class UserCreate(StrictModel): + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-z0-9_]{3,32}$", + ) + display_name: str = Field( + min_length=1, + max_length=100, + ) + # lab2: пароль обязателен; длина в Unicode code points (len() в Python). + password: str = Field( + min_length=12, + max_length=128, + ) + + +class User(StrictModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(StrictModel): + token: str + token_type: str = "Bearer" + expires_at: datetime + user: User + + +class ChatCreate(StrictModel): + title: str = Field( + min_length=1, + max_length=100, + ) + member_ids: List[UUID] = Field( + min_length=1, + max_length=100, + ) + + +class Chat(StrictModel): + id: UUID + title: str + member_ids: List[UUID] + created_at: datetime + + +class MessageCreate(StrictModel): + # lab4: text="" допустим, если есть хотя бы одно вложение (проверка в main.py). + text: str = Field( + min_length=0, + max_length=4000, + ) + # Отсутствующее поле == [] (контракт); значение по умолчанию попадает и в + # канонический JSON отпечатка идемпотентности. + attachment_ids: List[UUID] = Field(default_factory=list, max_length=10) + + +class Message(StrictModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: List[UUID] = Field(default_factory=list) + + +class Attachment(StrictModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + # lab4: синхронная загрузка, в API видны только готовые вложения. + status: str = "ready" + variants: List[dict] = Field(default_factory=list) + error_code: Optional[str] = None + created_at: datetime + updated_at: datetime + + +class ChatPage(StrictModel): + items: List[Chat] + next_cursor: Optional[str] + + +class MessagePage(StrictModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/objectstore.py b/src/objectstore.py new file mode 100644 index 00000000..ca01012c --- /dev/null +++ b/src/objectstore.py @@ -0,0 +1,226 @@ +"""Приватное S3-совместимое хранилище оригиналов (lab4: SeaweedFS). + +API работает с отдельными credentials, у которых есть только Read/Write/List +на один бакет (см. infra/seaweedfs/README.md): создать/удалить бакет или +прочитать другой бакет этими ключами нельзя. Административные ключи есть +только у one-shot сервиса s3-init (src/s3_bootstrap.py). + +Все вызовы ограничены по времени (connect/read timeout, не более +S3_MAX_ATTEMPTS попыток), поэтому недоступность S3 превращается в быстрый 503 +для операций с вложениями, а не в зависший запрос. +""" +import base64 +import hashlib +import io +import logging +import threading +import time +from datetime import datetime +from typing import Iterator, List, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Settings + +log = logging.getLogger("messenger.s3") + +CHUNK = 64 * 1024 + + +class ObjectStoreUnavailable(Exception): + """S3 недоступен, таймаут или 5xx -> 503 для клиента.""" + + +class ObjectMissing(Exception): + """Объекта нет в бакете (404/NoSuchKey).""" + + +def _code(exc: ClientError) -> str: + return str(exc.response.get("Error", {}).get("Code", "")) + + +def _status(exc: ClientError) -> int: + return int(exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) or 0) + + +def make_client(endpoint: str, region: str, access_key: str, secret_key: str, + connect_timeout: float = 2.0, read_timeout: float = 10.0, max_attempts: int = 2): + config = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"max_attempts": max_attempts, "mode": "standard"}, + # Соединения на реплику: не больше, чем одновременных загрузок + скачиваний. + max_pool_connections=32, + ) + return boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=config, + ) + + +class CircuitBreaker: + """После отказа S3 следующие open_seconds операции сразу получают + ObjectStoreUnavailable, не тратя время на DNS/connect/повторы. Затем — + одна пробная попытка (half-open): успех закрывает, отказ снова открывает.""" + + def __init__(self, open_seconds: float) -> None: + self.open_seconds = open_seconds + self._lock = threading.Lock() + self._open_until = 0.0 + + def check(self) -> None: + with self._lock: + now = time.monotonic() + if now < self._open_until: + raise ObjectStoreUnavailable() + if self._open_until: + # half-open: пропускаем одну попытку, остальные пока отклоняем + self._open_until = now + self.open_seconds + + def failure(self) -> None: + with self._lock: + if not self._open_until or time.monotonic() >= self._open_until - self.open_seconds: + log.warning("s3 circuit opened for %.1fs", self.open_seconds) + self._open_until = time.monotonic() + self.open_seconds + + def success(self) -> None: + with self._lock: + self._open_until = 0.0 + + @property + def is_open(self) -> bool: + return time.monotonic() < self._open_until + + +class ObjectStore: + def __init__(self, settings: Settings) -> None: + self.bucket = settings.s3_bucket + self.breaker = CircuitBreaker(settings.s3_circuit_open_seconds) + self.client = make_client( + settings.s3_endpoint, + settings.s3_region, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_connect_timeout_seconds, + settings.s3_read_timeout_seconds, + settings.s3_max_attempts, + ) + + @staticmethod + def configured(settings: Settings) -> bool: + return bool(settings.s3_endpoint and settings.s3_access_key_id and settings.s3_secret_access_key) + + def _wrap(self, exc: Exception, op: str) -> Exception: + if isinstance(exc, ClientError): + code, status = _code(exc), _status(exc) + if status == 404 or code in ("NoSuchKey", "NotFound", "404"): + self.breaker.success() # S3 ответил — он доступен + return ObjectMissing() + log.warning("s3 %s failed: %s/%s", op, status, code) + else: + log.warning("s3 %s failed: %s", op, type(exc).__name__) + self.breaker.failure() + return ObjectStoreUnavailable() + + def put(self, key: str, data, content_type: str, md5_digest: bytes, sha256_hex: str) -> None: + """PUT с Content-MD5: S3 сам отклонит повреждённую по пути передачу (BadDigest). + + Тело передаётся как BytesIO поверх bytes (без копирования): botocore + отправляет файловый объект блоками. С bytes он склеивает заголовки и тело + в один буфер — лишняя копия размером с файл на каждую загрузку (замер в REPORT.md). + """ + self.breaker.check() + try: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=io.BytesIO(data), + ContentType=content_type, + ContentMD5=base64.b64encode(md5_digest).decode("ascii"), + Metadata={"sha256": sha256_hex}, + ) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "put") + self.breaker.success() + + def open(self, key: str) -> Tuple[Iterator[bytes], int]: + """Потоковое чтение: (итератор кусков по 64 KiB, длина). Память — O(chunk).""" + self.breaker.check() + try: + obj = self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "get") + self.breaker.success() + body = obj["Body"] + + def chunks() -> Iterator[bytes]: + try: + for chunk in body.iter_chunks(CHUNK): + yield chunk + finally: + body.close() + + return chunks(), int(obj.get("ContentLength", -1)) + + def read_all(self, key: str, limit: int) -> bytes: + """Для сверки/тестов: прочитать объект целиком (не больше limit байт).""" + it, _ = self.open(key) + out = bytearray() + for chunk in it: + out += chunk + if len(out) > limit: + raise ValueError("object exceeds limit") + return bytes(out) + + def delete(self, key: str) -> None: + """Идемпотентно: удаление отсутствующего объекта — не ошибка.""" + self.breaker.check() + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "delete") + if not isinstance(err, ObjectMissing): + raise err + + def head(self, key: str) -> Optional[dict]: + self.breaker.check() + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "head") + if isinstance(err, ObjectMissing): + return None + raise err + + def list_page(self, prefix: str, token: Optional[str] = None, + page_size: int = 1000) -> Tuple[List[Tuple[str, datetime, int]], Optional[str]]: + kwargs = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size} + if token: + kwargs["ContinuationToken"] = token + self.breaker.check() + try: + resp = self.client.list_objects_v2(**kwargs) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "list") + items = [(o["Key"], o["LastModified"], int(o.get("Size", 0))) for o in resp.get("Contents", [])] + next_token = resp.get("NextContinuationToken") if resp.get("IsTruncated") else None + return items, next_token + + def ping(self) -> bool: + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except (ClientError, BotoCoreError): + return False + + +def md5_and_sha256(data: bytes) -> Tuple[bytes, str]: + return hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..8437cf17 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,237 @@ +"""Согласованность PostgreSQL (метаданные) и S3 (байты): сборка мусора и сверка. + +Почему неучтённых объектов «навсегда» не бывает (подробно — REPORT.md): + + загрузка: [1] INSERT attachments(state=pending, object_key) COMMIT + [2] S3 PUT object_key + [3] UPDATE state=ready WHERE state=pending COMMIT -> 201 + Ключ объекта записан в БД ДО того, как объект появится в бакете. + + Сбой/обрыв между шагами: + после [1] -> строка pending без объекта; + после [2] -> строка pending + объект (S3 прошёл, COMMIT [3] — нет); + ошибка S3 в [2] -> API сам компенсирует: DELETE pending; + ошибка БД в [3] -> API сам компенсирует: DELETE object (pending уберёт GC). + Процесс может умереть в любой точке, поэтому компенсация в запросе — + оптимизация, а гарантию даёт сборщик: + + A. stale pending: строки pending старше GRACE. Сначала строка удаляется + (DELETE ... WHERE state='pending' RETURNING object_key — атомарный захват: + если запрос всё-таки успел сделать [3], строка уже ready и не удаляется), + затем удаляется объект. Если удаление объекта не удалось — объект стал + «сиротой» и будет убран проходом B. + B. orphan objects: объекты в бакете (префикс originals/), для ключа которых + в БД нет строки, и LastModified старше GRACE. Строка всегда создаётся + раньше объекта и после удаления не возрождается (ключ одноразовый), + поэтому «нет строки» + «старше GRACE» = объект не принадлежит живой загрузке. + C. missing objects: строки ready без объекта (или с другим размером) — только + отчёт: восстановить байты нельзя, удалять метаданные автоматически опасно. + +GRACE (ATTACHMENT_GC_GRACE_SECONDS, по умолчанию 900 с, минимум 300 с) много +больше самой долгой загрузки: чтение тела <= UPLOAD_READ_TIMEOUT_SECONDS (60 с) ++ проверка + S3 PUT (connect 2 с + read 10 с) x S3_MAX_ATTEMPTS (2). +Строки ready и вложения, на которые ссылаются сообщения, сборщик не трогает +никогда (FK message_attachments -> attachments ON DELETE RESTRICT). + +CLI (по умолчанию dry-run, ничего не меняет): + python -m src.reconcile # отчёт + python -m src.reconcile --apply # исправить A и B + python -m src.reconcile --apply --grace-seconds 0 # только на учебном стенде +""" +import argparse +import json +import logging +import time +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional + +from psycopg_pool import ConnectionPool + +from .objectstore import ObjectStore, ObjectStoreUnavailable + +log = logging.getLogger("messenger.reconcile") + +OBJECT_PREFIX = "originals/" +GC_LOCK_ID = 2026_0004 +BATCH = 500 + + +def _try_lock(conn) -> bool: + return conn.execute("SELECT pg_try_advisory_lock(%s)", (GC_LOCK_ID,)).fetchone()[0] + + +def _unlock(conn) -> None: + conn.execute("SELECT pg_advisory_unlock(%s)", (GC_LOCK_ID,)) + + +def collect_stale_pending(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict) -> None: + """Проход A. dry-run: только подсчёт; apply: строка -> затем объект.""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, object_key FROM attachments + WHERE state = 'pending' AND created_at < clock_timestamp() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + """, + (grace_seconds, BATCH), + ).fetchall() + report["stale_pending"] = len(rows) + report["stale_pending_ids"] = [str(r[0]) for r in rows[:20]] + if not apply: + return + deleted_rows = deleted_objects = 0 + for att_id, _key in rows: + with pool.connection() as conn: + claimed = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s AND state = 'pending' + AND created_at < clock_timestamp() - make_interval(secs => %s) + RETURNING object_key + """, + (att_id, grace_seconds), + ).fetchone() + if claimed is None: + continue # загрузка успела завершиться (ready) — не трогаем + deleted_rows += 1 + try: + store.delete(claimed[0]) + deleted_objects += 1 + except ObjectStoreUnavailable: + # Объект стал сиротой без строки; его уберёт проход B. + log.warning("gc: object delete deferred to orphan sweep") + report["stale_pending_rows_deleted"] = deleted_rows + report["stale_pending_object_deletes"] = deleted_objects + + +def sweep_orphans(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict, verify_ready: bool = True) -> None: + """Проходы B и C: полный обход префикса бакета страницами по 1000.""" + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + token: Optional[str] = None + listed = orphans = orphans_young = deleted = 0 + seen_keys: List[str] = [] + orphan_sample: List[str] = [] + started = time.monotonic() + while True: + items, token = store.list_page(OBJECT_PREFIX, token) + listed += len(items) + if items: + keys = [k for k, _, _ in items] + with pool.connection() as conn: + known = {r[0] for r in conn.execute( + "SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,) + ).fetchall()} + for key, modified, _size in items: + if key in known: + if verify_ready: + seen_keys.append(key) + continue + orphans += 1 + if modified > cutoff: + orphans_young += 1 # возможно, живая загрузка или ещё не истёк grace + continue + if len(orphan_sample) < 20: + orphan_sample.append(key) + if apply: + # Повторная проверка непосредственно перед удалением. + with pool.connection() as conn: + again = conn.execute( + "SELECT 1 FROM attachments WHERE object_key = %s", (key,) + ).fetchone() + if again is None: + store.delete(key) + deleted += 1 + if not token: + break + report["objects_listed"] = listed + report["orphan_objects"] = orphans + report["orphan_objects_within_grace"] = orphans_young + report["orphan_objects_sample"] = orphan_sample + if apply: + report["orphan_objects_deleted"] = deleted + report["bucket_scan_seconds"] = round(time.monotonic() - started, 3) + + if verify_ready: + # C: ready-строки, для которых объекта нет в листинге (или не тот размер). + present = set(seen_keys) + with pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, size_bytes FROM attachments WHERE state = 'ready'" + ).fetchall() + missing = [str(r[0]) for r in rows if r[1] not in present] + report["ready_rows"] = len(rows) + report["missing_objects"] = len(missing) + report["missing_object_ids"] = missing[:20] + + +def run_once(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, apply: bool, + orphan_sweep: bool = True) -> Optional[Dict]: + """Один проход под advisory lock. None — сейчас работает другая реплика.""" + report: Dict = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds} + with pool.connection() as lock_conn: + lock_conn.autocommit = True + if not _try_lock(lock_conn): + return None + try: + collect_stale_pending(pool, store, grace_seconds, apply, report) + if orphan_sweep: + sweep_orphans(pool, store, grace_seconds, apply, report) + finally: + _unlock(lock_conn) + # Соединение вернётся в общий пул — режим транзакций как у остальных. + lock_conn.autocommit = False + return report + + +def gc_loop(pool: ConnectionPool, store: ObjectStore, settings, stop) -> None: + """Фон в каждой реплике: проход A каждые N с, проход B — реже. Lock — одна реплика.""" + last_sweep = time.monotonic() + while not stop.wait(settings.attachment_gc_interval_seconds): + try: + sweep = time.monotonic() - last_sweep >= settings.orphan_sweep_interval_seconds + report = run_once(pool, store, settings.attachment_gc_grace_seconds, apply=True, + orphan_sweep=sweep) + if report is None: + continue + if sweep: + last_sweep = time.monotonic() + if report.get("stale_pending") or report.get("orphan_objects_deleted") or report.get("missing_objects"): + log.info("attachment gc: %s", json.dumps(report, ensure_ascii=False)) + except Exception as exc: # БД/S3 недоступны — следующий проход повторит + log.warning("attachment gc skipped: %s", type(exc).__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reconcile attachments metadata (PostgreSQL) with S3 objects") + parser.add_argument("--apply", action="store_true", help="delete stale pending rows and orphan objects") + parser.add_argument("--grace-seconds", type=int, default=None, + help="override ATTACHMENT_GC_GRACE_SECONDS (values < 300 only for lab tests)") + parser.add_argument("--no-orphan-sweep", action="store_true", help="skip bucket listing") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + + from .config import load_settings + from .db import create_pool + + settings = load_settings() + if not ObjectStore.configured(settings): + raise SystemExit("S3 is not configured") + grace = settings.attachment_gc_grace_seconds if args.grace_seconds is None else args.grace_seconds + pool = create_pool(settings) + pool.wait(timeout=15) + store = ObjectStore(settings) + try: + started = time.monotonic() + report = run_once(pool, store, grace, args.apply, orphan_sweep=not args.no_orphan_sweep) + if report is None: + raise SystemExit("another reconciler holds the lock; retry later") + report["total_seconds"] = round(time.monotonic() - started, 3) + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/src/runtime.py b/src/runtime.py new file mode 100644 index 00000000..57904cba --- /dev/null +++ b/src/runtime.py @@ -0,0 +1,39 @@ +"""Состояние процесса, общее для middleware, readiness и сервера. + +INSTANCE_ID — значение X-Instance-Id. По умолчанию hostname контейнера: +Docker задаёт его равным короткому ID контейнера, поэтому scripts/ops.py +однозначно сопоставляет заголовок конкретному контейнеру (stop-one). +Значение не меняется в течение жизни процесса. + +DRAIN — фаза «осушения» после SIGTERM (см. src/server.py): процесс ещё +обслуживает запросы, но /health/ready уже отвечает 503, и балансировщик +выводит реплику из ротации до того, как она перестанет принимать соединения. +""" +import os +import socket +import threading +import time +from typing import Optional + +INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + + +class Drain: + def __init__(self) -> None: + self._event = threading.Event() + self.started_at: Optional[float] = None + + def begin(self) -> bool: + """True, если осушение началось этим вызовом.""" + if self._event.is_set(): + return False + self.started_at = time.monotonic() + self._event.set() + return True + + @property + def active(self) -> bool: + return self._event.is_set() + + +DRAIN = Drain() diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 00000000..0f832097 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,226 @@ +"""Bootstrap и проверка приватного бакета: python -m src.s3_bootstrap + +Запускается one-shot сервисом `s3-init` до старта API (depends_on +service_completed_successfully). Идемпотентно: повторный запуск не меняет +данные — BucketAlreadyOwnedByYou считается успехом. + +Проверки (любое нарушение -> exit 1, API не стартует): + * API-ключи могут PUT/GET/DELETE в своём бакете (проба в префиксе _bootstrap/, + который сборщик мусора не обходит); + * API-ключи НЕ могут создать бакет (нет прав администратора); + * анонимный GET объекта и анонимный листинг бакета -> 403; + * административные HTTP-порты SeaweedFS (master 9333, filer 8888, + volume 8080) не слушают в общей сети — доступен только S3 API. + + python -m src.s3_bootstrap # s3-init: создать бакет + проверки + python -m src.s3_bootstrap --check-only # только проверки (без admin-ключей) + python -m src.s3_bootstrap --check-only --sample-from-db + # + анонимный GET реального объекта последнего ready-вложения -> 403, + # GET тем же ключом с API-credentials -> 200 и совпадает SHA-256. + python -m src.s3_bootstrap --wait-ready 90 + # прогрев после рестарта: ждать, пока запись/чтение пробного объекта и + # чтение СУЩЕСТВУЮЩЕГО объекта занимают < 2 с. SeaweedFS отвечает /healthz + # сразу, но первые чтение/запись после рестарта блокируются на 15–30 с + # (выборы Raft-лидера master, heartbeat volume-сервера; замер в REPORT.md). +Значения ключей не печатаются. +""" +import argparse +import hashlib +import json +import os +import socket +import sys +import time +import urllib.error +import urllib.request +import uuid +from urllib.parse import quote, urlparse + +from botocore.exceptions import BotoCoreError, ClientError + +from .config import _read_secret +from .objectstore import make_client + +ADMIN_PORTS = (9333, 8888, 8080, 19333, 18888) + + +def anonymous_status(url: str) -> int: + req = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def port_open(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=1.5): + return True + except OSError: + return False + + +def wait_ready(timeout: float) -> int: + """Ждать, пока S3 быстро (< 2 с на операцию) выполняет запись, чтение и удаление + пробного объекта и чтение реального (самого старого ready) объекта, если он есть.""" + import psycopg + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + client = make_client(endpoint, os.environ.get("S3_REGION", "us-east-1"), _read_secret("S3_ACCESS_KEY_ID"), + _read_secret("S3_SECRET_ACCESS_KEY"), 1, 2, 1) + started = time.monotonic() + key = None + looked_up = False + while time.monotonic() - started < timeout: + try: + if not looked_up and _read_secret("DATABASE_URL"): + with psycopg.connect(_read_secret("DATABASE_URL"), connect_timeout=3) as conn: + row = conn.execute("SELECT object_key FROM attachments WHERE state = 'ready' " + "ORDER BY created_at LIMIT 1").fetchone() + key = row[0] if row else None + looked_up = True + t = time.monotonic() + probe = "_bootstrap/warmup-%s" % uuid.uuid4().hex + client.put_object(Bucket=bucket, Key=probe, Body=b"warmup") + client.get_object(Bucket=bucket, Key=probe)["Body"].read() + client.delete_object(Bucket=bucket, Key=probe) + if key: + client.get_object(Bucket=bucket, Key=key)["Body"].read() + print(json.dumps({"s3_ready_after_s": round(time.monotonic() - started, 2), + "last_cycle_s": round(time.monotonic() - t, 3), + "existing_object_read": bool(key)})) + return 0 + except (ClientError, BotoCoreError, OSError, psycopg.Error): + time.sleep(0.5) + print("S3 did not become ready for reads/writes within %.0fs" % timeout, file=sys.stderr) + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--sample-from-db", action="store_true") + parser.add_argument("--wait-ready", type=float, default=0.0, metavar="SECONDS") + args = parser.parse_args() + if args.wait_ready: + return wait_ready(args.wait_ready) + + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + wait_seconds = int(os.environ.get("S3_BOOTSTRAP_WAIT_SECONDS", "90")) + report = {"endpoint_host": urlparse(endpoint).hostname, "bucket": bucket, "checks": []} + ok = True + + def check(name, condition, detail=""): + nonlocal ok + ok = ok and bool(condition) + report["checks"].append({"check": name, "ok": bool(condition), "detail": detail}) + + api_key = _read_secret("S3_ACCESS_KEY_ID") + api_secret = _read_secret("S3_SECRET_ACCESS_KEY") + if not (api_key and api_secret): + print("S3 API credentials are not configured", file=sys.stderr) + return 2 + api = make_client(endpoint, region, api_key, api_secret, 2, 10, 2) + + if not args.check_only: + admin_key = _read_secret("S3_ADMIN_ACCESS_KEY_ID") + admin_secret = _read_secret("S3_ADMIN_SECRET_ACCESS_KEY") + if not (admin_key and admin_secret): + print("S3 admin credentials are not configured", file=sys.stderr) + return 2 + admin = make_client(endpoint, region, admin_key, admin_secret, 2, 10, 1) + deadline = time.monotonic() + wait_seconds + while True: + try: + admin.list_buckets() + break + except (ClientError, BotoCoreError) as exc: + if time.monotonic() > deadline: + print("S3 is not reachable: %s" % type(exc).__name__, file=sys.stderr) + return 1 + time.sleep(1) + try: + admin.create_bucket(Bucket=bucket) + report["bucket_action"] = "created" + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + report["bucket_action"] = "already exists (idempotent no-op)" + else: + print("create_bucket failed: %s" % code, file=sys.stderr) + return 1 + + # Проба API-ключами. + probe_key = "_bootstrap/probe-%s" % uuid.uuid4().hex + body = b"probe " + uuid.uuid4().bytes + try: + api.put_object(Bucket=bucket, Key=probe_key, Body=body) + got = api.get_object(Bucket=bucket, Key=probe_key)["Body"].read() + check("api credentials: put/get own bucket", got == body) + except (ClientError, BotoCoreError) as exc: + check("api credentials: put/get own bucket", False, type(exc).__name__) + got = None + + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(probe_key))) + check("anonymous GET object is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status("%s/%s?list-type=2" % (endpoint, bucket)) + check("anonymous list bucket is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status(endpoint + "/") + check("anonymous list buckets is denied", status in (401, 403), "HTTP %d" % status) + + other = "forbidden-%s" % uuid.uuid4().hex[:12] + try: + api.create_bucket(Bucket=other) + check("api credentials cannot create buckets", False, "bucket was created") + try: + api.delete_bucket(Bucket=other) + except (ClientError, BotoCoreError): + pass + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot create buckets", code == "AccessDenied", code) + try: + api.delete_bucket(Bucket=bucket) + check("api credentials cannot delete the bucket", False, "bucket deleted") + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot delete the bucket", code in ("AccessDenied", "BucketNotEmpty"), code) + + try: + api.delete_object(Bucket=bucket, Key=probe_key) + except (ClientError, BotoCoreError): + pass + + host = urlparse(endpoint).hostname + exposed = [p for p in ADMIN_PORTS if port_open(host, p)] + check("SeaweedFS admin/UI ports are not reachable on the shared network", not exposed, + "open: %s" % exposed if exposed else "checked %s" % list(ADMIN_PORTS)) + + if args.sample_from_db: + import psycopg + dsn = _read_secret("DATABASE_URL") + with psycopg.connect(dsn, connect_timeout=5) as conn: + row = conn.execute( + "SELECT object_key, sha256 FROM attachments WHERE state = 'ready' " + "ORDER BY created_at DESC LIMIT 1" + ).fetchone() + if row is None: + check("sample ready attachment exists in DB", False, "upload an attachment first (make test)") + else: + key, sha = row + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(key))) + check("anonymous GET of a real attachment object is denied", status in (401, 403), "HTTP %d" % status) + data = api.get_object(Bucket=bucket, Key=key)["Body"].read() + check("same object is readable with API credentials and matches sha256", + hashlib.sha256(data).hexdigest() == sha, "%d bytes" % len(data)) + + report["ok"] = ok + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..66564638 --- /dev/null +++ b/src/security.py @@ -0,0 +1,97 @@ +"""Пароли, разбор Basic/Bearer и генерация токенов.""" +import base64 +import binascii +import hashlib +import re +import secrets +import threading +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +# Argon2id (библиотека argon2-cffi). Параметры — рекомендация OWASP для +# Argon2id: m=19 MiB, t=2, p=1. Пароль хешируется целиком в UTF-8, без усечения. +_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1) + +# Хеш-«пустышка»: проверка для несуществующего пользователя стоит столько же, +# сколько для существующего, — время ответа не выдаёт наличие username. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing-equalization") + +# 32 байта случайности = 256 бит энтропии; base64url без padding = 43 символа. +TOKEN_BYTES = 32 +MAX_TOKEN_LENGTH = 256 + + +# lab3: у каждой реплики лимит памяти (deploy.resources.limits). Один хеш Argon2id +# занимает 19 MiB, а sync-обработчики FastAPI выполняются в threadpool до 40 потоков. +# Без ограничения всплеск входов мог бы занять 40 x 19 MiB и привести к OOM-kill +# реплики. Семафор ограничивает число одновременных хешей (PASSWORD_HASH_CONCURRENCY). +_kdf_slots = threading.BoundedSemaphore(4) + + +def configure_kdf_concurrency(limit: int) -> None: + global _kdf_slots + _kdf_slots = threading.BoundedSemaphore(max(1, limit)) + + +def hash_password(password: str) -> str: + with _kdf_slots: + return _hasher.hash(password) + + +def verify_password(password_hash: Optional[str], password: str) -> bool: + try: + with _kdf_slots: + return _hasher.verify(password_hash or _DUMMY_HASH, password) and bool( + password_hash + ) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(TOKEN_BYTES) + + +def token_digest(token: str) -> bytes: + """Токен высокоэнтропийный, поэтому для поиска достаточно быстрого SHA-256. + + В отличие от пароля, перебор 256-битного токена невозможен, и KDF здесь + только замедлил бы каждый запрос. + """ + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]: + """Authorization: Basic base64(username:password) -> (username, password).""" + if not header: + return None + scheme, _, value = header.strip().partition(" ") + if scheme.lower() != "basic" or not value.strip(): + return None + try: + raw = base64.b64decode(value.strip(), validate=True) + decoded = raw.decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо + if not sep or not USERNAME_RE.match(username): + return None + if not 12 <= len(password) <= 128: + return None + return username, password + + +def parse_bearer(header: Optional[str]) -> Optional[str]: + if not header: + return None + scheme, _, value = header.strip().partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token or len(token) > MAX_TOKEN_LENGTH: + return None + if not token.isascii() or any(ch.isspace() for ch in token): + return None + return token diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..361d81a4 --- /dev/null +++ b/src/server.py @@ -0,0 +1,65 @@ +"""Точка входа контейнера: Uvicorn с фазой осушения (drain) перед остановкой. + +Последовательность при SIGTERM (docker stop / rolling update / stop-one): + + t=0 SIGTERM. Реплика ПРОДОЛЖАЕТ принимать и обслуживать запросы, + но /health/ready -> 503 и ответы идут с Connection: close. + t<=~1.5 с HAProxy (inter 1s, fastinter 500ms, fall 2) помечает реплику DOWN + и больше не отправляет ей новые запросы. + t=DRAIN_SECONDS (3 с) + Uvicorn закрывает listening-сокет, дожидается завершения запросов + в обработке (не дольше GRACEFUL_TIMEOUT_SECONDS = 10 с), закрывает + пул PostgreSQL и выходит. + stop_grace_period 20 с > 3 + 10 — Docker не успевает прислать SIGKILL. + +Повторный SIGTERM/SIGINT во время осушения — немедленный переход к остановке. +Запуск: python -m src.server (CMD в Dockerfile, процесс — PID 1). +""" +import logging +import os +import threading + +import uvicorn + +from .runtime import DRAIN, INSTANCE_ID + +log = logging.getLogger("messenger.server") + + +class DrainingServer(uvicorn.Server): + drain_seconds = 3.0 + + def handle_exit(self, sig, frame): # вызывается из обработчика сигнала + if self.should_exit or not DRAIN.begin(): + # Второй сигнал (или уже останавливаемся) — без ожидания. + return super().handle_exit(sig, frame) + log.info( + "signal %s: instance %s draining for %.1fs (readiness=503), then graceful stop", + sig, INSTANCE_ID, self.drain_seconds, + ) + timer = threading.Timer(self.drain_seconds, super().handle_exit, args=(sig, frame)) + timer.daemon = True + timer.start() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + config = uvicorn.Config( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + # Дольше, чем HAProxy держит простаивающие соединения к серверу + # (pool-purge-delay 5s): соединение закрывает LB, а не сервер под запросом. + timeout_keep_alive=int(os.environ.get("KEEP_ALIVE_SECONDS", "30")), + timeout_graceful_shutdown=int(os.environ.get("GRACEFUL_TIMEOUT_SECONDS", "10")), + access_log=False, + server_header=False, + log_config=None, + ) + server = DrainingServer(config) + server.drain_seconds = float(os.environ.get("DRAIN_SECONDS", "3")) + server.run() + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..d906fea8 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,240 @@ +"""Сессии: PostgreSQL — источник истины, Redis — кэш проверок токена. + +Роль Redis +---------- +Каждый защищённый запрос сначала ищет `sess:` в Redis. +* попадание -> пользователь берётся из кэша, PostgreSQL не трогаем; +* промах -> читаем sessions JOIN users в PostgreSQL и кладём результат + в Redis через SET NX с TTL = min(SESSION_CACHE_TTL_SECONDS, + время до expires_at). Запись кэша никогда не переживает сессию. + +Инвалидация (logout) +-------------------- +1. UPDATE sessions SET revoked_at (COMMIT в PostgreSQL — момент отзыва); +2. SET sess: = "revoked" PX=SESSION_CACHE_TTL (обычный SET, перезаписывает). +Заполнение кэша идёт через SET NX, поэтому параллельный запрос, прочитавший +сессию до отзыва, не может перезаписать «надгробие» валидной записью. + +Надёжность отзыва при сбое Redis +-------------------------------- +Отзыв в PostgreSQL и строка-«долг» в session_cache_invalidations пишутся +ОДНОЙ транзакцией (миграция 002). Строка удаляется только после успешной +записи «надгробия» в Redis. Если записать его не удалось (таймаут, обрыв +связи, Redis заморожен), на кэш больше нельзя полагаться: в нём могла +остаться положительная запись отозванного токена. Поэтому: +* пока долг не погашен, положительные попадания кэша НЕ принимаются — + каждый токен проверяется в PostgreSQL (отозванный -> 401); +* не чаще раза в секунду API пытается погасить долг: пишет надгробия в + Redis и удаляет строки; строки старше SESSION_CACHE_TTL + 5 с удаляются + без записи (любая положительная запись к этому моменту уже истекла); +* наличие долга перечитывается из PostgreSQL не реже раза в секунду, так что + и другие реплики (lab3) перестают доверять кэшу не позже чем через 1 с; + реплика, выполнившая logout, — сразу, до ответа 204. + +Отказ Redis +----------- +Любая ошибка Redis (таймаут 250 мс) -> кэш пропускается, проверка идёт в +PostgreSQL; на CACHE_RETRY_AFTER_SECONDS Redis выключается (простой +circuit breaker), чтобы не платить таймаут на каждом запросе. +Redis работает без persistence: после рестарта кэш пуст, и это безопасно — +следующие запросы просто заполнят его из PostgreSQL. +""" +import json +import logging +import threading +import time +from datetime import datetime, timezone +from typing import Optional, Tuple + +import redis + +from .config import Settings +from .models import User +from .security import new_token, token_digest +from .storage import Storage + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionCache: + def __init__(self, settings: Settings) -> None: + self.ttl_ms = settings.session_cache_ttl_seconds * 1000 + self.retry_after = settings.cache_retry_after_seconds + self._down_until = 0.0 + self._lock = threading.Lock() + self.client: Optional[redis.Redis] = None + if settings.cache_url: + self.client = redis.Redis.from_url( + settings.cache_url, + socket_timeout=settings.cache_timeout_seconds, + socket_connect_timeout=settings.cache_timeout_seconds, + retry_on_timeout=False, + health_check_interval=30, + max_connections=20, + decode_responses=True, + ) + # Счётчики для наблюдения и тестов (GET /health/cache не нужен контракту, + # поэтому они только логируются). + self.hits = 0 + self.misses = 0 + self.errors = 0 + + @staticmethod + def key(digest: bytes) -> str: + return "sess:" + digest.hex() + + def _available(self) -> bool: + return self.client is not None and time.monotonic() >= self._down_until + + def _failed(self, exc: Exception) -> None: + with self._lock: + self.errors += 1 + if time.monotonic() >= self._down_until: + log.warning("redis unavailable (%s); bypassing cache for %.0fs", + type(exc).__name__, self.retry_after) + self._down_until = time.monotonic() + self.retry_after + + def get(self, digest: bytes): + """None — промах/кэш недоступен; REVOKED — отозвано; иначе (User, expires_at).""" + if not self._available(): + return None + try: + raw = self.client.get(self.key(digest)) + except redis.RedisError as exc: + self._failed(exc) + return None + if raw is None: + self.misses += 1 + return None + self.hits += 1 + if raw == REVOKED: + return REVOKED + try: + data = json.loads(raw) + expires_at = datetime.fromisoformat(data["expires_at"]) + return User(**data["user"]), expires_at + except (ValueError, KeyError, TypeError): + return None + + def put(self, digest: bytes, user: User, expires_at: datetime) -> None: + if not self._available(): + return + remaining_ms = int((expires_at - datetime.now(timezone.utc)).total_seconds() * 1000) + ttl = min(self.ttl_ms, remaining_ms) + if ttl <= 0: + return + value = json.dumps({"user": user.model_dump(mode="json"), "expires_at": expires_at.isoformat()}) + try: + self.client.set(self.key(digest), value, px=ttl, nx=True) + except redis.RedisError as exc: + self._failed(exc) + + def revoke(self, digest: bytes) -> bool: + """True — надгробие записано (или кэша нет вовсе); False — не удалось.""" + if self.client is None: + return True + if not self._available(): + return False + try: + self.client.set(self.key(digest), REVOKED, px=self.ttl_ms) + return True + except redis.RedisError as exc: + self._failed(exc) + return False + + def ping(self) -> bool: + if self.client is None: + return False + try: + return bool(self.client.ping()) + except redis.RedisError: + return False + + +class SessionService: + # Как часто перечитывать из PostgreSQL наличие непогашенных инвалидаций. + RECHECK_SECONDS = 1.0 + + def __init__(self, storage: Storage, cache: SessionCache, settings: Settings) -> None: + self.storage = storage + self.cache = cache + self.ttl = settings.session_ttl_seconds + # Строки старше этого возраста можно удалять без записи в Redis. + self.stale_after = settings.session_cache_ttl_seconds + 5 + self._lock = threading.Lock() + self._pending = True # до первой сверки с БД кэшу не доверяем + self._checked_at = float("-inf") + + def create(self, user: User) -> Tuple[str, datetime]: + token = new_token() + expires_at = self.storage.create_session(token_digest(token), user.id, self.ttl) + return token, expires_at + + def _reconcile(self) -> None: + """Погасить долг инвалидации; вызывается под self._lock.""" + try: + rows = self.storage.pending_invalidations() + done = [] + for digest, age in rows: + if age > self.stale_after or self.cache.revoke(digest): + done.append(digest) + else: + break # Redis всё ещё недоступен — попробуем позже + self.storage.delete_invalidations(done) + self._pending = len(done) < len(rows) or len(rows) >= 500 + if rows and not self._pending: + log.info("cache invalidation backlog cleared (%d)", len(done)) + except Exception as exc: # PostgreSQL недоступна: доверять кэшу нельзя + log.warning("cannot verify cache invalidations: %s", type(exc).__name__) + self._pending = True + # Пробрасываем сразу (-> 503): иначе запрос ждал бы пул второй раз + # в find_session и отвечал за 2 x DB_POOL_TIMEOUT_SECONDS. + raise + + def cache_trusted(self) -> bool: + now = time.monotonic() + if now - self._checked_at < self.RECHECK_SECONDS: + return not self._pending + with self._lock: + if now - self._checked_at >= self.RECHECK_SECONDS: + self._reconcile() + self._checked_at = time.monotonic() + return not self._pending + + def resolve(self, token: str) -> Optional[User]: + digest = token_digest(token) + # Сначала сверка (она может дописать надгробия в Redis), и только + # потом чтение кэша — иначе прочитанная до сверки запись устарела бы. + trusted = self.cache_trusted() + cached = self.cache.get(digest) + if cached == REVOKED: + return None + if cached is not None and trusted: + user, expires_at = cached + if expires_at > datetime.now(timezone.utc): + return user + found = self.storage.find_session(digest) # PostgreSQL недоступна -> исключение -> 503 + if found is None: + return None + user, expires_at = found + self.cache.put(digest, user, expires_at) + return user + + def revoke(self, token: str) -> bool: + digest = token_digest(token) + if not self.storage.revoke_session(digest): # COMMIT: отзыв + долг инвалидации + return False + if self.cache.revoke(digest): + try: + self.storage.delete_invalidations([digest]) + except Exception: + pass # строка останется и будет удалена при следующей сверке + else: + # Надгробие не записано: до погашения долга кэшу не доверяем. + with self._lock: + self._pending = True + self._checked_at = time.monotonic() + log.warning("logout: cache invalidation deferred; positive cache disabled") + return True diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..092d3bb4 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,565 @@ +"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается +со значениями пользователя. Каждая публичная операция — одна транзакция +(`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении). +""" +from datetime import datetime +from typing import List, Optional, Sequence, Tuple +from uuid import UUID, uuid4 + +from psycopg import errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .models import Attachment, Chat, Message, User + + +class UsernameTaken(Exception): + pass + + +class UnknownMember(Exception): + pass + + +class NotChatMember(Exception): + """Чата нет или пользователь не участник -> 404.""" + + +class IdempotencyConflict(Exception): + """Тот же ключ уже использован с другим телом -> 409.""" + + +class IdempotencyRace(Exception): + """Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом.""" + + +class AttachmentNotFound(Exception): + """Вложение не существует, не готово или принадлежит другому чату -> 404.""" + + +# Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py). +ATTACHMENT_GC_LOCK_ID = 2026_0004 + +ATTACHMENT_COLUMNS = ( + "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.created_at, a.updated_at" +) + +# Подзапрос: attachment_ids сообщения в исходном порядке (position). +MESSAGE_ATTACHMENT_IDS = ( + "COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) " + "FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids" +) + + +def _attachment(row) -> Attachment: + return Attachment( + id=row["id"], + chat_id=row["chat_id"], + owner_id=row["owner_id"], + filename=row["filename"], + content_type=row["content_type"], + size_bytes=row["size_bytes"], + sha256=row["sha256"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +# Одна реплика за раз чистит истёкшие ключи идемпотентности. +IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003 + + +Position = Optional[Tuple[datetime, UUID]] + + +def _user(row) -> User: + return User( + id=row["id"], + username=row["username"], + display_name=row["display_name"], + created_at=row["created_at"], + ) + + +class Storage: + def __init__(self, pool: ConnectionPool) -> None: + self.pool = pool + + def ping(self) -> None: + with self.pool.connection() as conn: + conn.execute("SELECT 1") + + # users + + def create_user(self, username: str, display_name: str, password_hash: str) -> User: + try: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ).fetchone() + except errors.UniqueViolation: + # Гонку двух одинаковых регистраций решает UNIQUE(username) в БД, + # а не проверка «SELECT, потом INSERT» в приложении. + raise UsernameTaken() + return _user(row) + + def get_credentials(self, username: str) -> Optional[Tuple[User, str]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT id, username, display_name, created_at, password_hash + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if row is None: + return None + return _user(row), row["password_hash"] + + # sessions (источник истины) + + def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime: + with self.pool.connection() as conn: + # Заодно убираем давно истёкшие сессии этого пользователя. + conn.execute( + "DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'", + (user_id,), + ) + row = conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING expires_at + """, + (token_hash, user_id, ttl_seconds), + ).fetchone() + return row[0] + + def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked_at IS NULL + AND s.expires_at > clock_timestamp() + """, + (token_hash,), + ).fetchone() + if row is None: + return None + return _user(row), row["expires_at"] + + def revoke_session(self, token_hash: bytes) -> bool: + """Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = %s + AND revoked_at IS NULL + AND expires_at > clock_timestamp() + RETURNING 1 + """, + (token_hash,), + ).fetchone() + if row is not None: + conn.execute( + """ + INSERT INTO session_cache_invalidations (token_hash) + VALUES (%s) ON CONFLICT (token_hash) DO NOTHING + """, + (token_hash,), + ) + return row is not None + + # долг инвалидации кэша + + def has_pending_invalidations(self) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)" + ).fetchone()[0] + + def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]: + """[(token_hash, возраст в секундах)] — самые старые первыми.""" + with self.pool.connection() as conn: + rows = conn.execute( + """ + SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at) + FROM session_cache_invalidations + ORDER BY revoked_at LIMIT %s + """, + (limit,), + ).fetchall() + return [(bytes(h), float(age)) for h, age in rows] + + def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None: + if not token_hashes: + return + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", + (list(token_hashes),), + ) + + # chats + + def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + [(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)], + ) + + def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat: + """Чат и все участники создаются в одной транзакции. + + Существование участников проверяет внешний ключ chat_members.user_id: + если хоть один id неизвестен, INSERT падает, и откатывается вся + транзакция, включая уже вставленную строку chats и других участников. + """ + chat_id = uuid4() + try: + with self.pool.connection() as conn: + with conn.transaction(): + created_at = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at", + (chat_id, title), + ).fetchone()[0] + self._insert_members(conn, chat_id, member_ids) + except errors.ForeignKeyViolation: + raise UnknownMember() + return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at) + + def _members_for(self, conn, chat_ids: List[UUID]) -> dict: + members = {chat_id: [] for chat_id in chat_ids} + if not chat_ids: + return members + rows = conn.execute( + """ + SELECT chat_id, user_id FROM chat_members + WHERE chat_id = ANY(%s) ORDER BY chat_id, position + """, + (chat_ids,), + ).fetchall() + for chat_id, user_id in rows: + members[chat_id].append(user_id) + return members + + def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s + WHERE c.id = %s + """, + (user_id, chat_id), + ).fetchone() + if row is None: + return None + members = self._members_for(conn, [row["id"]]) + return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"]) + + def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]: + params: list = [user_id] + keyset = "" + if after is not None: + keyset = "AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chat_members m JOIN chats c ON c.id = m.chat_id + WHERE m.user_id = %s {keyset} + ORDER BY c.created_at, c.id + LIMIT %s + """.format(keyset=keyset), + params, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + members = self._members_for(conn, [r["id"] for r in rows]) + chats = [ + Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"]) + for r in rows + ] + return chats, has_more + + # messages + + @staticmethod + def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]: + """Проверка членства и вставка — один оператор, без окна гонки.""" + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s + ) + RETURNING id, chat_id, sender_id, text, created_at + """, + (message_id, chat_id, sender_id, text, chat_id, sender_id), + ).fetchone() + return Message(**row) if row is not None else None + + @staticmethod + def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None: + """Привязка вложений в той же транзакции, что и сообщение. + + Все id должны существовать, быть ready и принадлежать ЭТОМУ чату; иначе + AttachmentNotFound -> ROLLBACK (сообщение и ключ идемпотентности не + сохраняются) -> 404. FOR KEY SHARE не даёт удалить строку до COMMIT; + составной FK (attachment_id, chat_id) дублирует проверку чата в БД. + """ + if not attachment_ids: + return + ids = list(attachment_ids) + found = conn.execute( + """ + SELECT count(*) FROM ( + SELECT 1 FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state = 'ready' + FOR KEY SHARE + ) t + """, + (ids, message.chat_id), + ).fetchone()[0] + if found != len(ids): + raise AttachmentNotFound() + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) + VALUES (%s, %s, %s, %s) + """, + [(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)], + ) + message.attachment_ids = ids + + def create_message( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + idem_key: str, + request_hash: bytes, + idem_ttl_seconds: int, + attachment_ids: Sequence[UUID] = (), + ) -> Tuple[Message, bool]: + """Идемпотентная отправка. Возвращает (сообщение, replayed). + + Всё — одна транзакция READ COMMITTED (см. migrations/003): + 1. INSERT строки ключа. Если такой ключ уже есть и он не истёк, + ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если + конкурентная транзакция с тем же ключом ещё не завершилась, INSERT + ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные + повторы через разные реплики создают ровно одно сообщение. + 2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник -> + исключение -> ROLLBACK: ключ не «сгорает» на 404. + 2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем + hash тела: совпал -> то же сообщение; нет -> 409. + Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение. + """ + new_id = uuid4() + with self.pool.connection() as conn: + with conn.transaction(): + owned = conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, chat_id, idem_key, request_hash, message_id, expires_at) + VALUES (%s, 'POST', %s, %s, %s, %s, + clock_timestamp() + make_interval(secs => %s)) + ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = EXCLUDED.message_id, + created_at = clock_timestamp(), + expires_at = EXCLUDED.expires_at + -- истёкший (ещё не удалённый очисткой) ключ можно занять заново + WHERE idempotency_keys.expires_at <= clock_timestamp() + RETURNING message_id + """, + (sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds), + ).fetchone() + if owned is not None: + message = self._insert_message(conn, new_id, chat_id, sender_id, text) + if message is None: + raise NotChatMember() # -> ROLLBACK вместе со строкой ключа + # lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего. + self._link_attachments(conn, message, attachment_ids) + return message, False + + # Ключ уже подтверждён другой (возможно, конкурентной) транзакцией. + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if member is None: + raise NotChatMember() + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + WHERE k.user_id = %s AND k.method = 'POST' + AND k.chat_id = %s AND k.idem_key = %s + """.format(attachments=MESSAGE_ATTACHMENT_IDS), + (sender_id, chat_id, idem_key), + ).fetchone() + if row is None: + # Теоретически: строка удалена между шагами (очистка/каскад). + # Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом. + raise IdempotencyRace() + if bytes(row.pop("request_hash")) != request_hash: + raise IdempotencyConflict() + return Message(**row), True + + def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]: + """Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика.""" + with self.pool.connection() as conn: + with conn.transaction(): + got = conn.execute( + "SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,) + ).fetchone()[0] + if not got: + return None + cur = conn.execute( + """ + DELETE FROM idempotency_keys + WHERE ctid IN ( + SELECT ctid FROM idempotency_keys + WHERE expires_at <= clock_timestamp() + LIMIT %s + ) + """, + (batch,), + ) + return cur.rowcount + + def list_messages( + self, chat_id: UUID, user_id: UUID, limit: int, after: Position + ) -> Optional[Tuple[List[Message], bool]]: + params: list = [chat_id] + keyset = "" + if after is not None: + keyset = "AND (created_at, id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if member is None: + return None + # Keyset-пагинация по индексу messages_chat_created_id_idx: + # Index Scan сразу с позиции курсора, читается limit+1 строк. + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM messages m + WHERE m.chat_id = %s {keyset} + ORDER BY m.created_at, m.id + LIMIT %s + """.format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"), + attachments=MESSAGE_ATTACHMENT_IDS), + params, + ).fetchall() + has_more = len(rows) > limit + return [Message(**r) for r in rows[:limit]], has_more + + # attachments (lab4) + + def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)", + (chat_id, user_id), + ).fetchone()[0] + + def create_pending_attachment( + self, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + object_key: str, + ) -> bool: + """Шаг 1 из 3: намерение записать объект. COMMIT до S3 PUT. + + Благодаря этому у любого объекта, который API кладёт в бакет, уже есть + строка с его ключом: объект без строки может появиться только после + удаления строки сборщиком и никогда не «потеряется» навсегда. + Проверка членства и вставка — один оператор. False -> не участник. + """ + with self.pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, width, height, object_key, state) + SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending' + WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) + RETURNING 1 + """, + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, chat_id, owner_id), + ).fetchone() + return row is not None + + def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]: + """Шаг 3 из 3: после успешного S3 PUT. None -> строку уже забрал сборщик.""" + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp() + WHERE a.id = %s AND a.state = 'pending' + RETURNING {cols} + """.format(cols=ATTACHMENT_COLUMNS), + (attachment_id,), + ).fetchone() + return _attachment(row) if row is not None else None + + def delete_pending_attachment(self, attachment_id: UUID) -> bool: + """Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение.""" + with self.pool.connection() as conn: + cur = conn.execute( + "DELETE FROM attachments WHERE id = %s AND state = 'pending'", + (attachment_id,), + ) + return cur.rowcount == 1 + + def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str]]: + """(метаданные, object_key) готового вложения, если user — участник его чата.""" + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT {cols}, a.object_key + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s AND a.state = 'ready' + """.format(cols=ATTACHMENT_COLUMNS), + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + return _attachment(row), row["object_key"] diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..e8b64233 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,108 @@ +"""Общие HTTP-помощники собственных тестов (только стандартная библиотека).""" +import base64 +import json +import os +import uuid +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Resp: + def __init__(self, status, data, headers, elapsed=0.0): + self.status = status + self.json = data + self.headers = headers + self.elapsed = elapsed + + +def request(method, path, token=None, payload=None, raw=None, headers=None, + base_url=None, timeout=10): + import time + h = {"Accept": "application/json"} + if token is not None: + h["Authorization"] = "Bearer " + token + body = None + if payload is not None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + body = raw + h["Content-Type"] = "application/json" + h.update(headers or {}) + # Переход 2 -> 3: POST сообщения требует Idempotency-Key. Старые тесты lab2 + # его не передают — добавляем уникальный ключ, если тест не задал свой. + # Явное значение None в headers означает «не отправлять заголовок вовсе». + if method == "POST" and path.split("?")[0].endswith("/messages") \ + and not any(k.lower() == "idempotency-key" for k in h): + h["Idempotency-Key"] = uuid.uuid4().hex + h = {k: v for k, v in h.items() if v is not None} + req = Request((base_url or BASE_URL) + path, data=body, headers=h, method=method) + started = time.monotonic() + try: + response = urlopen(req, timeout=timeout) + except HTTPError as error: + response = error + with response: + content = response.read() + status = response.status if hasattr(response, "status") else response.code + rh = response.headers + try: + data = json.loads(content.decode("utf-8")) if content else None + except (UnicodeDecodeError, json.JSONDecodeError): + data = None + return Resp(status, data, rh, time.monotonic() - started) + + +def basic(username, password): + raw = (username + ":" + password).encode("utf-8") + return "Basic " + base64.b64encode(raw).decode("ascii") + + +def new_password(): + return "Pw-" + uuid.uuid4().hex + + +def register(prefix="t", password=None, base_url=None): + username = (prefix + "_" + uuid.uuid4().hex[:16])[:32] + password = password or new_password() + r = request("POST", "/api/v1/users", base_url=base_url, payload={ + "username": username, + "display_name": "Test " + prefix, + "password": password, + }) + assert r.status == 201, (r.status, r.json) + user = r.json + user["_password"] = password + return user + + +def login(user, base_url=None): + r = request("POST", "/api/v1/auth/sessions", base_url=base_url, + headers={"Authorization": basic(user["username"], user["_password"])}) + assert r.status == 201, (r.status, r.json) + return r.json["token"] + + +def actor(prefix="t", base_url=None): + user = register(prefix, base_url=base_url) + user["_token"] = login(user, base_url=base_url) + return user + + +def create_chat(owner, *members, base_url=None): + r = request("POST", "/api/v1/chats", token=owner["_token"], base_url=base_url, payload={ + "title": "Test chat " + uuid.uuid4().hex[:8], + "member_ids": [owner["id"]] + [m["id"] for m in members], + }) + assert r.status == 201, (r.status, r.json) + return r.json + + +def reachable(base_url): + try: + urlopen(base_url.rstrip("/") + "/health/live", timeout=2).read() + return True + except (URLError, OSError, HTTPError): + return False diff --git a/student-tests/dependency_failure.py b/student-tests/dependency_failure.py new file mode 100644 index 00000000..b543fdee --- /dev/null +++ b/student-tests/dependency_failure.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Автоматическая проверка недоступности Redis и PostgreSQL. + +Сценарий явно останавливает зависимости, поэтому не входит в discovery +unittest. По умолчанию использует docker compose из корня репозитория: + + python student-tests/dependency_failure.py --base-url http://127.0.0.1:8080 + +Команды остановки/запуска можно переопределить (--stop-redis и т.д.), например +для стенда без Docker. Результат — JSON-сводка (без токенов) в stdout и, +при --output, в файл. + +Ожидаемое поведение: +* Redis недоступен -> API обходит кэш, всё работает через PostgreSQL, + readiness = 200, задержка ограничена таймаутом Redis (250 мс) и + circuit breaker'ом; logout по-прежнему отзывает сессию. +* Разрыв сети API <-> Redis БЕЗ рестарта Redis (данные кэша живы) и logout + во время разрыва -> logout 204, и отозванный токен получает 401 и во время + разрыва, и сразу после восстановления связи, и после закрытия circuit + breaker (регрессия: раньше доверяли старой положительной записи ~60 с). +* PostgreSQL недоступна -> /health/live = 200, /health/ready = 503, + запросы к данным -> 503 unavailable за ограниченное время; анонимный + запрос -> 401; неизвестный токен не принимается (503, а не 200). +* После возврата зависимости тот же токен снова работает, данные на месте. +""" +import argparse +import json +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import actor, basic, create_chat, login, request # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +MAX_FAIL_SECONDS = 6.0 # верхняя граница ответа при отказе зависимости +MAX_DEGRADED_SECONDS = 1.5 # верхняя граница ответа при отказе Redis + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, condition, detail=""): + self.results.append({"check": name, "ok": bool(condition), "detail": detail}) + print(("PASS " if condition else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + return condition + + @property + def ok(self): + return all(r["ok"] for r in self.results) + + +def run(cmd): + print("$ " + cmd, flush=True) + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_ready(timeout=90): + deadline = time.monotonic() + timeout + started = time.monotonic() + while time.monotonic() < deadline: + try: + if request("GET", "/health/ready", timeout=3).status == 200: + return time.monotonic() - started + except OSError: + pass + time.sleep(0.5) + return None + + +def is_unavailable(r): + return (r.status == 503 and r.json and r.json["error"]["code"] == "unavailable" + and r.json["error"]["request_id"] == r.headers.get("X-Request-Id")) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--base-url", default="http://127.0.0.1:8080") + p.add_argument("--stop-redis", default="docker compose stop redis") + p.add_argument("--start-redis", default="docker compose up -d --wait redis") + p.add_argument("--partition-redis", + default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1", + help="cut API<->Redis network WITHOUT restarting Redis") + p.add_argument("--heal-redis", + default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") + p.add_argument("--stop-postgres", default="docker compose stop postgres") + p.add_argument("--start-postgres", default="docker compose up -d --wait postgres") + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + + check = Check() + summary = {"base_url": common.BASE_URL, "timings": {}} + + alice, bob = actor("dep"), actor("dep") + chat = create_chat(alice, bob) + path = "/api/v1/chats/%s/messages" % chat["id"] + first = request("POST", path, token=alice["_token"], payload={"text": "до отказов"}).json + request("GET", "/api/v1/users/me", token=alice["_token"]) # прогрев кэша + + # 1. Redis недоступен + run(args.stop_redis) + try: + r = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("redis down: existing token still valid via PostgreSQL", r.status == 200, "%.3fs" % r.elapsed) + summary["timings"]["redis_down_first_request_s"] = round(r.elapsed, 3) + timings = [] + for i in range(10): + r = request("POST", path, token=bob["_token"], payload={"text": "redis down %d" % i}) + timings.append(r.elapsed) + if r.status != 201: + break + check("redis down: writes succeed", r.status == 201) + check("redis down: bounded latency", max(timings) < MAX_DEGRADED_SECONDS, + "max %.3fs over %d requests" % (max(timings), len(timings))) + summary["timings"]["redis_down_max_write_s"] = round(max(timings), 3) + check("redis down: readiness stays 200 (cache is not critical)", + request("GET", "/health/ready").status == 200) + token = login(bob) + check("redis down: login works", bool(token)) + check("redis down: logout works", + request("DELETE", "/api/v1/auth/sessions/current", token=token).status == 204) + check("redis down: revoked token rejected", + request("GET", "/api/v1/users/me", token=token).status == 401) + check("redis down: anonymous rejected", request("GET", "/api/v1/users/me").status == 401) + check("redis down: unknown token rejected", + request("GET", "/api/v1/users/me", token="x" * 43).status == 401) + finally: + run(args.start_redis) + time.sleep(6) # больше CACHE_RETRY_AFTER_SECONDS: API снова пробует Redis + check("redis back: requests ok", + request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 2. Разрыв связи с живым Redis + logout (замечание ревью №1) + carol = actor("dep") + for _ in range(2): # промах -> запись в кэш, затем попадание + request("GET", "/api/v1/users/me", token=carol["_token"]) + run(args.partition_redis) + try: + r = request("DELETE", "/api/v1/auth/sessions/current", token=carol["_token"]) + check("redis partition: logout -> 204", r.status == 204, "status %d, %.3fs" % (r.status, r.elapsed)) + check("redis partition: revoked token -> 401", + request("GET", "/api/v1/users/me", token=carol["_token"]).status == 401) + finally: + run(args.heal_redis) + # Опрашиваем дольше CACHE_RETRY_AFTER_SECONDS (5 с): после закрытия circuit + # breaker API снова читает Redis, где жила положительная запись этого токена. + statuses = set() + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + statuses.add(request("GET", "/api/v1/users/me", token=carol["_token"]).status) + statuses.add(request("POST", path, token=carol["_token"], payload={"text": "revoked"}).status) + time.sleep(0.25) + check("redis healed (same data): revoked token stays 401 for 8 s", statuses == {401}, str(sorted(statuses))) + check("redis healed: valid token works", request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 3. PostgreSQL недоступна + run(args.stop_postgres) + try: + check("postgres down: liveness 200", request("GET", "/health/live").status == 200) + r = request("GET", "/health/ready") + check("postgres down: readiness 503 unavailable", is_unavailable(r), "%.3fs" % r.elapsed) + r = request("GET", "/api/v1/users/me") + check("postgres down: anonymous -> 401 (not trusted)", r.status == 401, "status %d" % r.status) + r = request("GET", "/api/v1/users/me", token="y" * 43) + check("postgres down: unknown token is not accepted (503)", is_unavailable(r), + "status %d, %.3fs" % (r.status, r.elapsed)) + r = request("POST", path, token=alice["_token"], payload={"text": "должно не записаться"}) + check("postgres down: write -> 503 within bound", is_unavailable(r) and r.elapsed < MAX_FAIL_SECONDS, + "status %d, %.3fs" % (r.status, r.elapsed)) + summary["timings"]["postgres_down_write_s"] = round(r.elapsed, 3) + r = request("GET", path, token=alice["_token"]) + check("postgres down: history read -> 503", is_unavailable(r), "status %d" % r.status) + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(alice["username"], alice["_password"])}) + check("postgres down: login -> 503 (not 401, not 201)", is_unavailable(r), "status %d" % r.status) + body = json.dumps(r.json, ensure_ascii=False) + check("postgres down: error hides internals", + not any(s in body.lower() for s in ("psycopg", "postgres", "select", "traceback", "5432"))) + finally: + run(args.start_postgres) + recovered = wait_ready() + check("postgres back: readiness recovered", recovered is not None, + "%.1fs" % recovered if recovered is not None else "timeout") + summary["timings"]["postgres_recovery_s"] = round(recovered or -1, 2) + r = request("GET", path + "?limit=100", token=alice["_token"]) + ok = r.status == 200 and r.json["items"][0] == first + texts = [m["text"] for m in r.json["items"]] if r.status == 200 else [] + check("postgres back: same token reads intact history", ok) + check("postgres back: rejected write was not stored", "должно не записаться" not in texts) + + summary["checks"] = check.results + summary["passed"] = check.ok + if args.output: + Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(("PASS" if check.ok else "FAIL") + ": dependency failure scenario") + sys.exit(0 if check.ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 00000000..18e85e0b --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Уровень 4: последовательная остановка КАЖДОЙ API-реплики под нагрузкой. + +Для каждой реплики (по X-Instance-Id): нагрузка -> stop-one -> 15 с работы без неё +-> restore -> возврат реплики в ротацию. Собираются RPS, ошибки, задержки по +окнам, время обнаружения отказа балансировщиком (по странице HAProxy) и время +восстановления. Перед следующей репликой предыдущая уже восстановлена. + + python student-tests/failover_series.py --output evidence/failover_series.json + python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json + +Остановка/восстановление по умолчанию — scripts/ops.py (docker compose); +для другого стенда их можно переопределить --stop-cmd / --restore-cmd. +""" +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import request # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def observe_instances(token, n=40): + seen = set() + for _ in range(n): + r = request("GET", "/api/v1/users/me", token=token, timeout=3) + seen.add(r.headers.get("X-Instance-Id")) + return sorted(x for x in seen if x) + + +def run_hook(cmd, env_extra, timeout): + env = dict(os.environ, **env_extra) + started = time.monotonic() + proc = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=timeout) + return proc.returncode, time.monotonic() - started, proc.stdout[-2000:] + + +def one_round(args, target): + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + up_before = load.lb[-1][1] if load.lb else () + t_stop = load.now() + rc, stop_s, out = run_hook(args.stop_cmd, {"TARGET_INSTANCE_ID": target, "STOP_MODE": args.mode}, 60) + if rc != 0: + load.stop() + raise SystemExit("stop hook failed for %s:\n%s" % (target, out)) + time.sleep(max(0.0, args.post - stop_s)) + t_restore = load.now() + rc, restore_s, out = run_hook(args.restore_cmd, {}, 180) + if rc != 0: + load.stop() + raise SystemExit("restore hook failed:\n%s" % out) + # ждём, пока восстановленная реплика снова обслужит запрос + back = None + deadline = time.monotonic() + 30 + while back is None and time.monotonic() < deadline: + for e in reversed(load.events): + if e[0] >= t_restore and e[2] and e[4] == target: + back = e[0] + break + time.sleep(0.2) + time.sleep(args.tail) + load.stop() + t_end = load.now() + + detect = next((t - t_stop for t, up in load.lb if t >= t_stop and len(up) < len(up_before)), None) + lb_back = next((t - t_restore for t, up in load.lb if t >= t_restore and len(up) >= len(up_before)), None) + errs = load.errors_between(t_stop, t_restore) + # Сколько ещё после SIGTERM реплика отвечала (фаза осушения), до restore. + served_after = [e[0] - t_stop for e in load.events if t_stop <= e[0] < t_restore and e[2] and e[4] == target] + # После возврата hook (реплика остановлена) и до restore она не должна отвечать. + after_stop_ok = [e for e in load.events if t_stop + stop_s <= e[0] < t_restore and e[2]] + return { + "target_instance": target, + "mode": args.mode, + "lb_up_before": list(up_before), + "stop_hook_seconds": round(stop_s, 2), + "lb_detect_down_seconds": round(detect, 2) if detect is not None else None, + "target_last_served_after_stop_seconds": round(max(served_after), 2) if served_after else None, + "target_served_after_hook_returned": any(e[4] == target for e in after_stop_ok), + "errors_during_outage": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "error_window_seconds": round(max(e[0] for e in errs) - t_stop, 2) if errs else 0.0, + "restore_hook_seconds": round(restore_s, 2), + "lb_back_up_seconds": round(lb_back, 2) if lb_back is not None else None, + "target_back_in_rotation_seconds": round(back - t_restore, 2) if back is not None else None, + "windows": { + "before": load.window(0, t_stop), + "outage_first_10s": load.window(t_stop, t_stop + 10), + "outage_rest": load.window(t_stop + 10, t_restore), + "after_restore": load.window(t_restore, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + } + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--stop-cmd", default=None, help="default: python scripts/ops.py stop-one") + p.add_argument("--restore-cmd", default=None, help="default: python scripts/ops.py restore") + p.add_argument("--mode", choices=["stop", "kill"], default="stop", help="stop = SIGTERM (graceful), kill = SIGKILL") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=15) + p.add_argument("--tail", type=float, default=3) + p.add_argument("--output") + args = p.parse_args() + args.stop_cmd = shlex.split(args.stop_cmd) if args.stop_cmd else [py, "scripts/ops.py", "stop-one"] + args.restore_cmd = shlex.split(args.restore_cmd) if args.restore_cmd else [py, "scripts/ops.py", "restore"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("probe") + instances = observe_instances(probe["_token"]) + if len(instances) < 2: + raise SystemExit("need >= 2 replicas behind LB, observed %s" % instances) + print("replicas observed through LB: %s" % instances, flush=True) + rounds = [] + for target in instances: + print("-- stopping %s (mode=%s) under load" % (target, args.mode), flush=True) + r = one_round(args, target) + rounds.append(r) + w = r["windows"] + print(" hook %.2fs | LB DOWN after %s s | errors %d (%s) | RPS before %.1f, outage %.1f, after restore %.1f" + " | p95 outage %s ms | back in rotation %s s | data ok=%s" + % (r["stop_hook_seconds"], r["lb_detect_down_seconds"], r["errors_during_outage"], + ",".join(r["error_kinds"]) or "-", w["before"]["ok_rps"], w["outage_first_10s"]["ok_rps"], + w["after_restore"]["ok_rps"], w["outage_first_10s"]["latency_ms_ok"]["p95"], + r["target_back_in_rotation_seconds"], r["data"]["ok"]), flush=True) + report = { + "scenario": "stop each API replica in turn under load, restore before the next", + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201"}, + "replicas": instances, + "rounds": rounds, + } + ok = all(r["data"]["ok"] and r["target_back_in_rotation_seconds"] is not None + and not r["target_served_after_hook_returned"] for r in rounds) + report["pass"] = ok + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if ok else "FAIL") + ": every replica stopped and restored; confirmed data intact, no duplicates") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f0a2c95452152698c45f6c6dc3930e67eba7c55f GIT binary patch literal 13121 zcmeHNX;f3^);@@cQ%Mmhf}PN6CDsgmOm_ElhjerxNC*6buqMbA($gI~cC4-*_@k$%H%@=@czy7~O`0@evLSBD z6hoY$q0uMPO^uArO$-gE&X{U$VS&fvjZCeqtt_plTjDJTjp%?!^~dQOj2~}cX>4e0 z`ES3lGRSZ|^hD=#T^%cEjG>OMp$_%{!h?D0>1bZi&`W2G?pQF(@e?La0$(Wl7#gFa zt2+kFA1nZTI|lqeG}cgW%8Ugo#u@o;)whZ;b~v1Las13LZj_s>s}Wi|Zu|Dggo&S+ zPBk;PnPoeBj@?41MT?g#b#`6p=I*h|lkB&CgFhu;@X_NZl}~Hy>Yu;h zzHDf0X???QYwzgn>h2SX`z6u=nOrfrE*(f$yDacmyRZ@KG6d@y1CC{^{@}WF#zcX) zuHjg{84JcuS>da{HNwcs;qZ9lFOn|aD4#IXah=d)+qX3nKe1leV$(OcG|kHX-olRj zM=Sfs!hT;@BQ#l82OOTRAw+`2R~5a|gBX-dx`;tdkI__5suhI`7jRWcY8?8S)SpRK z8^x1;n~B=U9ot0VZxEm7)A52x403p^F2x{$c{3b>$O#I(`X1c1jKQOGsp7eC6A!J) z*Oz_=3?o{@DU}<)MEI*QsDf1AWQeLtDV;HUspBZGBZWKm?BS_O<@B5_JEe5^qNMn( z5`(PcR$@?bqI!06iBBUJ-N94>V~tL~p&UfCYM+lX8iQ(&6OO8XhTAuSFCyx!py+V9 zInw7PakfP}OHDiXzy%njVmvb%T*d{mqL%HYW?~ROKa?r$!=Otd%8S&3-hpEn6ekHT z!=R58%!!m2cnIbvhayrQ29d2T6x5lgh&7c8bO7VhpO8LV4~aDjq(tumIo6@)1uXe#**4$HJ9! zY7j81#AJ5B@Xtk$z=j;msDIgt?DCPPlLS2&w3A{av`5L8Phrq*4C-IWXrk9M#q;4N zE?S#!+e}q4yr-c}7}R;3U?^F})TMgKJXERA+BaEccPm+T@`I+fY+Wu~3pR zs9G0&4NDm&aO}X;W^kKA#Od7;JUxVec_&E<_H|`x%p9td>JtE}Gnc@wgsZ1W`m^WS zm#ICN@I4O|DUYbf%EzD(mVBlbk1H7>o0ur&a<~%zE13Tq&7H1^&QjNsTB#`{VkMHG zCavEC7jacpc+S1O197-F_YXv3Zu%? zU_mbig>s{mL~yHT)R45Z%Quo10Z@gYufb`YifBRjyD;dXeSIfWnb}XhSdMZqNJLqv z#s2|94_R(6g~5j0&V+&YN~xr7TE{_N&qVRh6LnRKyJ>T%0W^TVWY0{nVKDF}Y#-J& zA~=LfYYmW6JvZOx2~Am@)k5a9{w3j!2)cu%1=W3(E_kp&`=Fss6txIeRZ*pq!Cm@Z zrVp-2h@18rTal}97-TijWnbS&eMrn_{gC%=lWf;*2WGKI?R^#RPtyaIe6*JEHsHb( zQB+m1)>MgX50uaHHr-9=FkPGE5w;y=i4K{eyTkx0;!q4mE4f>{)p49c+ zr@Y|{hOWt8atjrYgxL%)_u~!@_W7I0j`}U(3ZHUH-F+D*Jni1omuJQBQ09{kV14 z;ZQ|fbBHSWnHHaN!U^>iQv2qiOP^E5k=taDG~P)>F;%XGtM3BqBqg~(`Sp6zYkIh3 z3F|OVw0<9~{TWtHPdje-K$lMhxW~u9I%RD3IoS-bOk@zM8h0X3_|s4<;)qU>sLdXs zbtJL9)|uoxHm9pn1vAkmxbqC5Kz$i*2kv?hmq8)r_VPK6T3kB#`?NX%cU?mAr03K* z^_9jXQK!{YI@f^1vL^z>7mo_!{UkjS|IAMOAZ|T8gQ5fk4EhYGJOI4)`VA=VRd6i8 z4)Ygk?qk{rJkz+sK$QGdipIfsacfhrBf1jH7_<*BM6by{Vc%6>K=_GTe1d@VgdyR* z1$WXKb_FXKjF*U8YvKUdk>m8$D&;KHrC;q^j*c3YHxG;+%+n^JD>*7hwf7BtZ`E^p z9M?&>Bc=lcZT*ihsEU`~9ofzd=}_PU>7xea&Ah(|=8t~>n~;Y=#)w}g{O&{zJx*Uq z0?-W7dIL{B>t_NVX}}>BQ{x0uCcOQ3m@j!wGr0!t9u>LVeV8)(LIY$Rx{& z?CUaJup=o$nDVvR-vO6xL9!h0)53pkL4Ze*ry*K+O%mcz@-IWru4Md1e~v-od4nOX zCf~FfuVS1x9SSsaNvadPONif2*jRCgZO*zK)^e5ql+LS?EV>f2QD};K5{Jyt7!~Mq zs(2o_YlCLqG~26^W3*=EggEt8q&-lBwydCjgon>ld(XU4p&AD~1S<;$KFZK0#*a#% z4Il7YfZFwA(1L>2i3(=3hbsBJHq^=qZSf-pg2IK|+@HM@2vQIv&g>neCyWTv7~s{f zh7S3T&BX#wz$-cid4*jo9eAh8^%^1o=t+(Wm$E2M$QE}y%QD$q5WXi`kjvFs-T1>y zk=H^EFthmi_R^n#^(alm+RxM_@zX+q#|EAWuokLd1+d<$z6YF`9Y=6uEe0KsBlWFp zeYn!M1`H1JAU2bAnmBwQd->x;)Cw%qu~nhi+Z?A#Rs)CvLLn!#I}abQGf;74UflkW zX-dE;s}ZdiDd8NUiu?)_m4KgQZqMgP&wx3?nmI-=$0Gb3Eg)A`ZX_M!;?QQbje@zF z`$!^nqlNoYFD^Az-2*hVmNzPPH@s9CgM->}2?o*E!X|=O$WoP&G{nG{A|pj4;81sX zDLuzxIEEQwYI(SPPKJ*r+}l&h>+Y*?^#=x&58w(#LGz}6&B0CIb8zw7v$EbgiHH8^ zaXE@NdOY_Ju$LQ=gdEAD2QXs4L4gkf!Fu34%`9m8V&K&1)Ld>x43pUzv-_{&auhsj zJhz;Ny0cX4)N24>@7TL3rsDyB0*`wU538D4IkVLuos0L$kq6cx=lRkF7RL|uS*o(Hk;AbOHBp$~EWK@D6;`=%jC z*_V;z1ZT~YfB;7-Dd2VhMgbz2dkOUrEb`w+9g1Mr0h6$&C#$`)Res&wvR&*p*0Fs` zm!H3o5(u+5eRn)?mzBg;VZgrt<t(&YNrp0|>T*(0KM2ONsWbz55B z{dISLosN>!xNIQ#^RazmBe-2L_M;?Bm3`c;G){O0Fo$4zxsEQQ=lEMLV}XR>i-56x~l&?JPXs82cu zdER*IsuOqQ<`H?1{2WL1=@ww|CkndS3otASwYiERd=$Gov&lWA=oaq!lb1iHecNF@ zIWoAZHqY1B>-gSXYkCZAyX8oKUPAQTWzWx3&wYsM8EXlMXI%)Sw%A$7f*6Fj;58h0!n9e+XPUy3@ z4!<{9b|gbORn`b-H$6{6jp*P$m4B&De|0dM=1UE23@VSpoelpq$?;^u<1VTxrL%a_ zdEexww7?aO&P(O&6E$0UfB7x(+S5Ir_TkGF@M&K+6M6Z&?~Q#w*Bg49?mHo$JO71S zaNCVBZapP%Vm7S~-nn<{fqCco^CtH#;<|U^B$n!)&J1zyiALk_uQCMp$TuI9&tEll zevVFnxPH398F}1R>QBP?bYaQ&hp(wC1#RLgdcws947ilL6jo;|S;-5HD9*$Tn{x#VcY$2S%56TsNxskt?z0k=-XHFl)L#}HBf zL_o5KJr&!e<^@wj)2up?_Mjt8Cm*acT*vL&7% zcQ4xLpKF&AwfmNOqr!SymWj#DiL)~cX3x%yhu**KT&`H%DN1YH$1{Es99e0$?lX6f zB2Tv^l@BkU&(IBSZYxZq8uTldk_&(I+G}L8Y)bJI-NbKzZu)yxz9p|^qrbPs$GWE* zQaiXy(ad{!f8h48+m6#n{?^=9=So{~81;dCylcwTH|;=?h?iM_o95y1UIS6k*F>qoK(NG zCe_}%w$El+tl`vmUG>zX35_4k4Vv~2S@>PHu`l&$N9PJRYgY6RA%T3!l`pK6pYD zob-@p06bP!mPT>}={$-0*~pPCuLc%8#vsFk@R`1tn59Bj)RUg{UiTJTj2zLYhp22o z-Qm=zQWYg+^hEd?^Fde~2JHdKj?d+=HwTqg?8R4Dzn(jRZlQs|6h3;ac}t%NS2_Us zyckVLF6nKgpkLxOyK*H@g)e!mk3lm@=McYykwHh!YA#l#KC>MZ*wR_qbGFj;yO(Km zyp6(nf0Cq=2t7zc54vhdTK2V}pLmFW2liu%>Wp0Kt%~YWWW}oUk@f(P^2fF{i+(5AJ+U-`3rk; zzQHsQg%SJuo@Yc3r=NIB&4l?sc4}%vKv$cf1Q}&@0SL!H_`ZuY44|F{Af*Mux3A3} zW)VwOMbZl9i9idRNkQk(b3lgsn`x%n05?LBrc{osM5PVMdC~Y5#IJ=sBD)pEjFhwl zzhKZIP^SnvD&t+O_@hQ;)a;O_qSPeSJ6tarYxtu_10Tdo76`}xv@8|yt7n^lSgZDK zQ~5Q8e;O8cdw0{)Q}O(h+xIUS=vmA4Y*pIdjxooe8?*HJ@MSvV8CN8YN~@ap=$F{0 z$WAFx2>g15jOPdKVE77Phy3$1{)i**v{kLg+ZZpF)Oa~AC|m2C{N2RJ2+}8!$o0;i z?DARj?TKjJ(+AG|OQ$7TjGwuOe{ne5?(V^kC1M^w@lil8aCsLMswK(->Fb`b| z=f5&P@Xd4rvGpg;EK}m4FX<*_^xq3Y`E{f6MCz~gU5^+S^|lvoIVQK8FUw>1{%c*i zJEtW0gRcBRSN{Lhm2U>r23(jk&bK}fw>IoC4c9uG=8>8GVw+b98hCmwGwF7D$GMLk zoCnOT&81{n&Wh)~h2G@lu0HA8bIbJ^k?*d##9|P4>Zj(iuRFePVQ$@?DeyOcs2(^W zf@8UN=sCN*TL3AoVdB%baj(WumiPVWuuDKU zODNm+nMs?gq2)&Vo;5ws@PXt)!iqHpoq&73aRN;A(8Sr7H@#}`Zk@FJqj$9~O=ZuF zqpbyN11>o4F>pND%ypkODbQrYx-8Q<#1jf7?+e={bth8KC(rAW-rVxrCBs-T{`BGV zS+`l|B&Xav{OddF=RZAj`RiLXhoGqt?sEwLeN1dlK;X+Sf)D#te_Q>kASUdMotkwt zdjC@IrR(!T`~%rlH`Q>nrOE237u)OU#xbs=myiF-TX;(w+Cet+L8AU3Q4dzo{<|gW zAM#S>C}B`&z9g8?VeBbVihCieH%!3hU_h;qWX>#9I@1m znBgR<#h~v(awoSJZn%`PEFGO%_xs22PCB1x)k2 zrnqHfvmKooL{i4~gFY5V>NIYHNnfcO{V1O*61ORsUbNBbV3godn_k?g&FV)>NOCh( zo8ny5@QN7a89cnx#cPs~-KMIdh!2Mxtsm($ae9Be=W}Wy8UJE z$?VL(UHX}QZDdDDJmH^nbqY?o0j>tUErVSt@q`QM$~O`?_tO3)@FAXPL;p~VCF^Qv zW?HanLl+tx^1wtwDod2xIBPpIMZRufr%-c=Q|^t2^ig`SXJ-y?<@y_aT*m(J?)W$* z=NXz~B6LB?35i-odL^S@ZM^Qno6!X8RNu8x?eKljVYh@gI-fKi*-dG@{%^HX8vj*i MRjTiBo`^O451jGYxBvhE literal 0 HcmV?d00001 diff --git a/student-tests/fixtures/sample.png b/student-tests/fixtures/sample.png new file mode 100644 index 0000000000000000000000000000000000000000..e7c920789e90d16586ecff5029a87d1fc247b086 GIT binary patch literal 111817 zcmX8adpwi>|3C0bDCg50hY&IhVa-}PAeB?jayAv^IER`~Iyjp-)LWvQLWRh=OeBYm zoGEFQvV&768SgUXUB9RA@1NcF=XKp~yIqgH?$_f%K51_)Dx@g1Wy==Pw!jr)^g?I<%@C1k??w5p=(q(7=5 z`O)3FZPp38c>W(UJfUrpsC1dH4|d6t6J5HOg!bOh-6L~LXiXxo=aMmpV`1f-lBN^-U^$ZC)OrursoPQL4nqDJjNOl@^j=pB-ybDDfPPX79G`*_N^9O*vMc%u6}|94&9p<2h~vcw#jcs+B+ z9RC-(0KXC<)T!ZAe6>F%@sZGTDL>tfqpVh;=h8E}8;4n#l!F#7o}7ks&}Vq+H1}nwTAdz zJ?%HQ6(uesSdI6XQ$vX(>|9kIG~FKTb)7_(v5z6FtAo*=Rl(KlV8kQK5JZ>@vZDME z){Ra?y{is3z*Q|)Z9`?t6}!Z^O;}mE6B7reT}$q*oF--rI=hxUScxH~4u+%JYw*rR z_g6ym7OL>>MU^Y@d2&@>)zB_+HwH^km&g=X-}0u`40E;<;jW>8|wQ>v%}NXE9kt7HAQ>0s*7r)G6t&&cLqP9&Q?*ar)${eh_xEh zp_B9^l!}v)^RN@QA;a1Qk5X|q@}eUVITUS|VOJvudM;ZB;ZQ?5N{`Po6?QSNSux8S zqsX~*R;_5}3Dec{3~J)7=4#lcc@)>qx)o9GgF#JH{=;~ijz>|q2QRvG>V9-?^rj0T zbm=HWjcZ3eo3HUGJq=~<#BJnR(koHs&fGTM5xOVhi%YGXl1r#d<8k^~?)?mX)RTuB zC*mKiDCNyO+^~aygr zUO(t_UcVt#(KYG8VYyfnc|}{%TMed@ubIJ-DV1d<0)7^|j)>k!{a`{EXvIQB+D#Ra|??I2ITFD2k7h?qZK|x)sGINL9fP zxrSLICMtDh^E*VX&tp$mL{A*Cuo1Nm3D)!s39ewti-yq~6|-NB+gn_kIBa1pIv&}v zc(1HE6|ML*L)xUIin}{Yo8oLz@{ntiRY@^6x$uhN!@14Y%L)p?TXWjj_^gGHjzfXs z#Iin$$zO%$b>m>OTFG3+h&QCP zzMX9Qtc&M5yf~Id$2q67oK5TzuT1RU7+UlmVEc-$oum5o@+B&&OfFwfy-m24nr4bG zXE|~UQr{}h2a~+Z8n;L34lhh)tJZ@6(TvM*&F z44!_#wdUsbm74x3=elzX`@Sh&eM7a_DC0V0T|KwqwNXK=?7M2Z^zwW{>|_Y3Z zcRb)Tu5(In-P@ zx`cxq`h@pdFvCF(eZu=K@Z%tdVtwy5SB^ps73VL0eN+KCWbLP%|7kOaI46|dKS2(C zp?dajQHC7C)x9=4)(<(9t%j?6Lxda(EXY=h-OQmyWOek-QOF@(=pDCE#+Bd-4w z_oIL;bICB@;$|K_lCW$3Dl~FF+0z8+?vGz5qG2Fog4) zSce>1AQabs+03D6^5}gW&GcW4k;_ehQ4fs915glKf*x{ec3p=(29c``m%9&0aFY)6!WKcRjwFv zC?@>PdiyBkQ1R8z8#W7&L&f`J|I|e1`wEBNJ!*PaVcbfYFLmpyYnq{av7%T3Zho?s5tTh+dr1x579uLBJBs*Wf8(Syr7E+Yt{Vc`d&j+7498z~O$d0Brv%=~ zWBU$y5o;4TA}z+_zA=X56I(=(1~`Sq(7Tj`I`;M?ZAJV5bATPZJge@e+IbgyOeMBml8p%3ly{5tp1C*ayL`NR zN05J80z$wo!&Sgdi8RwCNXSYM7mYv74CIm%97N;KGNZXE36BM)6g&(t@{C~r+=LOf z4WE}ts3|Xx_l~(w;JGQUnD>z>F3_s5;ka>nIVkD5{D$ks*~s?p8PO$)tMVJKn0^9( zW&g@gpIi1zYR~lamo|O4b0iu4xxmo2Kua&6uqvd-h$S>;c|f|8d?;Bdw<}7<(a0=fdh6J2~oSHGfg)J`pU@z&+L{0qx>362mZ5D@n1Paygra* zr$SgcIWajKpjCJqKW#Z8sg-R)yh)xfPa)sZ%EnBbXu(YM(Tud-WdzzGgUXA^Kg*NI z*;@W=(!_vekhG6&3_`mAKQVh;%R#oX0RMXSoR))JP{G%;P3;55^ix`6n7HhLXnLI1 z*uJ=g0TuRAnknHf zImAw6FR@rYl#ovTVmG}j^^W{m_EG8Hzx|S0)_0I+RohV=9Hpc81tr>}kJKlt->Va0;d#(K2@%^*y>;uw6cD06$PPKa*-_d?+ znM;J&#s{-Cq*W;nw(;lL#?qw}1KV2D#!=e;8oOynr3-CCW&Q3j2!xt}dfFSU=SJ&~ z2ENnQw5}Sqjm+k-wQ~;^ZqR4z+52+e+x-cTO5^p;mTLXoyWYy{pS92ZduRI0)Hbgw zNBv;g7{~s^fbJImo$rtM2xs%VlVm@iTGrpL-Xd{|F>*vtO1L`ZgZ^pt){`E3`b%mk z$`AdebBzhX!9)W}mL5jdPH!oUs7m>r@t@vO1a}KXFe6D%C6p^cNy~U3E8ff)X0v3^ zXE#LNOR3N^k<-^}X^Ct}aVvkEf|1Q`9Ct&8u&~14^jhSU^iSB4Ksgo|O_fta7v*QK9f%1$- zmo;c)yg#B!tWN2aMK?#bQG0swWb2y79sX^Ne4MgXwoZ~y-#aAWdde4?zp#a#cLYno zXtKvpc%P^IpM%*A*q7`*lzrqv{f`8$8>NCSPYKk|ZYCAVM{qAtE_xL0T52KX)7$m! zWpefGpTMQL7>%%FgpKv4HI^PnmZtpH$L|Ve9TpZpRiJbB6p7KJA*+JOIaLrAke$*` zmzJIPEC^y9=iW@I)W-(~bkgte%a%Uo710;y!m@rXynpB^gd6mH{Z0bcm3xOym-TDq zrIKE;r6@M~otht?P>0$3WoHoIPkDp~q@?5#@={*Qer#P%@CamC2&40Y5%N|)_Fa0Z;olkgK^a%Yc+MTVuBzlDjo{JWDd|Mk4?NBIcX;FCTZRqLd#$ZBp zMB#77wENef=&OZ-4E#yt=~e&2GzQ+qmPQ!zOhsF8K*jy6im(=` zM`10E#LTQGY!9XWx4}Ub&ldeFDnuDr-CEglo4l4;<^(K~#tcLmeH>Xj@=dCc8-hI_XKZzrce8l`pw+Z|AG4XcR z5FLT3Ya!-kT`7N_RTC!CK+KX4=6EV4g?eiXyq70%{-?Ary$qB3mO9b5{Fj(Kasy-i zw&VD8=&EdC>pP@=9|IW@T{QBZMk;K5-_bk2ome^29{MpXy1d|Ybkm3qrZVFkrt*D3 z07r#bpY=O*I)bzHu#)&Y(u+P@qoKZDj7lT7iZt98suUfHN0&4hz3Ia%Ib>9Z;=?!! zN~>X0dJXHfBMYJC2EGmJtr23BcqOYGVtyKDsH{7?t7FG%gt8H%d4Q*(g z=+javq~~LvFc{}ob-9m6B0{f(tnTMN8&M4nZ(gt94a|>Wt~RdM@J8k@6h41Ht@fv3 zy>jGv=pX$*jcu>ypJOh*ecUB>-bL^ZpU))GA8V-W(R1PCn*>f5q9o~mV)23rJCrTO zE<{UEXoo-hUlhFkLRm)4Chb!CV8G?HOSsToin!^a0FAUPE9taL`=|R^orHsFzxU(2 zSPAT1X@dLlG*(%8b#E^c6HTHf-O)fIKQZ?Rj#vo`>MJ9^-OUUBb~h_25ZQw0$6-eg z?-%@LB`Bz@+||u3&rKQ?dx~8|*+-JfdvlQ`diF%p!`@fO67*;!VCxV87^yY)#!NN;pqWaRy%8sxvy|0#Qa z^S_o9LOUe*1}8tu#2GQVB?RB>rydrGH5~5F5=;G?qk+th!49(Q*k{vPu^q2j!A5;7 zk2DVQqgLPkjt?wlG41sI${**J!_%bcyVG)&?TJ`d+i2{EZV|z?^h?Uq3(PBm;yZAp z@oo#Tvl*wvR4y#1Ct1@2#m??8;4q2YIEL9*N+|{*1CV=151pX)_YL-&)jkZH>bB)>~Ab} zsG9$RW5v^PZ=e?8-jq*3EtIEdq+H#y_hdAyeTU_udQ@W}#bYIt*cvciG2#ev?`# zx<1}>KrP?d=jHgkfm)tYr|7%nfLhS0RO-h~YM}(6!(%q7g%x#g$Po?H(%wOzdxQpR zLH9pJE&#QBoco6U9tqU)X$75su^p(Tt>|l1#t=};&?od+Dp1Q8wz=F|JL%9kJqfK6 z1=JG7ZJ=0t1GPjOy{9APfLgqVV~xf(sYTAAozzE<&o>qJHvh6>mJih8-MP4;nGe*G zZ_pl3-=vm&icja#CbbM{qVt~uwS0@ZJ#^6&-)c z-aswUZ6D^oqk&qy|9o08$OmfiUW!?-AMy#;Z%9*ooiq>Bf|pmZ6~EPRVDbl(6q&M6 zRw6h9)FOT(?E{{s&b|IW#s{Dl@s$3TKrJgCGn^6?pca+26~+so78>WH${H7_WsLJj zMSK~kN=^wyZe~tpe17FInX7&H-w{mwe`$#wzodYELn_r*g*PfLfSbs~m$kpq9A!w;Wv+pq4neo7fznmIG)mb#jwha)$7o zoK0#;n^SQk0ky<=d}Cb}2Wn9<|9AOD8c>Uh`H$t>X+SM1Q~&c=0BT_dRHWId0JR(l zHns3bq%P6#dXNJ8VA(EAy)QZJ+QQL zKH>6YmAQ%P(<)Bya=@>K|Zk9d*k<>_*>>c zEn~mT*Zu`+`L`{9?5H_Vi}~8mwwy7b7W1{WxNBoTE#~4s-hHzX68UG>-d=kesiFf%f7SCm196HrTM(C-+)>st^HK< z*MVACoD-_<>p(5Psqgx?r~G57*LCO z1UIf`45($S(Y${hsAVm#eQbX`P|L55p@k3TKrQClzl+#FE%VV&a}>;hT7FT-`u{Um zk@%N;HzzF~s3ji%l_RMN)Dl1Z)2M%wT2!Mi53d5X4EDb=FX0-^0JW5k{fZymq?S^H zU-1J#EiV>^%>B4PEh7tr()!;(EuW&vWB1L0T7G-auPT=UwZ!YKHTD3t4ECQ_{r(%M zh086=$%zMQ;c}~U#^QlmmK&$~ajHNqs!EqbSAbeR1yXVznFF=_-soFsH3w?>z0tjZ zDFteY|MPWKt`w*xK74JxeGI6j^y;sTp-pNj-GBMdm*{*yVdUMTqIdbnt%|oC-C}%A z^XryaKI~R+7Jrpom$*c_;UO9S)-8?4w%yx$LHs%ye@L=H1gK@(lsGl0OQ1ZU>)AGp z5Kzlb>cy_h+kjexsaL!BwgI&Wks`Y+wgI&W;Pw2QucZlAnW=Z(*qSW_)UsoIZ`Z_D zpq8!gwqvz`T0*aJwzUWYwd@!-+OiF(g?|xw*XCLsJNcTn0A2^E<>s|Y0lYp?OXjr? z+b)O$wdgY6ZXMeR)MCI~-)gV}s6~Fc{MrFNpccW7-TvRV0JUuWB|y~!YRTB*EV=p*tNr3fm--Pc1B*lrbHXx(z|V1ev?|%nJ=zYYy)Z$BiVF$@d34L z%NE6=fm-6b-t&Ew1Zpv6`fU9!2Gk;iHDlhq=1gz7_KI(K2T;qtuFG4O#Fwx6UlZiR z%K)_?n5->1+kskS{V!ZI+X~dOy+Fy|_Zm>k))KL0+iUAvLu9wdW0=uffm*i58!>IR z0<~=Q6XTue0&3YRCm4SKsAc4u%{H_sP|JR1)3xv0fm-DJP>a>I$6KfPfm*gR?qO1GV_Q_|HN`0;uKWizEvbDWDdU*#Nb|6g(cNMJ-!|=#oBPZk2vZEn5tz zMQlQgW~2txVu6$^_euX*Zjqj?CIQrb3uMa@qLs6}W! z>V^9epcdga?Ccjcpq5l`si<==fLhe@Q{uIOTAoP^S=8h&`zr+0qCGo${cq|tVrrXpm7_+mWQ?PBqGWfb|IYnKKJ2jH z?oN{YY_+VhT|HC6ilKi*@1Gs#l0RsiR?j(Usi(1|_Avd2#*zY1i#)MDJxd*^MSV$` zSeg#hqQ0ccy_OEtqOOAEUQY*Vkrbye^w=!P^V#W-O6K=F_PJ2 zpcW+-b;mb#pceI*%xgPYHYHkLk3WbCc9eFfdbT`L1<0?paIm9LApST zRtIXKkgm{#)qz@$cI+XHr31A{8ju$yCf1xpVSg3oe0=1a*H0;=Sbdy@rv3uE7=|C+SAEmfc=@s;X^gxYl3Q&tG_h?j7P*KrL1zY>$Q{P>WT8 za)5m@P>baJ(E>T37MJ8o4ZL)~71|ws$vMWB`o!)J_U z$UrRyeJk?11fUi|Z8rBxra0}6ylNkB4^T_4d}$v~52%GA9}-wg0cuHaH0paG57c7x zgvL+=YKa|=AY4%bYPmeDN(iR_wdl-`8C@jTYD_S-L{ zWp)lFUrI*>{Fi)53#dhFS~s8~Im;?H`I7duHc-p+lNMD@*SJYH>~+-T8Dk zP|GnAp(j@gsO326e9tQ>pq66{{eZP(pcbi;;^m@bpcbi;(&fZtpq8BmCm6;+EyNxh zsh{>hEqCrlBo0XdwH%8)lUO4K)FSO&?0+aRgx0nFjh6gPpcZWR_3dx8fm-Yg^}4fm zZc>X>wk;L~)H0vky1T;^s6~s#NajcZwP<%7U@7keYLWh!x4e`rO~03%D{X%qsKo|* zpgVIrP>b|*KJ(9Z@f}(u!)}Y6KrK5}?3a%v0=4Wst6h)_)MA;~wsT$+s3nj2Y5Umj zg3^YwhkJsg&GUg;j`YY$ciseQxw$--T)!Krt!-fi!|DXXWs+V zvh$X<#|@yCJpb6lHEE!h-8{o?>zzO?yLl$vK|6t3c6${1`y~D#SR`ic#ApJw*fwZ( zPbCAjNMFfaexD4~BK_xjq*o76%g(FzJewY%mYuEI8wY?|ta}bh|H%Vtap)=D`PYuB zTg`9pSn*)oX|dd1NF(KTkJN>Xr#eM4b<0kZHmu4WmIdl38oi*^qVrOwz4C=+N%^r; z4}e;9zEn;d1GN~=`RXX$2Wq)Dy?@SJ2dL%#^uf8UIzTP=@z^;v9iSFxm0L}x278?+ zkvG^cWUTK4wcHE7&0dswWcfuV%n6wb)PkWCb>7_vYPq*quuTW3#VN*qLKCRv$e^@y z$xWb^6NApqCAWZDjt++Fv{wSPIAe}JO@ANoV^Fs zQpoONWG5~6+mT{;gY0)=snhHCa z7p}D31Zr{WELgdJQwQKtqkIsvuVG~($Q zGP?AKGBwT}r9drO^fVpw6F@Dy>6JR>Hb5=v^mj5}oPb)KLTwv$fLhLE=<7Ur0Mt^v zQh0Ob0Z>cnO4`l$PCzXu)}sggoPb(v+78Tp*8ysA`g41w{w7e1(~|9a#GubP{m4{B z)1+HKEqmn^hDB~Q7*CEcPe-QYm-z}DwAj@n;+XnjFHL>m`dP*S3#}CqtG>NJErT8h zIVB1}EedOcj6FatyEz{f)&_uD3^;!j#9sil?BR4Phz|g@+~GE4j_g$_DVt9hu|A7M z0kx=G42f6=0ks4L=dub!fLau?hsM!BEov71A}@eiZk9!-q7{Hz_LdZI?_~nD>@B&& zZOH^`*?VD#aR8{LHZv#)uMO1Fk+~4mu?wgrppUZm=P*!95%-_WXU0G+XBP`uO(H-o zd%bV79*X>D`B%g}Nd7udi(&6$1%Wd_Eu)6Jdw(bZwHS*GMFwuRTGBH$jI#%UTFf~5 z3K1hfEdy8#hnfl0q7X4gvW8a6Xy(N;9s7Y=bU3q_&U=Ac_S%mbY5=u-6j?h%JqXmI zpfbFC5?U>8scFXee}Gy%Qr{}fp9X5V5fGjlrvTJ)hJ@=iR+tZ}J$0S?HgoI@P|I~L zJ=5R}P|KNkGmfqTP|F#(n;)4#Eym~pstHibx6Glvow-0QzNvExZl{4-&Uh5F{u2Re zQ7|7~cHC^WD435dpW1A-C`_I8$Ome<9uSynrvTJq96Hco3e@r?bLR902B_tC=E2j` zw}4ul`$~=f`~%eD)%Q)|>KIVV4enUx)iXdXxx}EptHw)%=M$nQtIS%EPpfi!Uj)oI zBo|kNi)p9nb|8O5@yxbK&ZkFxy`>?0Jlmvy$!zb9g*{QX%+@4aM}L{E4Fk0dx4DlV zH3MofTN`b2*le|!t&PPU+HAF$iI2t|gI3F{qO`ftTdT`RQ|npm-hM|jpcXUFc`Od7 z1!rA|?4Jc{86kRSUoZn|F~iLtf>uiL+4yfgy)ybmDIiQvs&0eER zWmZ;%TBW<7-GBP@PcnGKk)&I(@WWeZP&`4s@*nB)1s3oPyU_M?4sAbQ>keS~A zP>cQoyQqEvsO5I_@zMKcKrIX2w^oabfLfyT#v0XuT2TF;k>3}9S_ZfQ**Q@_Ed$(h z*<(>aEiW1|0MK9PsqnWebP%ZJcA!o6BQv0ug$=ERRx_ZMg$=cZCq+OlQGbe7^NWC5 zqQb}4FF~s%r|9bZhAvP`Y0>`ZKZVimzQW#jkLuq=0JUs8y0saoWm~K;mfxF&sB-Tz zkVtPhqJKL}kg0BWgWzU3d&0cxpduJar0 z0c!DBE>Aol4AdgmanJwzHlP;%U$WHNO={WZE=zp~)bhORHNSuiP>WtUt~NiF5f7l{=@KrLvJO}CdYP>WEuKDKU?TDsp0e{=+DX=eKHe@6qg zps;Psn~6XziLZo*_W-rDc3tLQG5~5x5ERC{0JXR>S=(}CfLh%AFC>`p1GUH$c=`Ko zwp#d0(95<7>--_IGVxEC(fmLyGVzT}8-Ab`em^wt#Ad67UrsLm*(S9l+6bZbfm+&` zO^M%SfLh%B;}Q`3KrQ@ka-`>*)RG`B9PhG8EeQ_7@ot;clJJ;+3IWt2!|?EThE~ha zHZP%2eV~@7%zgaN^?_PCnd1Dd2%r|1<%PuOGC(bEk(at>gn?RQHkz4!{6H--)2Eha z65BI8nNp_bcRm7YxoLQ!KvORuzsjpehb2^R`Aq5xv|2RFrP}k-t(?^L8!thtMKj}{ zGtg>rZMrgWzy_$rX34e55n3(B2cFq1Id4+Sz<)L>CpW2OAjw9>38eh5=v(=(GhK{q}Y_({P8N?liR!h%6a(Q1*0ku$PH8n5g1GVT+ zw9?LK0=3+vHql%)fm$@{&_;JRsf8S3BZ2~I@g^K2f3X2-xh3z$?vvW27ESA0NY_nj z8Spys(G#e}IVyC3s0q|!^U-%D`yZed&F`qVi_mIu8W_;5yb08jSL91Ly4h;cT+$_8 zg;t9V-h*8x1=J!Xo=>{BNi9;%Hp4nVE#QZ>Okpo2?ej{Ji+eO=^)EvZ>tz)Pib!O8cz| z)N&%;gIys7)N&%;lie%@)N-O$zp)OerHs}mRd@oZGC>2&hHqT*?RC)9UpnUG#K;S~7m?l;L==-9NYB^pblRjAIvwFx57g4k z{wKpm7pSFU1ml~AcGP>W+gSV}Ac zs0D#>W4HjdH1*sP^3nxrspHwvBXogU>UkFQFa%JG>qZ;f00GqEI_=9{%K&OY{H^7= zZ?;+x7u_~m*>w+PspjV=e~o{(kc<7?TP5CXt{3pe`opit&%FrW%ab+Y&*3G}l(oy> zgV~>YucjyD@bRQ4_P+aFr}UwCbA>YfLHzifhN#uC-psFGl|`G?P1mI#jF@x~3U9-Tf!FHzmY9HMne6l4#(+x%V$$s z_jI^UKZq>q;RU>yYc+12tA249giyi8;NuRIq|;Gl6!-#p8N8(L_tKYgf61islgM%KID-juc60U zrVjB%rp}#um<8>;lpNjoBdnv1$62{TrB1vS(ot5kP^mMogLIg6_Wy<_9komZzn4rX z;=N4AL#8SJd%75Zy)Mb3yO6EVAE8UK?zV5V>24Rgpi6S-sAux=tD<F+Y`J?m4iPze5Rpw94 zq=1`aO?JjJm#_MtA2O`zov6>`Uns?@;HvT)3fJH9r8SCa}QFU z7yPI^Yw!u`J^3dJk@s*=?1*!9jfu;h@`@8S_aD8K6s(SM{U7SGWYyPudtDUB9%ss% zng?C19@$#evHcM#dt6;m=UtHMVy?u33^Yni&c_v5R$iE~RGyZh;d+SfSYDY?;rg%B zgx9~0Z{6pNP`}B4P&uw~MT1+}q&)2^yyM&<;y;6jQGzu{_0)&>Q*$~f2SjBx-e*n$ zl}0vpEx8{>&1j~Zp+;SaB^j%9eiYip=m_14O+e&VQC;X~*ii`gDykFR6(%Ncs$gCc zOSqTu#dZ2FVe%^0b51@l(Dmb z**=JN*ADHE?%ae7ao70W=5AbahJ))BEpzwvltBYmct*LcXAMTU!ZXT!J!w$Y_0R4r zZtIjmKi5B6SKQar1`nYA%3Q79@R~b_`s*6P7gFnL&GA)SlZ`c9sv!Dh{jj{8C2E41 zWE}fiNxYgR7>qw@GR3eAbj}*FjLy{*isF)K=kd0I za#?AVI>nrEhF##r2@B2*#WL=m31`lP;-zz6^?wD!C`A7MwjllIvG%=Nh#jfYCPh^N zcd^HNcZnvMibz_R7S*z3L@P61nH1HrL_~XOw-rZU8hUfyvXNN}=kX4lb@sL_x%0>} zt~#-?&&I^}A$Kpatxwm)x0;J0KJE)K@vY(FjGQ<>6eGs5pAX7E3dl@tR1BBdcaD0h zS6?w4(Rxn4GN2=MC!sv`y{YZX$TX^R?`ctI6I;Z^a|I6qo~9}g9;BX93<~M+nyw&b z_gyj_MNDEHIPqD+w9ATe=e_R{8$HU2)qTHB(b7Yv=6AV=vvg@EvT97~rg+y=Jwr_J zwJb@|5IRoui)rZiauvz8m!nuR-ryMea`|3rr3qfYWMa8Cb-@I0;5*)Ms+XeZD`gN8 zZ^e<#@}gt1>d(j9aV|7kaK0$cjPo9lY&h=~XC|o5UtjX-`aUSmNPiF6IK)X)JSe3a zvSG=oR5X_^4dET>%~$M{@(bY|?QK=;l%5IU9q!FhydrflglE-TuXsiJc?i#VU!pw4txdGmkN&T`b5sDQ^f`3*&sS~FNBLq@_9%V{ zEpM(KJzL&33MJdmp_-mwLMtjBWnUZ&a~3E!`qk);+_w8z-KGV>zLD{hYcsJ}E}X1(wXw;=Sns4}{9 zl%H*0EF6n|+2)})(dM%7A1n0XaXmvpkwI`iM|C>K@zq zAiAAVwi;U;u^8{%aepd^j)kJku?Oveug3&g-A#W}{rU!g%1%ihITw{`5A!8X2Pkz@gNNu^^VF_`O$ zwRsA|qAH;c68F0Y{j<6)Xk`N8#tb@>RbJDrA#%Z(;q$K{>E2;^{E2@}NG;vI0+%S} zFbaXu$>tWMQ6Ssc@DMVBxt&j2QGSp8Uc(n%{(Kg&1<_W*(z~kJCVZ7j*xoKf;)8@{ z5s`hQC)j>wo4}Ycsgv5pOcAh_?>Ocix-6BHuh?Ph9lk7-ryw8UpI8vgyug<$GOe~`V)(X8hR>Pq zBk(|>AlyH-;GF-h?zaNpW!L130+)T0qWMKfjj=P#ZN$><6ahH}Z@pXc-shLEBn{D` zlEmfX$C*NnFPW&M8d3Apcc&Ej#Fa?2F3-l%uK)2ViGUazFH;k=MM7l?MeygCI(!b4 zAQ5~B(~J+CSR@p)T%M3m8xrs{qSBdejU&t&0Y77^&sRg<)9!sDojM=)@}6{G6j_pK z7wHU%Z0%m7{T4WA$Sdp4q4f(KG~!h!jL{YZIt`caC&0l(7p@&F>L7XDrFn+T~*@scRfEE=AZlcbHVh`?8oxDfy=f_Ax$M!5lsmL zsai_p|7@2cR<`5{X7p(l7KEN}x*Z(elrnHj%Zogv^}ZlDq`Wvd;)qe%oyA=y(oc8x zChOqtG}tn9)E~k2knX=Nzbktm zF_A27CnBSLTmCemfviTTBInu_5!f!oJLFe(MFE5Y>?_(St<*xiS5e@KckaS%yq~xK z%8^_RTi@MiOdNT@#?Ds@y)Q0hK-bP!vknuNH4tLwt9|rN$H!SwE$jaTE9^3jjHYN= zY%S@1g;YB25gQ}TvCZ6PblBE}*iJsKwIp%zPC+OkgIvu;hoD6W2>Em3RXc2PuB0(ETmNG`<|Eo)4OEl37P{tNcwpJ?=>xgeNp;q0V9y@ zz|AK=x7!G0`!!=4=$^59OJ|4?j&IoJvTKh$941>CT?7kC7VV1cS}W-qozE(eVyB0B z%&;|SW%LkiDA2*Bgz2?B9=C64z3XvOKStsX{|8v4$SCW3$c74MV_zhObT>08T}QNq z%TkWfWTk|Q_3z1soRN*D2zu#Ro@;FK2;@o-tLVLY7aB>_#q-=9ln;6rM*^lG1G4swkq@yW zJ)dMp8w_oW&aw1_d-Y57M;l3X^i{nQ19Staj4ml_ASLJ3L1W*SHIUBt>gZwT$<|A` zdv)}&AIsKDYv}t%a0Mt8f08C-`#j`BSVxJuDTVse1eP1IFoh`F=T)#v%ai({XSeXh z^!@s_PYmfjF2wwlbFvS-3e?Ve7MvkIp|{fy$S!zvykUDErmX6Xwo(sG2NBEYRBN@IgtZF9rN9sywhV?BI0Yp1Xh zT}#-(lQh`#imf0#=1CgufkvKip?*g&OOx9~ch?JVq`HPS^B&UQ=!MJe*Xs=Dic%sn zTJ<`^xXP6MbTs9==X4nN0JoDau6HGvtHEugJLp|G&obt=r5NbNpJTyVASFUCK7<8p zffQA}jbN5Gw>ia6Z{s{`AGbZ_px)QUjkXYPWw9`0vI`9SoB24* zn4BD=V|;!`p&XeK=1W+0D$FMjh4}`oo+xxD*M#{7u6h=}A>Y8%x5g!nIR5PpRtgNw zd`zsM4)l#+!W)S-)R8`{5+}@7LON_ZJn)~a{j_2XJXr~W52-_aCQ7fuh9!){hMUaZ z&wpU66h?=B42iD0I}(Rkc<;Syskv=@UcHb)mnyUg`xp^@+ar8+YoYi%Z#B1f-e;m8 zkEDf8dIoX=lyVpzx>IjGA_9}L;4L>srInkUl|{(S(hkF);hf`UW(9@egE%lF$Qr@; zws8L;y`9&^v@)hqPu|AAo)5vaQj{@%t?Q{H`{-VnnJ4SnBgKULk&9tVPu5FFGxLI& zgDvYfc#L^%%)!Up+q~&MIZS5@7bcE<`!Jo4xiE3;6USU>;pXz*_8DNVJmwbiKJ=+# zelmE@8)rBXN`D!=H+7<48{CxBE}2bxO~z7ldM_(aN&isBL^8$$G#X2L|5H|BE+3j; z1W2dZDPyEZl%GU0UIst~H$X`+Ou0q+oAMqp7eVFZ`9l|BkYZ&mc#H3Sf`(WW6Ss`y zsYi|v9A4Zd)tE#(93)uX>v_9(n$;~P?1-yHPGRwu$j{4$X+B65^iv$ZCtzP1I7^x} zF1Z(+CCz&O*SUt)MGj0t0{?Fbg8ah73WBqU*+)i}CxNpFCLu*G7+Q+NMm8lmBa2Y> z(UDJ+jFBSJK{)JCcVrqk3$o0Zs66?cJ9j9lRX9ZiaRM+Ww|Dq(Wm{vMjEc~M3KsPvxV6L() z(a?2z&i_u*C$Y2Xx5dty6@6OZWb$vUWV;s_gO&$9O-z+*JVVS7cw029@+VM*)$EDC9Qscd@l=}B`soqd_T*+ zafsC}c5#2aL&x`JzBEr^v566SC08>eD2FHW308>!#AdYu@ z<1156@HviWx3R<&7rcbr(D@U!+?sS1x$$LE;KHez?4f7gOZuOmPWR7sqQ8)Tp_Qni zi%mBeV{?Dd;*oK8gTFSBhaR8{$@6G0YUrmX*bOyhQJ$dRk$<9PV_Eq2nc~i?ix6l4OPzyS$k)dFIjZnWYu~{FFUDZs-ti6?L)LxPl=x zT3pV@M}^f?I0Vizq~X&u2hK87;qx=w{#t}J)*J=`+=^2P9+=r6wMK+L*OjvvDmnhA#fJ7L5x|?95_pUKDA?V z4xA<5o!aqo4xAa~^#3t+ zC;m{c@BhHF@4K-MLTKzmX=oZnQrRkm>?DfHpc*w~HyB1C%AhDz3TYW9$ueV*$Z1nK znZ;JAW@s7Z)bEidA8nSMrRhfTELO*7t zXo9n7c7%)`PrXND&LAtl|3$T!PpW;iY^_Q98DgE%IXDy5PM_IR`Tc%~>ACyFo|$ax z08)i@KxfFuMMKRyFKC1bhOzaF!eiDi?oY5d;4DSVP%JyDVPcxIL(`{|un)`S1Zj?S z5+Fqo)lsAw+O;O^_;TSRE9-lOhpoepEnj#9&hqGGXg7NioaNEI&~9#0)5OeTXyyIT zZnL*1mNOqIg{H5;9bYbZ+~rk~*F7j3fsrGC0o@P^@R{p(n~*r^tOCO1qUoMqD|E}heJS896f5lAFW@ZSCI(!CmB3k)3Zn}b`D$3|JUGix|7xY|P2en>s0(a$C2$rj^%GlO z37kcLbkTNF=q!}Elh`q#vrsfoBH2P`VR~T0-V2?DX@L!UCv+BOJT~kDILqJMulh%M zLTC9HNzQNA2WLs8z3R8p2WLsunv3kp2WQEDv&nN2oaJf%ekETlI7@2PY~)}0;4Jw; z`mv|LS-AKT=BZ7ih%cBiE|CJxqOW-}WEz}>gTKoBxe1(Q6OP5-LIG!?T*u!0CMeC{ z7mvj*uvL`6S(H9(O5lOBjP!H$YkA--i@eMEO;~UiKCdNz4hzo0YkSpSqz}$=ipbz< zP{3IzttW|(!CCsbAM~eq;4F{0Z}g}5;4F)Ro6LXo!C6kN^lQT@&=9b^ZkpFKY>|rAqKTd;o>-_W$Co7M&Hf|<`;~=23ahf|B0OdW~r+f z)2RbyNzjH1sdE=B%mTBpm$ujOfLY!x#nn9^0<#dme8rO~z$}#LG~^sG%Uh-l z&ie~6%XD-pr56Xxg7f}{FQfpo;Jm-%^C-Y92C>UppM=aZmYh~QCuEi}xwH}t+DVlGe>hE*b-784F13o&{!MFCD8pDgb7gSvp+zSpdwk zOnfC zfLYR3*wGXUFbnS2H$H`O`rp?*BJs~GqDbs1shS6VI>C4B6e5ii32bTE<<&*0x*ln7l@xmN5n`)YP)EPVyR?Hg~W)MKC(?YrpG+Os|T1xL`iPK1ehhh$5`~Z959PCVgF`v zU>12oN>7*wFbf1iAaEkgfwYU1$N{q$uyR(JNdmLT+OF+kt#XANh?x||1eoRaswOFn zF)+)4j1EzGC14gaR;S3A95BmjR<4M!2gfm!5j9ec`G0keoSN>u3s zvuH=`?Rg~v%py)zr-6K#Q6>{!<_ zu8ZN+FTS|2N=b^a2AIXV=f^5tQD7E19wEa_6qrR6tIfK)3YbL{tIO(K1*hm!X*evskgNi-gMovq%e0_G}OZW|0*f?>Qj~%p%&Yn|PuJm_@`{S`gU-%p&3} zBRH4=%p%e)&5OwZW)bO@;T_HZW)Vr3=0#@!vxub2@ZvLoSwwzGx0wO6w5~cWB{1uG zxaBab^v?U7Nef_>BZFx!pLBs)3~p^1%rgXLartDhm{AJMVpyG%paaaJf6Hy~oFOoa z;hUT|U0@ai7nw_W^Q&(wpOiKRX31-FnJ`4+cS&!^(d^UQLh?53Q$9m-U1R8!zuoo4 zu*7RbZC&(<+AdbAefMuy$))fLZdePWxL+AGp;ir@124 zfLRPU6Uxk7U=}633rWotm?c+BJt~o87!^kRv+N^1b)~8Rv)I#v4D<4VS=1IM zOXqTdS(F$1N{wBBSrk)u40^Z%vnZvy4O+MYvnZx|4aU0yv*dN*rWFi@%;HLxK|27m zG%9b%rhQKVj#EsJ%4S`t^w4!hjWgBekVxNm%;*yWm}LV8 zr}DiRn1wR2I;x)p%%bw0LfA|fbK{b{j2w!AStt`mQIANle@i-Sbh{Xsg>rH0F5?vC zed3!SjiRn+9H_D&U4mJgrQ2J?Dq8K*BpsQvovw0Ly2kjjN_>pU5=qM4a;tWGVW1i? zOB}u5=y(Aziw>Ve`o{>E#YkUUMFp6}SYNL_fd01L2l610Hxb4_5^}gkz%1?7>60~^GHHwQ5o2(r3=jBn^SE(sS3;zoP$>xa0h1D zM%&bPSsIwd*t5je1ehft=b%cXJ21<(Dg^#G9hgOBZfn;DPDn3l$!L@U%%X{pq`RpA zv#3;V>v9HW!E(e@49LJNWSTkWJxR%}`HpE{zlweVFiTMzt}jm-n8gUI&n}SGWi(3r zY$KQhvz(V6+eWYkX33SVHtx~_W=Wz47#%49W+@UJqQ5i(W-;p4=NFTJS&X{%_%)=B zjGLs-+a~pZS<>lBM(J9>EO@$)QMwK=OKOgd5m6hM5ZELSPo%dvjI)RF9Hje2qrqon|tW!-Ef`aT_)r8sr#$PqL!3z|YM zr~+m|Q%YW00JE5Fo6=&^fmxUshXFhqm?fI3%%y7qv!EpzT1a3P9qzPN6CIdEgZo7* zmJZB9#_uJw(ZDQp?1tG?G%yRj%XXGj1ZZ6);P5n99;{6);P5nA+0oDqt3L zD<#!+1ej%BD>{l?49tQWc#J+$0L)@FK-1zz1G7X=>PPP(1GAt_nN_O5EEWT|wVp-; zvqX;+I&K7JsjW(mdaIj!Gh!qaJ;wxQDeAPHt*4QdYF<|k zFpFwfRpQn@U=|sh9fX5@z$`MY+X*p!z$`MYI|zsSKF|kqUYd0o3YjG(3oSEX3d~Z@ zniapd2AJhsL`W822ADtA_m}OMFOwDr#FiSYz$`a%j+zn8fLScq_hn9)0khQcvU3bffmx`$vYa|oV3t}|k$AW%FiQ=q zT71eBn5B-DFYas#%tB>Via#_3W~pUS#80ULv+U%Z$U3dMQvFQQJLtOi@Pj?TERB<9 zGuu^RUa*Dky``IawXB1fFi0~o-a4@R4Kr?ovlwUTo-f0GNyKsRi3-ncP}b_4={^Y zYT&Rn3Yf)L|IDr6;ZODM!^XZwz$~%D4Zh!Y0<-KG-@rhlfLTx_Uigqg5spCJ>@o%o z%(AC&3reyHn57Zp$UvchS*~CwsAn%L6WV*#Wb>FMoNt%K?~W|L_Ojx1~a6abi@dNBaV^?8GM*2JZo8@jzQM1nO#Z zy86AQs&ZhK*uo)H<7Hr$ooFk@7j=6&tak4i^#*1!o1vC}Z=_mOC4~(yqkviVSbKJ^ z1!h?@^LP38CSaB(;_8`f6fg@a;7Ul%;wAMvFD?@>3}X~93+hGFL@6*!U?I~Ndj^=L zoHL~Ea|M{CjPpu;>9h%oBSpZO?IZc17>p zF02}u#g6+``{68{Z`t*EHPS@kwjQ$iOPFp?>P+2V&6Y4?`sFl z5>s_9@+7RY9LNjOj*SIoDZ_gYoU$K9j9_fID`a36ZOsEA<-jaw@dpQf+5@xL<4XCx zWDEK*`MTZBQNb=)XR*Ul*=OYA8Gp%tuum8SX0hz&YS#_|vkde0=QY^@vkdVP^5*P- zSq9r2`-`-JSz?JM+;ik$#$9sj0iqQ!i!t|u_S7ISiv{MDF5-Tw7uayU8 zv0JHO*UA4ra3?eQ#kI2JRw^)y?{R2U>3yWLzajHFpHc6YT`C9OBcKfZY2lIB6k<{FpDK2r8i6hm<7Th5H=BJK-wirEPz?USvjI+8o(^( zw#_{(QD7EPGfm7DA+v}!X<{0L%#zt5D{lkL(!}bN7_$IoxyZ_ua90Or3E`d1s?-2x z3F8%J1!w@Xgz_$Az0?3^u_UDR>dFe4MN`W+B0;##B5}kz(L=b+A~Ar_3I}EhWPKAY zlLcn6wC(IE7X@aKXhc+n0kZ@}?CpIe0nDOK_2s?IN|ultwZJ?SGD~l+geC%*MLlF6 z?-wlpE@mmI6Wf4UZn3UqU6;f80keoIX%a32v)t_YA*w42%woYKWSYqWv&dqDSXV`X zS!A)ntWHs27TE-AUX*Z~MIszUy8_H|gLPdZ+ya9$u23cShbHVZ66SBZ8vfaUn zCxqK963%M{k-}{j31^hxpm3W-qI)ebM!3x)(T(CA7H+dhq_5>g3%6M$(owv4;WmrJ zueEJWLS_*?ttn{gdFXbSb@tBt^2zH$W*JJ``6(EfCH&Twp*#~{mYtu%7c=Q!MhZwc>uFiX!%7Y!Zu4>L74}2kC0gko_bJs1G99{ zgG}-&fmwVPC(q7R0JH2`>^p0`6PU$1b;pp02QZ6Gs@sr-2QZ6us@G7w2QbUIuI_0C z6CtyBkWr982WC;-P)WNYWENHKPO=R!izWV4!44B(mUC4hQ9EIq<)-S>ot`$pESBi& z(+g*TS$2-1Qjfwmi^*ICH5{0wb-M3t(N18Noz_;Vr(v5Vbx3L_E)bX{D9L~5xru=} zFiSzc2X+rIOF&dwfr1GzO9i2eu4V$vVp40#_Y-cjnABSFLxkHbCJvQ^b|JHzZ7rYN z1I)5_@%h=-3SgFU;)7{j6JVAK;{EB3CcrEvQ&ye#h0Jo+pnRoG$Sh~;%2#d+nMF0c zTyRUsEUHuGg1bUyQFSgCvl3Yn$gp~s3ZXX=(iB5bpmfAR-r zxhH(S{su5h>%_&V{z70D#P?QME(T^P^fGt20nE}mQ6KfF5SRtFgm2#fW@)|Xx61;U zMdAS@QPlM;0uc+cC79-{;NI)3qKhs~vXQXOf;cN%V{sV~AA?vblnSui8q|I*&{xPT zjDGXuSAkjf@kxdMm;j$YIfLSc`L)!yjo5csRAdoc?MnDpBx#qwu0hmjD1qfgk zbNOqwG+>t7eM1QEtH3Nb`raW%n}Jz^*e$aC7Qif52}c-9=D;k?gh+-80+=PB>oVsG z0+=PBtDaMV0A>m3YUJEO0JB(xh4b?Yh0KB&@Y4zwGD~i?#pE6#v*h9t0|CG+dub2) zF3SS5Sa{yFy&`0m+=Gb50AQBARsQ(n!fh7BoL|>HV3xuq^U+pdmH>Pt!wmtq+TEZ5t*`|@OgS;hR}7FnOYglj@( zksaGhxG7{7*=mcfKq0d*0?dzG1!lP}IK+5q4$NZS9mX#%1ZFYs4&~Pr0<#o;-a8p8 zWEO^!d3vCbSr|U%>HCDtA`A!w37IALs5voM$Sk=A<|~0hW)Uv^?-Mc$?D{WU1!lS4 zH--3hm3P!%mzHm>NUYQO6x{)2}s&J6DyvK+pqeB<-| zm&fk^mZQ8m_TXlCnCyF0bL+*-xaNyzvIb-?kQ!w#pyF;`%#FKxu_Eh*?CZ5Hao5{w z`o76-B$dc+MER*7*SO~5b?IhXk*)d#S*zT;w9S_qlx0Klmt1@+69Z7 zdnbI@>t$OJ|02Sgcqbv1q9A+Jk5JZE&9IU^>QAWdd&AhAdtz;uFMBmQkgc5SigPZZp@q!rHA$%mWPZQC7nHPh0XNTmrt|j_! zVZ_UZ1@Q8+Qlq-FBxK+E@y-!~dtD?xuO$R@HL=np4Aus&(Xa@tkGPrzTYFguS)5|<}styWJrcCc94%RV9d zP}6zs%5Jug>|N3g+4Q}ne`B{Vtm{}5YT&C z_8aMe>^IazD=TA@Zg((h>Uq#bZOejN}wcoZ{zl#lqU}-PJGtl16^`GlkpTEL9Pq{+G?|&Rk#X7t&I%?r(0rd)&+I1paEF zzPYt8zH|O+ePyAA-xrO5YqXti1{G8NZ6L!Z|;0U&qiH z*bbkutmyEECt-Fi*AG|vbz9ChQ?D-~Dq62K6TBE7%-`-p`-}v%Z)Q}Rzg0bYJ+PUh zhk$sspRj}kQACeIc=%cRw?l-f-31y+AVO>Iyr*;D47|fz5$~6NE$K#JQ__RsF~4+Z zvsu0}-0F8kIsE!$b5hyxQTl;Dtt2zwIZ_qQ*H(5gG}`?cW9I3*+r1J3_}2@k{8p@g z?cz5T>X{!e3^o6?hkvi|H2nzT_cehJLm&QM;1ECkk-IEJ;#-;<`O3x58JBdFo)TyS z##qGZ#mqxkZ!VuBUTk)#)=_QVA8Ky8)SU-a19{tYbmHTTE*y$$ywsVeQGX?Gua2;rq`m&ez;T^7 z@e8Y+4lP{XJw6fCth2uOuL0}6Cvk^vuX_z4kdke2DU4j5O~u^qONms@DT z{k96<<7fesP3&|9r8q6?ZE;#xBg^xo7_K_o;~O>`hk9Y@&1 z-CA5m_tmL;As`P7W$84l?-HY^7`(2 zzP|#rkq|=rb#-P7mUT4Z2)_OBM5cn=`mP4Hu#M#Vdg8^%Gl4CU1$kF=&J_sDNd&w4 zp>cxit`iWy5=gAa7Z3QXhp^kan=b|D2ZZ$`51$Dt1{9!;M0fp>zXin;E&Sl(se>zF zD;N3h#m)yOjP4#11ohkK+%5R1lm47noj1iW(E0VO&9xsgZvEjpzh3Yt6F1o53cn9d z7=Jr7(G>YGPi&t_L=ED3ihg~^Ul_l+NGGM2sX5KUV2ArV>^@|}QeX}99X9MwTgduj zFI3ImwcJ?RU(-bLJyzSuP0@=b*&m}`;U?%EqdPi{KF8OVEV+e3!^yLk)@|ByGlUL0- z@8FP&Pse3yZqjx&H9dW)P1?nDb28Fv-=324{LFi3VQP4SZ!g*7_l~yJeS_XVj4Mtj z=V_kInfj)ZFsGYO#)Ctya?SP9QobPqJ}fjX>D0U|{N>d8H1)x#2uHGJ!~3osbB{S1 zHTT|kdCh%{x(K}^Z?7hoYPBSjMlU&ic9K5vL93CgulK$1XUb$NS46L$A+2Y8jQltM zB4Il-PA@uzj8OVO*vf2h(pTfAkYA#=2p`1M^LbBgPCiT6Irr45Zrm|w*PMEd?=hT3 z?Ndjuxs)2%v#RMx8GQlC>Me6cPT_wq->PazzBw9@(tWLgvtFLzuF92Jhj~k9RBE=ML3C#yaKZ`0~|J z*pwLk>g4=%8G4WADxDtw-MNK$Be`MJ%juy?^|AEpOB-vZiVRYIH7}Xz1=IKHIe!qi zF&)}NnXOLgPxiBDuGn6PZoXB0gh9M^StEl4e@O z5NsrGhnSjzr>c)O;dPI_q^X=vRaC^a%veV9|H+(WzLErS(ISlZcu0n6B+HMi=M`oK zDD3Szz&nspG&u{4QfRMxWbp`RBAr?QXX`}S7rjjIG&&fDpP^2i>bMg=bMeY@4 zntrmyHx+CVhEq)QF3>)#etS5nYUq^bTZ>QjkN}bLRMdn|)$1IGr?WYRWer{>Xve+S zghiDy$_1h)+1~N(w0IdjEaizi7*$`egM(7@sU{52_is`2_4HiNZy}7**Qq`Af=`xS zBX;OFblCD&v$;SRqCbi%Dd;o}L-Vf<6~6TBKC-A#wnQhE$$54kT!gxSps~}l9J+mg-ed|N9ZX+T3r6VBAD|X~ zK`>00QQM2p~{dn2K3oDPOKXa7I zPG6Xegl`Qoi#BDat5@6?9F|*$etUjWIpy`qzH@3?q(ikWxA2gLV*9R8Xo5VtwufjY9=xme1}c-pci*U)Au2z7U2z{k+DQuq(uggDjfc zhaKZA*WYBE)c6n#FW>D;*x%mHFs;AJIG|A-oM5Hqa2h`fuUmcTNG#5ccRMZj zGKluJZxuc}SJ!HCKl=c_C^y(@@&Nld`bnRj#;4o1+vV@t())JFozSD-w^HmNP#C8e zX&RsJ2{IV$s2s)_4fi18F6@5%Z3bF{S(tByi{kGtq7>jYn8DNtK9mD6%&a_b+wQ=| zun)?O1y)(cMHc2>)u=^`1yvo0{3{pMH*~DGn6#aZ}Z9+R#Dd( zbymNk@YNGX_yI)^gH}RUBKbZ=8!Z6@Eq}%Ej~4wV{iE^wj-ZaQDXJBI65oTd5&oig zckg~YVCZnI$n`Mt05?(BouuPXavNV>+`vFPl-$8ni@DGf;(dGf(`{Dxv&F9+yzk-* zG0z7MI{4g6&BHvM4RRQ}8{|%fu8``vSJ?1pcY?+ar2T0xDNb;>`)cdDJFi9#Bv)H) z^{Bp)aQE)3fo*QVx*rp6-y7eOa(jFzFRGzq*Woya6y4CA$o|`rui6+A4oKZLcA?y1 zf&)=EQ~Ge&e*0V6Mf3v!)mI;TPCLj|Yre5`t9a~@aJ=JHXvm?C{Kt{t3W6y3JH_b^H{0qwczVb6B-^MMYIqwdoL&wGG>x1p;fK^Gu5 zp(~DSqH9y+=rH@5z!>x=PRa-i2-L~BL7@mT0(0ol~&O*Ua_2)M$J+nsb z6SKlV3VfhCe6wx&(qkyGDW(XUNuV=eKnDs*{tKgU?RDX8m;~ZMu7$2H{Uv06CiZi6 z+g}abbLj5iKM?c|P#n^4b-E?oS?+ioo^V9%&Ea2%mNryBEOJixbuc;qPX+}_Ne(X= zw7-E;lHVPj?hAK1$7a*3r51MfD4Jh$#L%}K!#8L5Baxq2(!2Y6&nT88CEmsi^tLOO zBqw%ZdV32L4>>iPcqi?8(>tW-tP;R~{(KxW!rr9#8S&k*YlY5&1bWs~>blu`b1SyccvOyGakrjkt9&8>QIjMA(a+`#*V; zs*fztm26GDu4z^bn%Ns7)ruiJk7UFvhMy4R^@FbTODImIz#CI<*e>~>G4RIJ94{^3 z028uCV`B&>@oGQlNVqC}R#>H$ZKFq=iO3eb z?%gI;jU*a7Y!tlcJ)zjG>5LJK_xdQ_Bi;C~j0AM0KV0wEEDv-A%1AyX!WYaWL1q6# zyf3m)qjOnKybJHoFC1)O(^ozuHN(W|ahtC!57m;7>zQv8%$&e)=s#LW;YI z5P~;kiQtq;%3adrp1?H`0k#tMXR;*-XqaPps%^Rxp&_#0R@sQzXm08aV<94r;!OyM z2`TuSko+O?auwWmdXDvc?OBBvOEgYYOYERY&=52Q7+=Ldt@)zwu3v!Z(#fZSvDuR>o6o&&NqbV~s zV<@XfJ;I#VLQt%d^pt{Cm>$ar2Ut`QA*c{!;NObKtZ3`1)+|J8W^4A*446L>MFghs zyVnho?iPb*F2$y#oL9j=6qaft3f5z!0x}G;U`%BhN37?y3&aD(h_bL5lkv4Dwg<)z zmIuoMe&sq7(g~v7D%F`sGe2j*|B)U_j}J?u9kHUlA_3EB_QR})nQ$=lIQQzYyxZkg ztnC_XJV}3 zmk%+c)Z9lPBb5Q23*7ov+G{sw&6hOV-Y+NcHANpn<&`(kXyXho>mwTXXtd{y!`4SM z#X=|99B)mK5lz<+`D1waD|jSpHKLQ!CcE6A0CbbYI zW{j;3jxASd%_Vg`lCd_>^q~>|W^C028L_TC&LQhNMF*`L@5ASSzp7@U z?iPWJScj$X&lZ70Bz;T3=~8#+^zTC7X%VHQN=|*q43uc(sF`-z6fN*x;uqmQ3Nsq z6=4ZU{NjlPPO#=wryy*F#c|hk?wmA&BAQn-HrDqFK3bnXwp{%PWW@S+N@xEJq|#=> zt$!b1rc8WX46pp%IcfauugSTjhmXXLm_#Tco@0@mOMhXW^+)O}eoxG4X0x{|xu;_G zA-U{8C1xrnjLKqfRa(F4nW|Z8^GEi6B{&&2QD^xn6p)ckwKMz#3do55*jGX=6J&() z75m~lw2Xfdw$o5RMxcVm;X51;UnYj5<0*%+ez(603ulx-MwDv6F`eXuEjZtwxczCt zkqQoB_3{{Uwti~o5!#u1{8GE<(Hkg0Bl@QZ zT_aqe5oVZvD{mq=1hPPsH0&2dz^VB;wvG}U7Wy}r5+CFPjVNt6NgL`18d17;lJ>YC zXe56lcJf_hsn$XyDgToGCnwU0Q^;AqJ_Tq5`}Q+mgaS13rv+sFMWNcx*p+#<83kyB;(Tg_!F6c=huf-udP#6!(9f;YKS6+{hSZfu+$sIvD)%@2 z`oMRl0F6+7Vn2N+^jsHTXsEwHaV5e(kRO;4CL0GdLPWmft~CG}AtFC;F$O>*MCyD) zBL!&0;7%HKfeSQ3c}lGPv;;IV)UHn1 zu>Qhc=B0@neMvntHW+KES{Ituh_gTQ(~=mr(4h{}&42Yc7I=P4A7`_a+ByHOqjJmw zf<^&9x`ty741nixKbC0&OyGIS2V%(Q=nPsfvlw?|5#K>$FmK?FEaC5K&2tqEPW;eX zBl%N{6WvU?ugLiam%rQv!5@hGeE&)8nM*|X&26u1AmY~QFFL-S)2aiW#~JXKo5$dv zOdU7Po3{)_n{#D2(;ro9Y8UypDX~OPjW1{h)6W2S{{N(k3_8+;2{C>>rIOfXIV-68 zz(p9e{%C=snGd)ro}i>#n=9m5}n(6+$BvuDJU*|!z{@Uqj`MtAmE*}?A z=#RrKw!dZq&l|)OiSMIt1kOc2_+Rr3{|cp*n4lc~b7DTai~>AwF!e()$^@P_aQ-0} zn$u{1J~xIt&F}O|_`ZC7Y>N2H`WJ7xX-p3X->GrG7nkpi0ngX{{;@JPr{Df^4p;Xx zO`m5WBGufi!75z^JTIasmq^Gk-deoMT@qss^G4MwrXOIzhH#{3Nfd?-$rjDF^gs19+eCDw5{xcLCTO&!_Gr|{ z^z0Q0mP$~&SbsR#72%Ian8UMB5 zu-rr%GN1+LxI&fW$X5L}X#&1ykDmn8%-qX>b4BEoc$Xz>ral%zMy=9tu4wbFWu%G1 zxgs^M$+{pMLPT!LLCuT^CvVjTNgAr>G3VT>dy+J(o{v!x85>0>wIfQkkP+K6E{O=w z6)B_!uQmhD716f_yv&S#1}Vc>tV9;hm9?z-45*_)C`l35uo^_*ToEl%!WgpPVcZNK z$ttagvjp>=ry{j7wn4VSM3ESrD{{70JzrMA&gH5O(Gf-A{%lW?NSG{~D{2u(G6F=q zblNy~GFqjG#w;w_nU$4sR1(e=y~Ice{lK{*CrIy65`}X`%vy%Fjs>{@R$+#LB%CV} zg0(&GSHZa=+9Ja{m;qZ;8DB+ka&WE~^;C*<%fh*$7ZIJ|B?9M)7*UsXe-)f7qAOag zri@_viwtL}6%&?2`zBVa$Z0t^SBzP8A}7S*T#;C@U`>g@xgz#Um-jH^6#Z?+Pti|G zgw@UB$ol$yBv%(WSMo5HbhaU!D=tWFPO~zcD=tV~4ow-(6&I>j{Q=TG&Sm90u2dvP z9e;@Q)TLH`Q67I_u*0QR?^X%MY`W8MOb*T!-CJRUKMdhqG1SdXK;9~d)43HiIBN*! zieYeWg5E7JQsG_0*ST>9_1W`U^}f<3TVpTfJ#m>g{0lNi+I_NcuDD>>y;J4^AzFQM z?0o61T8s6iy9Wc;7~aqGbUk3C6qM72*^j#B5~g}9c<>YF@6t^!VH%4`#2JE(xPFntdy(SZek;GsrQztS@UA3X*GYBG0lWt(*mY8aa~y3;*Mqbedb_+L zdJX-Fvd~6c73ETW2Ej%QKPxS79%M&t9Xw;`o(ndjo|H0(HiX&L6{os5UJ5p1$jqlI zFT!j7gJ!Ot^5`8{1m~UdSWcCtmM~Fd2sV;ir4eLE?DTn8RjVQLG>5a-@BZl1y{7@3i zw1(;OeBVJUD50ry{iT{zJ)yt&vh-mt*oeyFSm{O=uo0JEh{eyPU?YaV6_&#XH$@#C zOf&qRk5%3)TC!`mRiB{>*oZMwgUyqMYm7<>Ingf%Y(%A`I58v#Y(&MIyvyFj3cn}k zwTgF1Vj*UI-$4~03Ly_;&k0f)D}lR=8h?^B=LR-X9B32uFA3^Kgv~T16I<66Pgq7R zk*u~t(#Ys;$%Ltk+Z$oo4CTp|6!H^Pmg=RBUC@B@MLIM`*nr~(Hljk6Cb0_Tj0q|$ z(wWl6Ve05Yt)ewiFG&aKp3^EtBu#fqH-EZ^(eWa<*yyyo(;pdujci*$@C!*`BPy+h z6J{>q_+vTgDmM!fZ_FF8|B`;`4mLtcjLxYx0vjnvjL&&v1U9mbX53e!0yi9$yscU~ zY_O3N^hlM@1%y_b4d-7K0}>{L{9hpr*oaX{3C5HY$WNrZ8{u^LXtXs)KvFXT8zBky z=M<>GRmTXc$38DzCSmMZV7tDrgVQPf)X0-!>pDWVwdh-r1{*Pgyw4SBuo38?A(-@C zSFs2y5}c%OFajGfwk8P<(7{GjG`A7WIPgp^>9hOFYK~Dm=1=_$*og5>k|2^UY@MMP zwUofkXHDM)>AG!TBW{5P><($L5n&YQx-{6xpVk?+Zo6aOJ(cbv-UESQA4Mg-kmi<9 zEJ&t@8-a}&|I*>#Bf%BO=$AI$oesef`a`2%dVGWSjhu%nzl&(@3&rr0_(hVc-`RsU zzsA6<*>(&cNA8b8>T{)c8xNd8mlWb}V@wCy(IrLrE{x$o0s7DY+>e3F!_^mE%}CJi$c@wP@E>^r zMBN;x+Yky#Vz3dk6Ju9YipC*%2plyO*+x}C;OH)Gb)K3SIs_jn|6c>nNKw=_bo3ee zf{uOh`2y?#l_gHqmF7aIm=R`-JfgpmQiDbwxs6plhze8bxJR>_^+*4t!#%iU+-6`s z8f+vwL=!4zz(&wV$ar5e*a-SaF+Q05hS8&Sf=;s^SdGRElxn#~3vHyt(Q(TH6!LeIpQE8YgdV8IeX0d( z-dZ1`i5gsZkeh51O+-XzLl$U2t2zp7geh*7SW{cY$@%%N-){0(< zfm!`(NW(*GFm}DWWlA)dYcLMIc$pGSvo#p>mdE0-c_MSj4CB~~;ye>?GQnfycnlGSP}dBoitA|K>H;7D0P&0*&RCQ@yDU1UcN6+If7YZ6sgXIfZ$vP^ht;zpi?-F<5#EzjnJ zsx`{kpE0vk4WqWH`$O^vYG_X25BBNH*mw{+=g+qV^jWCFXlvSK9$6<2qpi#j6Pju7 zo%+*RA5=r?A~I-(P%m?2XJQA_oH?doZU#6-L{g7$!;q>@pcsQy6V` z@=~&5RXsI4ZB2TgiNk2C>gi#-sTVf6dt1ai)L^vT&Wp;bRE5!YO;t6kLcGo2ZKL%{oe-6$XN=d%tu31cFd(jrR5I zF`3glXkH0Eg805ERTyoz3u629WMH(F`MpzcqVF{QQ_gSG6&tpG`v!J*&QCS{?dBp) z&CTaHrRp%+q7*&wghCi?)!iF0<_s8Z)tQYLYX*$A>Sm42=j&m#b^EAp)&!&N10}I9l-E5lx`D@wv7qrZlw;xXgj_Y;4;DqpiBP?Bzs8E~-mCy>N35%4Vnb z6|&peTk*r|VoVpy-G+;N&()jHn4s=o0blWYWjqsu0$+iAQL5YUTh4HKz?FqE@D(Sw zhw3jcgRl7DLkmCng0Ebrp&4E%@RiH7^^C(P@D<<5Gn`TNY&uKbb&tH4&{sCjw3UCq z48Bsi_^Vv$GWg167U#UzY4$GjxS1uFn7<6^;6;!P^ z-?MNiOBg1)0KO7Zco5at1isQ(RS|W(5PSs{=E>hvSP3;W$-Z4@InT=xtyhSfW^_^D zD=5v&M0@a+kL7ZZ_W@rC8~*N_UJAZKP1-Sh1qHr>3b?#t3%+8`a7XQJ0$({B8)DsVoi>-r|_1YeyfG}o}F<-u3%spt4#$lxpXwH5pYGWd%2 z*keN2z>?cPjZXJYc(0Tqfa+)1ngJS0^*%##D8l-*mQDRyiT!Z8{ z9H60qSI)`bJ3zAnUWv%tXg7Htc!e6dJ@1mX@ER10Jjbsk!!^k6Z5=<84A-DPO*F!5 z5O`KNkhQKQYWS;|7RlqCe{ZF-Phu|92UoP)!57(eawpRUL zqdvoM4Jv~IngF}53+{se=iT$AiI_!-oZS$2IYO##yt{VgM=xdVYmj> z{HdFPYmhy$ntfj$u0eJyRqQ4*T!YBYu`4FPE1S5j+NU20uR+{8?GyXq8Wgi)!JX2E zYmoh~YW_nqT!Y9z?LLhWR$mvVHq`GcaZQ42&@9H1$##TmP!jSyw;2i7pd@59mxhFE zP!hGO;XnyogOGQUsZ`*VLnTj>YHNX44vcmr)z$#7n9X%Mjy-~FP<2Nbw9>#e$WeDD zfeO5Gp(AK?))B5jj=?htH630hg?Ala&&1UMuT(YoDw+VV%nGwca1F{o4_Tukxg?BZ zX+J`9miJ%vP1oAka(j&(qc6SHy0fh zGRk)}n+C%_vowTMONPyp8G;kj-qBQUw(4y3p%U0UacrEas*TR8Iv>5ag1q>;z`oh1Rud%Mp43HiY0$y`Y3(qk%Yo+Mk3EPIc_=85Cf*z%T9wP43w7`xT;0T*1$q6IvDCUN! zHA}^>arZVe@2q#eS}Y3X9#;(=#g@=bWQX zjGj*ZRhdkgs7?+a{nt@xx*|i)&(9)pd**x26VZGGrmIJ#ep{xo9MYc^hxg2kmO}=> z=Ny+zHMwV&yF93V7msI_$${`&CGP8eDhI-EN!Z)lAqT>*j=jf9l>p(Fdtnjy5WeZO zMKokV#7MbuzVZ>u&8z#eG!bOk0VCAJorrz5YN@^TEQW-~Zhr{v(prN+W; z-mNTHJdrqs=&FYgJTbl3VGa06KuGAt@XUBwSUl0B-R$`)0gETHlYy*8QCK{Yo!rN| zD+-Gz5}(#4#%6AbO3XBr$kPxmo`fa__MVc3FFh87)4fu%uy`U{Vu`uN%H~|kYLF{I zVfJD}cxSRG8nAd`f$3)DWWnNzEG{S_Dsu-1Qazf4#-9DXqQ0`8R`AK^X3uR=D5O!g zmm4v+z25^1`Jy%&gsVM|qP#OZCBjg=L%oGBWxK<8#aT=Al`J{g?jRnNCqlkx3MIHn zXk`ggKD6zY*0js4zyAb{EM(II6R!mNvl>LN%faFas;#l7Ndgv6BwlC=THp&%k!ZMv z;98F)|5#?RgaZN=PppMmAX!*E`I7~bg~b!OU;mG(J8_44|NqB-PBqln3WIEwLdIUw z&}6G9l-&Qe*jD1c~qi8`XODIHHXUr1Ha$1a%%&4?TGxUzm>D~D~Ue5dT zyS{(Ixvtyw?BRZ1_qhkcx6`jU-!pque%_h-Dq>`&t4w&1ZV7zEjhX4H@V|mgmQJ$% zLBoe9GINIQJyRK#ilp4UC|!92k04EdQIjH_2_K$pB_GMHa)u9249GFL)6VeW$yPEk z_p&a0c;fubgdCR(AD-wQGE6zu4fVTs)uLZBbpP{>@p*KfY^yhp#^7Tu1Et}YQeriV*9+aX+W4)(kk&=29t`Q6UJ zz7Vc7L$!!d0^y2Lici0dE`%#C-weqIb0J(&9J|oojf5+?8ZHywNVt+4;4YB}*+3t~7A|QoMEn!W9qEpkl}c2vXeT%}CC@xqXc8%b*b}f%==FrTrKs) zDy(G#QE9X#?@6toGY?W0?EPBeXYBnNh*_{Lbq;@H;f1LisX;qqtYuo!{+MrA_=^bh z80+`XOsUP!9<`siGyiyc^gACtviI1?mTiBgwJwSu#3#QdC7)>Tn42?n-cnJYbI;jS z{n*Qj14s6Le~gVaH1u3L-L-)+(<83GPXBP&z7g$ie{%FzW;f6v6us??3YDGtDt?G0K(o*K0Ek4b==AYNtM-c|K;rfnKb$=!QE}5C0(%v&IOx z8ny=KE&61=xEg&@#S2^RFEqx$WJI0#JNoNw(&hGMb_cdRSO`DTfsXXXHM0RLW)jLG z82r`TWJ72;>T6E2A*C^M(QwI*>c?`zPy<=|qNiw^x~T=UWv(f9>0V|dcFCXW z#d?pv?=8L~@Mqme-zT*1nXM-gvN{Z5S0qHJfcfPobR)WD*%FpqvS3#Y`?HJPh55DV zS3PX=r2DjoXxY-Dc83)z=g0-X!!k1gAeu4 z1`66sK)qH@AdKw!<~`!_yYe@c)5_m`M%?Vv&FzGC$O4VZ9%XqSoGbjD>>%8qRizAe z(3q@RwoBIXB#x%B>%WsYCSJ%S&XOn&m67RgzlDe*Y~8PXL&wX^(Z1pO+MAhJ6afox zJV~vYWHc{7TG`bdnWg#Fmy2#hHfQ9B_t+YkNtmQD-zW6`-z<$d30g+wdUM_sCalmv z^>u4`1*sZQgZ{caGs87PHDmBMLW{0v(lPkT(1y}mp-2}9DvJh`K!}S zx#BCP4xro&DE?usk;*o8Vymn{?gBc*eW{WSKVHZ7K@$uEH^0DA4+sa*W4Q$7?pjh8 zdyMlL{lGX<<1jlK};4OYDG6UOb|jbp(| zWff90(zmX`g&<=w%BnRV>R~JftYBVyClE%#%BTwpN@Lu}!HT!M zkCp2?u<~yIg(`2bvb=}`Ry+yEZ-bQ!7o7ucZZ`4ySo_TztY9+o-1luE!u-sBZ?IzS zxUZpd?ae$ahER)GA-Cp{v0%m9)t!7N53z!6Bv>7I2UZ3&d?s(qgB2(MwcSQ0jOLRr zB;Q7?q%a3QV!_Jo@`||X0kDGoNjY3Pk~I9UTqLQ;V|Y{RMPMaw*qhow1S?!77WTHl3O521;0;!41=Ylk z0kARRitc*}9dBqlB zB?`Z9x<(zWSX^z8$QFW?LgO9x|3u!E@aV!=3$S8we@tLMJ+2N`Dyp*McYu}SxR5@S zI#@aWpSe8U>dCOGAse_f{l)yd*Ie1s( zTr7;9BkzjvfSlWLihp=(q=ikL#HtXisE3R!Rr0_J&j(6DU?qxrV0us;tXOo{lDl|d zMI9NOfl`pM+{tLcZ6R36UbN_J6vDfb&9I=>3&BeEum!at46Go7Gk+rQN?(LrfCX5I z5>$tE@W2W$NIf|ku_BQ!w2k_CV0Lt=nHORKR-P_(@VeB&O4P(HiD@BNDNK+4=``Kr zpBSq3Jn92(zY|z-iq(?-j9Bp?IPKN~E7tN)$?Zj8#k%OvQ9C|ZDRQ({|BP6f#$Dlm zMy#Y%4izVWl~4G6Gd0#=CE;qrg6tKra>Y3D{-4OZ5*d9ZHUX?8+#egapBc9XD^*oR z@rhvNBQAuGvIZ+3Kh#eXwZMwj>x6_ccvqf{x=1I06;K31B5b{<)Pa ze1_mKv;Rs}0$8!eT&W!bD_&#AKY$gadUOk{eEjq+0jy|c6uCQXA!-pm>`wqI=8jGc zRcmh+VYLXK5G&-?BC<7DNpKw@-zh?@ST~MY9hd%k3k@Ke9D4!KQA(>Izt~4jK7T`Ty_KfR#_w12coxUBrI7+E&fE6ed z(fAr*#f(zPDwYK+>Ui!{jTBf}E)>zg3Nl)=4c?W(&}dq$ELf4fzf-`P8kefQ5`MiZ zBVG=ytjE3SMM;5`^&euUh%3R$%Ga_9JK5$-#Wseu(p4PRZ~Y_k1>`ZrmyvN9uM#BK|5B|&Y! z>sUbLq+$$N2vYjW9dnJP*ON~M9m5gCo>It}4*u6}t z?rrd{@OmRw2FQXHbwNaE2MesQf~1l)z{*LnG|g5Wte7t~vqEIS%J!uWR+ki5QJ;tu zo6^7vEnVX$f2zkZF|_!3R4M+^5wLP3wph9h4^|G39obzBR`e9Mr?mHj6+Onb@!q#! zrQcCcqYMvL=5b2=vSVO{T$x+!T5fdtGLFls(Stb{*YXo0StYQdWPGq?8yT$ZjaG_v zZNYqTZP_`>;*9InT?xNe)gOQGs=oCl{F}EZJyT`XOS3Wa#A2|5l!11RgO%~YEw1pp zK(60+j1M0HD;zr|nyah|d{?l;{4octaP})zxd!TB^^}!rbHR$k&ahH7V^^h%YhIlN zE5gfS*S|z@#Tor0N45})38k*C)jHd)ct=j$TYIw~TTHmD306{C`^kFz{AkxhW+`|2 zKmVfT>NV^%@8^J(T#bVhM?_$SygxnMAlD$xOPUtObuAkn>3SD=Yo~ulVuaf z=D|w;qHFibd9c#Ya7~Mu2P^%uUeC54VC8K@ae!-@;r+`K5pf;4U?n$5 zFR2WElJ2BPTFLhE5?GNm=Z3ic1S^t`+%COeYtt_aB1NW3U_~jtY>Ch5(fhS%oBuq@ z5C4b=SctJO6a!f9`i&8H=K~fMg=R`S7qF->?g@HDfQ9R*q7i^#k>F(b0e}VR6crl* z7JnR>&#|W?(eQy;E69Eg`PKIU#Sd3&_%^tG>3wyszl@VZp z?on+PV6krt^#d$27g4V=H=8t1`HPJJOMV7-gt&zWog!DGYMo{rk9gwl+M8T#KEWTb z+@rK|q5hK}ZFH!Ka)*mxQE6y1>t_R&|Fw+FVRA;M%@4sMOp;0V11v2hX4Xd)z~WaL z7FL}FSXAbak`Z9ZZ#K3$LN2<_McPGVz(VfkqErCOVd{j)As?{hyBL|YwgVQ)fqb{a z)Rzvekw*Avp_L3^Q3-AmS7rg0EFTr$0Kno;>1PkB02ZUpsIaaqz@jqQB(BZ^Ecqt` zCXPt}3wP0|`=kW0a2ZBvF%rPS9X3ikK?W=$rb^GfCBPzz$PX|AEdCP_#1D3nu2_&$A=@p3eQf176VpdWLwQ2^S}ykzeZIISg};rs4WC5 zd)mSTU`6Ai+^cM`B77;10V`!0PevlP5X%Sx*BG#3#fv!6y!Pf3Y#HGtVkM>Z3E2{? z#2ji$x$^|EV%gAU-p>Oog&MIFN2cLn;dyJc2@osONgBxl#0r^N_|X!q2uj1ks|&%3 zfKlf{_slM7(w2_q&gW&x}`8IDOi5f4`0Gc9}WArH&@2|7ly>Z!_qA2DD1KXD=!6+)213=MI-&y5}((Tlo(2X9)-g{3Ii)) zv2^KOV1?{A7PgxXR^%1pDeZk=MV=8S=oNyMK1X?tU0~(27$?o&1y+z%m|{(^VvXan zYUII+X8GKdtTb4WHr8s1L*5npXz5r@u%g+LJi=m)%hz3*ZmQ~w*8(e6_%}k7JXo=s zorOXWSfRhxOiG4#B|$K_NfWGSeoqt(hk+HA9Wp(mvPV-k(d;n`tg!Y=S80M3d1dKZ znoHL577keXcMHcP9@)YHEA))Mk+3a9IsxaZS*;U~nv41QHYGXTS=sR2W)K11s`#%lk51 zBYLxz&0A8@^**|mjfm7xIzvM0Kw!m+IstVcutImyG)aVaMJPT%ceA3tv~P{n#DAW$ zk_IdC!6b1d4Xn_7CaAYfv7nXm)-I?V^Dd`AL$vng&*2ZD!(w7_9UyYIeiY zjHIHEp_v9tGm@IVVa>ET5?B#3<$L0gcSRThqcvc~dg48)g9cV;LGnqvz{>y4&&-m= z&9o3ruo5rspmoWE6>GuFl&LgWkxt*W#Ao#!N(^N_k7~z1Ito^f#xkWl5G#krj_ziH z6%&Q{l=eZeV#0_U?|lbW1|3Z_IuI-KI8}ZJVuf6pUmOTl9^$y%8WXS*SUx9`RRt@m z#z8G{$h)#HS~WHhtOT|skFvPqCSc`W)nI%OSb2bd^A1*GlvN+h&dw8=V1@ZQFew?{ zm4xv@-9WGs_&sra_$XN6+Nsh4!3s1#%pb$grLejCRjUHQ3Y^BZ`C!E%IjkM5s9uzL zm9^O)# z<{+E5aIg&0gKR`_u*}4D528%K$^+`eI|n9MVY&pGB*MEQoIk*Hdq91;uQf6d|5;?E z3RX;lNs`KZu#)cseG#zokkZc`GyyAtou7zZ`C!Flk|e2yzKAjNWXHq@O_^Oq)XvmUm2(o;c)Pc^AP92m~t+C*H?(^b?KzPc%6J|{!9NNIE8(#W=~S>H@f!=@ z%>XOf3Ja8WK3LIa%nN#_!3y6|TO$>*BEc!}Q^AUuR7o$!gOyV_ZeNWySowDi2COI; z?{1k#-jzMk3bA;wf^S(IVfBq`gO$5g{P^8qB?bRx8l??ZQf6l*LGM`-qaapq<_=~E0j`UST!B2XwNM!o#BHOM)Pi) zw{WlU>AP)2aIegeyZI<>u#!TZn08=*6^0AmIPfEDdv zvAB{BR_H$3zNuj46s5m!P#dg3^(3r|4py`$#o}r@SYez@o%n#&a%$o|xq}W?=t0^^sfZP!w1Vv^up$vR(?jrJ zWkK9Q@6rYmX8^0{f*4Pa%1@#&U% z7X@<;U#o0`VGHx%QfxvVbLlqUQ*eGwPWq_4Ei(yM(Wy3{8EW(aIY+H(b#~MAJmB%hySMH zo=}5(R1z+!mh%OBEo#LD5FN4ow1E1MK#QYeqW$|eSSd?N%bVUHX)Y0wcX^H4RRBUZ@7 zoMH#Cl8w{l)NBGP4&~aSg0)~}t?`}~G`uT@dq1y@bpR_4EefNXIOChZ%DwPM@q55Z z7T)$PY7w5ZU6yG-2d7q=DM6`YunBuRz%;|CZ9#D zkO?^-H-VM2rR$(70#-K7A>|XW@<;O?8|Ap7&__t?1nw2pxU@&8O<*O9ibv*XIHEt0 zIU2ZE*3I|-;g&_UhJeKZuOWJ{7OZRvUM(T!fR!AdO}=!nl1(w-3~mA|4xQ>SPy<#r zO|F(iCinrVaOs&RgcD9yGiwdT-zV z5#Rt;vL{sHQggscPSB<#I$}jcUu&Ck;h;qORvPHvyccnmJsG|c`biNk~Od(&{$nITD(*Y|wvV~?w zkgw>lAFZy^0W0fDR!5x$EB1`gySMOLiQvlGD7|Mwwg`%As$zQG|O%m7K;wtph9j zsCW@O2dw0{=$I5D?~0^9$88_g8uAq#yoT_>YOt~{I7du83s%nhtn+mOD~=Qc_TW0O zqSL7!cKR$>SvQ#@jyMZea!xu;sKLE*g0ra8y#el(Fb+c}O%v{wo7TVSLe8dWwSWrv^D?FSouf`m#5X-fv3)H}hnsInb9`dd@epZVmf)!#5V{{X5 z+#IYlhd+)F2P<5>?R%6tSmDlULQw>)l)fe=G2mS(7&p`-f)(QT!f{i`S2pr`)o4Vp zLX<5uH-dachxbUWiU?NBOVpwYz{(y*7zeDV`O5Bsd?jDV6BEHosr_TKh%KU00>_mI zR;=_Qw42u+eOy~g;2~C05+9Sz!3yzEPD<=!#EN+X!`uM!mAC?p@CjwOS9E#L)Y>?R zmFe$LKtZgK2~ZOOE1c4G;SmL3#e9zO!|pLyDQyn7QI0PPeM}FxQG|O%H6E#nfE6wk z4>b|6QtCo9DMa3th5k}EF4Y?H6(U|^`hgl)F%Qm>5DUOcfseT_AFS{w2E0LYutMxq z4?kT1R?H`JBoPH*rSv3!LJjVf6OR{(-5cOu346>SrfI^xa{TcyF-;rp6`l7?^PW89 zU3ve$G=K)Noj1<8nB{E z#S7X0Pd5aAPTYUiO_yp7`HCXGZ0f-(u(D>kZ~|5`eb)HugB3lB0c&s#SW)b}42==6 zvSxC>I3g3Qz|71;Z9nW{4tKaS^9xcC7&`dIIKD<0`PSDqVj`pVf09ajN8b z)D8Sr$X5)HekqZ@g;+Vf^JrHISTRs=NudmY6$6Iz_(sTA{;i21R_5E4`L_@&WMW>i zCs?V*>2hlfz=~)2RZ)R5SW!0iYH>#1m3^O;V?Du&XN%kDChoWaSh*KI5bp(6ZsKj< z!90wz>do2mc~J>iDS7Rg$st=Cs^_PesJ6r@|BI;US*mmSb^q;xe??mI^0Lf zRi0pFc>@QmIJkw~04vJA=v|PnxCpDoo?xZKe!%SL77+~2xO##WE4`yv;a(Z2Eg@7R zR#FlN$Od4=^U(g3*a5_fL4%vQ0pu%jc^Y05C6KS^a-S(f2L(opMBkN@Zy;94guIUi zVC6>XIbuW}STUG$`(X#IllqcoFPjp$R|e=_Hs|49VaBBmpbWsuO)CB!y9C~q5*JUC zgYd4Lo9{1iyGgZ%e8m%ACVHR@Rt$poONe=3CC>-?B4DMOV!$0V04tuImx-tIz>2}- zen|xMMedZGyfsmZ+$#f%p551wdu4#(nO2V6D+9xxX;+bZ z6BpxB^T0}8kU`Qd#EOWnY+DWQ$^l7yUWg}RMUtA=WdK&H1x!(+GFVYgzqO>#?Kz$} zRq#CO2!1Q%D~91;3Zx^!O26OE@U8-|vR+}3Lg9gx^^5_*M#xvfc#i8eA`vT+c3FNT zSb0PuW*2V*D-k%|zMA!5<=+_?up(=`y=4G-SN43Cjok)TwzUk8Z0Z|d4_5Am^WwLI zm7{puY1DeKa&)#_A}RnY1+TXy4a2+gSYWt$8(7)){fWR7@|BH!y|T1zU}c-^6Eh>o zS9JOw$yRLxE9*;Sqq4z@{cz|JumY`=U68K~PDO~fft3P#o>}-7Q32tI>$Ym0L7ZOr zRk&ApwFQI-urfeN(wIT4 zEI`*}2C+gW78S>VmG?MZevK7ai7UT4U7!h8G>zk129S5f>9b~R99W5K86MrlAGZQ4 z&Ed#2jK1|dyzLCi3aq@FEng6ogO&2vaY@7Qu0TUXKMt(KeSb1;3i--Ley=7i4y?q< zJ~20fd_{-28B>b0W0P9LuQd%MCF9Hu5n<+ zN-y#%+$%%1<%IW$m6XIGvK3f~JM<_eb_lV8v`-8mUx_Qyh@U7y?iKzs&9=9QmFe%A z$!`%WWJ1wLE3opm^jvsE5m>RB8~$N81Xf^##-;@Bl_7e(&3U+2mPcr;z{)!+euiBR zR?1!COr9X`%0hp++dHZ?F>1o&n@tnR#Cx z5C>M?Ph5;oEdnb=K~_mKh?Qx&rtN#giX^@$Bo476NiFKK0xRzY%;`o=u%ekhv!u`O zNlBc_d>*BYzYY0HUFergX;rXd;MW$~l?hg09fm?-ffW=3E4Ys2D=bHphALPI5Vx=3 ztAdrSBw|J}8my=w6EG;Sf-Vo9Dp&zlRv62)VBuY1}kV;W3zh5 zS1NlSt*AnS6;#QJs0^1ZUpv!KWw5fs_xBFSR}80A#AvXRY0qkcd?k~h?24||F~n7e z1~2d5!1xSU!BP@gWE5CIAKFTZWg%8j4W?!_NWPMxAv56(_sY%QXDiy2!HO@8&m=2@ z6(bTM<0A^JD3^MKMr42$)ST&(9Sf{vHp8F{+$$`)jEyJUD+E#+3xxtJ7;5JfI}@yA zx}Z&r;a%|%_h-6cs1G4u`FC0dte}Fo!n6!n$?!q>s)Ci}`UwiGpga9TPiKG?)Z|ui zLda&@WR8PIn3_ybwl?h)` zY6e)z2tp;Pf|Zad`U+bWuwo#N&j>+-6+>}qMi&aKs0avCjVr*)|BcTiPMv=qRf4|_ z`AXf9FXyG1h?T=_N4m~~6y6IVrQ^& z5qA@+CSb+6JXlnq2v!u0U0Sg4uGH=QtQhMIR-9W*M=Lnvx?tsAcz?VLSh;}jd5h8o zD;H)1=SAni%K6vMNv80w7>(C$aRw{S-;Ku`Az!)9=~bjTgB534WAl2*S1LJ=6sw%U z3e-}f-~$$42h*?;u%hU@rUUX7L*YfSGgvuq-`@oJ3e-(poxzG#^^st>SNdzu6D}fF zQWE>28Im9Ed}wP*?D7XJx(%l0HAuelzXFQ+P0lmLwi3jO=(}Qa31Wpz$o;4bR!T}e zpe+JcbmvT=e*#v{H@nz)!@bgv6i?t@A;hKiqjbT_1!^aCB~DoSvhJQ&`_R4=xSX zV4l?yzhJckxqJ3r!M?ad?80heKM^ENSnXis6BZwQZJt~$rRA6(*U={?kQ>gq>w|AV z^9dEXxC*qsukZ#I#P|wJpe~UWA!>nwMAr7&k-(PQ4uja1TJ|MTJ+`I6p$rRXYDc=! zTD7Dk45;B2`QY2~W^xbt;Cu2^y(JE056mutM1)OR7gQ&>llQs~9+`S#BepURhKsP@ z?+|;i-&I%Enl+(mwPsKXVUJ*C>mYBbL@!rR(zhZN6z6kN&K)KD_qznp{$QT-{61OL zWjoiPv0OvhwRpEuaz3w8IWNyr?uf;)uQe81FZ}24Q?}Z}>B8~l+KJMC4b|LPy=eF) z4~%S0=rL3uM~rs#HoKSUK}Snh?9=3JR5$el9ejIg-`&nk z45rhYH7L4(4!$$hiVjvi>D_cM(+AUOFj+hG1wC04+Un;ee2IQ@hjhg+M3{}<>AQ55 z(!*B3{%m3&B*tfL@L3{fUCVJZ{8`T)HvH)TD>ej7T99z3=oPxBPW(4IZLO`Z*b8M# zXxjJFlhufcxibY@HmG!OaU==~a_C>RQ(w_ZIqBG?w#yMV-s^v)A!N6)PV>$9Zfmt%ngPjF+;jkJ&+1g83fSX>=qO z)DV`=u%$6JTW*u?v+0MZwUKa_5~XEmR#> zJu4d1T?b$qNi=pL8b%rof7%J@=$*Mw(6*jSciC>7TWDLar3Y+N4inwuxzx%&$!SCP zcrA5hZs5Sbh@Rr+OkLhNL&2tB9?+uvM|oDcLwVM> z$%z$wQ?F@2{qLHIkfX-rGXpDqUicn%(!6o6VSwfX!>!0f%_;2Xd?~B~A?55qwrKwa zry9KfJ9ClW1%3@Kn7c-_7gOk6enWI*)Q@X`y?px^_K?r0EB83|a=r@IzZMs?&ybv* zx80}wrYHtBYhGZ7e8^V^)Mx5C?cQZ!O%+w^Vyfm9FEH~4y@MO>-5E#{p)Y*#Npc_e z2&}nR?0kFH=;7_z+IwXKb$gRBgt~UeeF{kb!yDa1R$;pH?DlQdtIhG|U=`2RmO?q@T)qSM z28(fg#KE+=1c(DH+5wqjsZHg~99(ytGDE2Brl9{KkmE`3S zd^)Q|i7@GP0qa{s@#?%Oip6}r?PvVrc3|Cn(1p*OQx`P6Pk6(wbjUkttgZKh12Mio z&J8ep(sXQmeq0(D_DS2+?fv7zK%-CE=5C)KR|dZN{M?x4Eh)~s<`c87=k}5h*B1k0 zjXpmsL~a=P&38V;j@=M$k<-uT zsUI30*}KP!T$>%Ce&{Lu12;0_+4oW1KX<#uzq2*@@fIyFmS<~T;6kQR>ZZ>q=i2?k zH42%>I=us{C2@tZJMJqHj>RV*};`<**5BbMZtD?wA%Hs>)G&&{@I(d%)p=2x`VC_g z8qyQla^eLgpBJ!WQt!kVh0fcxV^VkSh2JPHxJt*n@G>ve;?uvQIFlU;F>*yx1e(6y zG4YO7>JH`}dv0uc@kfq#-xj$`*@4RHdeK{VcypEIymq)gH4Az?Lb}Ggq#l|3_YRxu z68l1hHBsevBw>Z{uaWwO=S!u$2gIv9{tlZ8a<|`g$$>(*kD>QS`%Nalm1v=w-9;-c3Ts*tDCP5{Z*MU)R(AT(OiM^ zobj+`yhtg)`Oc87i^j;AI1i`ITKXSxhb>-^^NRGWFXwqBT0A9RE_z>?Rdmt%(AcPR zdBwD=R^&%qKzUVFE7B*q)y(^0Z90-tF$8}U}Gxpl+n=$^p3X=|GL?z5v-wmFTx}uQ?v(jTfGKLfqPbk6w z&5ecdD{!$K;0efTFu z$y(_jmb{inYC^1ky|}yQXVr9w);}MSks86Ba7Izrse_@k9pAUN_ptm$ztN=C>ydSt zN1;J9X$_c~j}n{F{v-QBRZdp>gm^Vgj(bjuv5nHgg5z0=v7OYJp~T6RQeThj@9nR= zo8cvEVLswr9(DZ^?A2JP8q2m=kRE3Eiey(ts^bQG$3>be@5q%$PvL1hhIUdiSTb_3 zy=f`nP3cORVwmeYCqe^gUpN@4R~oaSO;zoGhU(MI1cwvM1Kk-C!<0tJ69;>sHS#2v_vg7v>@ZI6hHT~ z++?WOkp{P}+$2%#OKZ!0B{vx+K155-?U3`8E!uf|_4DfyVr|+(Gug{h#Zi>6B#oF~ba$9tEnDyI5Y*fw4=$CR{>0{3A^U=8>_j9~Ob(0E zXbNi7k*PhIQjr=|!IY*Xhvg(^69IprM~cZbH#N9&)u?|?c}c0}-IB8jC!J$)rJ{_? z#q}98vPPIeS^UW<)JoI5cT&D;l>Xj95oTqhI^{|4XOY}WL%I5G;%drZ?<=WE^Ctg# z^Y#~09Z)*?kJL`JiDTm3l5UoQY}zqWu^>jgnYKaW=U!@*csuQy1{}D@#fG#Hjh~Lx zXz?!E8;vFVpU0;3SK4X_bkfYlbs2Z0U|w2dsjv5%6pUX<{n{pO%g_>SSox3o(r_>Q z#B=Y7m8r6`JEB&U)!)6=|KEcz6s1e?BTz0$ZY^!@Q{Zf78z3yY7^Q2;@ z(wMD97AYnDRR^^<7wt$X&z(M4zVVR7=p)Vny+bO62g_lTCAV7St@OmTJfiZ!v9QXU zxla$aC`Ojzo^lTB4XMnO9*Y$PD$QJ~CwKSDxCS4a*CF5Sx70H=PpN5t{PvyRtIIL^ z4QAt$N2H6jxEJ(xT{_!8RGHu3px1G!(>Y-$xfKRxzO5a*+&;9|O{r_|Nu|@S-*$|; zecSnO?}QVowA?#!mEHMo?}T$qDcrSSht;9Llk-x=J1=)0n24EIDQ>$Ae;1vYmoHAf z-03nAdyg2`d0^jdy&c7irO&d${sk0ud$c!3XI)M1 z0X2djXo`GKV<CNOgx!Od{ zn-|A(@x_cXD*G+-=jKKp#77cq{31jtN~Wb052KxT z*&G+WV3o#F%5zS%9{fF2>UW%YA$KP?KyT7gQk46gSlHjH_h!d~L-ji!u--Zr!>aVb zw9xrI#bum`;tiL63f}6Ayo#?~`Z@7dRpe3}a_Q&Gw>w1v#cwYCoa88ooRn-YEsb+j zB>A}^2V-_f%9dVpHt7AbfKk1j5@v4C!P&L{lrD{N()D0e4@xM4aAM}PUzeaN> zVIQwol%WzGP`_FJp5K^=r2x8<$o+fXz{;hN%@J23%EY^C6%Fjr29BNR7__4al^#I z1>@jQi9^0>ZocvTyOdn63-_psX`A2$aThGEd{^1!H@mf=MbNKfl%x`5l+-GCBJ!#1 z$f`3A46mTKzhEb-obvymdn({VctvAYT4hhx?@*^2jh?%H24xyWrm5leQ5M^@r8@t~7VPW^O{ðy=(TIZ6WiN zD=V{jxcoTzV*4A>N|g;QOMhJFj;n0Ak8{E=Z`T|yH++8>nk7mqGx;c$Yb{G9+$_=Y z{9(VqBjmeT>Ms8_OI*0yWQGGMEOtKeK^8_O_@4M4h0Cr}32qiYr1Y_!W$KLJZ|8gB z`&r3G^(xUu4XsNJ+(cp*S59T8Kh>=h+9mlL{3o_p2K;;^;;Y!MyXOD%v1mr6XKjf8 z&jHbxO3%7C{y&FAiz@Jkv;R^*dsyXuPKeRZsIa;$SeI7$)g-=^1#=QvJw`uc!{|tv zMCDgAjCI~?dd1e3NxgHq*D`80GCW$pT=*S zsta%vymgl9d7EQ3k4_)c;OB(LYDO+l%Jg%TVlD5boPSchFTCuQAcojngL@vaz3kR_ zjETP>=J-NF*&WV$%i)%ki%*)lR+a&;@MG^&EThU#POs2Vec8T0Fs7=o&?P1iYtl@9 zP) zJfYO~YI%%iPBHn_ZGqa!J?m=ap73^CCWHyhV*M=VUeoU%k^ z#d+nrH!A-8lgAMz=(6J2i8oNN-dDE6xo6JLunj|+0Z^alXyS<|Dcsd&Su`m3H3x~^w#{;n!$ElO= zWh_IKmzQVGKVfnWEkjy=Tz)df)vzSI3_QAUP;&7}TWoiDe5+r0_?^P!*m|YUu`x|| zM|r8D(;6EDOCvlzn4od8w7Dd}N(|ndlYNA2LA}=5BK5Tz_w3j86FCvY&Ykr*CD($oj%CMiO zaCH7IynQ*cJyWcC)ku98o*Sk_hl~YAXz)+M;z!wGF7#WwVswW{lp~?52i9%9XyW@~OF#ust*Tx#`-6ucR}58S>J(EO|Ok^Ao95ZH{!d@0`3- z?(&Gt=g{)@N!GuEH0|T8e+OyWU$eC2k&a0FD|;@BBAw^1bxeMad|jq5<700e>8nkP zmbH8k{c7pdNU|DEmb;gJu1^rA<4G^pjF6A_z?IK7CHP^yG!G@hJWqmopYXJNgmtOz zT(bBKO(w1V=IwOwAt%5S!b zr$`^RrVa@`?>-hT$cK=YD*fh2S$#~+ZlhrGQf=RuW;c`sU~C6Vhm1wD`-@nM4jGGd zv{vWm&?|mlLus@^`AMa(yOw%cba~6%C-OFN;=44+SmbTu#SdtZvB-DFiCbxqvB-DF zi@W+DW06mfTWaoujD_A~J@H8l8B3q7=C6;WGQphKl%Bq8V$(13Qe)pgn(#A3nsZ6* zd|~h{y{3HizaI$R?1PLYe`V0KBo8Oex0Ma~v>;6#`9yMQ{wb4B`BEnG$>g*7kg=#b zJ;1%>LdK%Xe}JES2N?^qF!1Wz`6Hu(cLpY9?J?LxiQ|vo9XBbpY61gfLG8U5)WO7|TWGp6gB=W6%$XHCUCcF;!v!5K| z$qbnwH4`YOblDrWa!@6oM#4+{c$CQ9cXmcQK=e<`2+?g;bdU~Q== zu6FRa$*uhLCST*tpK>8%$(Ihqe-Lf5S8Wc$&k~C$9B!3~cDy;^alE8GUqA4%s!SmM zqsTxNG8U7oW$!?)O;~qzV~}szzW;g5)aY4OiO4CgJXZm?X)$d`1ve$0iY( zFA1V>I?lxI`xFcitGr@4*FYTz@m3Yk$75C|ou_z?&)|?eW#zODI zKU;7Qu@IBm>6`Hu%1)pG79|4Z*w+dg9WoZ!GU?qDIm+EbP8CXHvQssM9;yVehuiX=R zCl*O``mWs*x^v&%C1#T}9dCup^i=!@wm9@u6MmvYA(m0(G*+drNBOo}N_&s_X!u+6 z@1;43Z`I|1Tq$F z8?m^84jBubfv3(Qzv4k3##85nkg?Dk@zgIu$XMuK@zk#p$XNK-@V{os`Ab~bD``9R z^We`pi2@`26#Q&r>hfMma!xvad9MUA7DmVJIZ69i-J01dUYgDKMGqIvscAcyBYCz? z`p50h)1ez6V>zw)zUnFiG8RVPX-#2O1@2G2hfUFUasloIpKQZeOv%KJMsB{Q{{w&6 z;(JQou%1m>p4b1!)V=sK-T!d|56Ss7XNHi&9L5NvwaT%a&xb5Bg-T_prdG+>FbPpa zQBmaBVag?kZH_r~RJm$4hfY^*bk&rr?)UoLkH>xg2R&+iKJWMQ`+a?1C+(NhE>3{4 zOq|F2eShg$J-h5(8^3~!tmby!5k^WD>sc#!G>>!zVmVCwD*>G`2*#pU@iXCL{Uv(8 z#|b?hN~%iOk9Tuoc&7q1{{D}53tY3QzZak3Tlp_t2p^Z7YB!(!hWP8`H~z~P@mr?e zF0QE+J1LLIJSoq2e7b0jnOT&qH9Sc})SsmB9iI<^u@q}lX5MkZSc)SlGatBMEXA8B zGw-=zEXCAfK3+vwnKHw5QO{9$k~`0Rb28y)D&;W)3s*!>;Rkv9-vFQ2zAdd`!u|Tbf#Zvope322p_TQE22{4w4#gjekWw-X3Md@0r6-q=m zXE~&{b%nC4dp70TL%~p@umU?gBu`!pI<>zUT&JgfvfJd%cmA1Tg_DVh zs*{vCZmb>{%Sp-tH$@MOarq<@j1K7X0adyPhqAvDBUx ztUg(UwwuLh{O@mFF00(-=-S^a@Hy6%nSWA>C;mF+!(L!NNhhVIgRzLvWXP0WFcuMQaU6<0Tu15+ z61A6_4zAW}i|BnKYOii7dRLZwyf>6nm>$0lXU=}fsZPHoR-?`9USl_|khCpbMvSll zEk9}8Yb1hJtq`Z%BE#EF!J)WS?GL&a3nn z;-56zD&Fa|`StdQ{LKF)BC!^!kp!J1N!xp`h~(zaig>I=Y9^Iw=p=dfwu`XxKZ?yL zvRP}uSj1)!?B{F1Sj1+O*yC%!Sj2)w3l%O)+NGoy}k4x3`B4KMOC^kGtOmAIBv1EVb6s4PswMYxDlg-($Mczu$cQ#A4+4nwx z+{wQp_G_m#*qiyM(iKE08oNuUk?i_4dJ+OxiWa+bO|68$nWD*VS`#fHaQ|OziNxx* zRSjMPRF=#boRSh8=)EO!TKS>Gs<5&IKJzT{TUv0eSBF!Z{(If36|}9Hvp=Vw5dAE- zUo^L|Jv#3{m!}AcEUdNbKZ;;1`X6$zScX1lQZYBXQN6AwL6O5#^vFsurOA?l^XgrY zhBP@6VTkKev))(srRm%beLuA#ml{OJj-l(Er)M8I*Qj-LB`9_59eUz&7ZIO@vz?33 zpHN%OO4M1JIlGwMNII1#<4n-t7?4PL_WEck(ha%=V_yGNPJ&5`pcq}@jmO~EHCHi zEvw*YFgJ9zp7HK%tTRmx<4e%vxuM8J#)=ks|5V zsDE4Wd)8#OQon3WHvO31EB#An4{i7JmVBQp>zqR*4w>#j+9>TC+M(Zhw#(GnS3V-A znTpbH$f;IKzF(=d)T3zYGKTn`OJ2{&)YnDGVm2`b^cV6xoN-#5cEuECw_W5 zU(YDjuaR!|$LVk`C>A(d$VbHP7>aknt0&(GS6f;+Yn4R_uu$kSnDf$)R%^|o*v%c# zk5)gNMbJzR83JR`e=EP@JM@g`lXpk|ja<8LvqHN*gQW<@;zCj63@BDH78Lce1y}}r z(ok4mf@7%vsV@;YG2nr-;@fNieolcicy>$ns^gp*-)kt^<&fM;m0%hB)~dtYX?^&7y#BG=8Rg`|;mTO&qW=s&>0LRGUEhC3X}?FKp*+>!p}$W_ zUMbfdYvKHt$-@_Y>J zRU6zB)%dsb;7yr4OT!x7_HwKt=YrA%m3$X#Lj0Jw=#H~;E?PDuD9-jvg0UFh&QIEr z1;(O;E}pb^-b8ZC`ljUM`p&ckx=ch3Qq^`Dq!vwD_J`D+U^E#F_gB!ZN{h#`_0N=hE?NF#R-q%!&4z6@ezYk|Y zm=m$92$ zWkYu*ZeIR&Ltkp8+nAdYDVbqI4Drs#+Hr2@Jz*B+eRr$Wj5$x=(l4f@TgW@RJL{#A zF3s8lhnSs`EOx6jUvMMi`jwRy3dEIa3VFd<*_`*16RzPZF(ihTmjPaz-<9XZ?8uw& zpy+c7h-nN}rD&?4gKWfUQi?7R+$S4yM)H~rdQfkuf}@OM1}$m_44R90hq8_kPiDzM zFA3SNTW>Z~QqO&QgT05qw0}?uuT<+Uu;^b@>QRh#7nt?GQ0h^7>n=d|e^)xKxW#=H z#hFmLmltjDON(@X0mh>AJ5LbLfLe+_g(ak1fyw_0OQ4p5zKK(lccT7cFJ}KI_1NDU zU@REQ|DPh1^Vv(xccN$6p+#i?_$7M{$x7z;)>kH5PxcWvw#jh|HL$SKh%W|1wx zSgcrFOidmbOCEnuVFMA2Wsq#ls?#WleIAZjepxxe1Y==R@Cz~+FqYVsyp_N)FcwU7 z(aO#-FcwT0i=Ya|5;}G-wnrI^MH%quSS%PzY+_zJ28`uD4Lydy8jQt}<$=M|z*si2 z;xYJQFcuqD7^a5?#`34Pq!^3^v)~7vC15Ow@$0!B{ZA zi&mn?pzdbuZ_IbbjP{GQzW%Wf*ZwpCV%eUwJL8oU5DVg|Pf}6_5Q~(&m%kMd%lf`q zDcW`*mJ960wH_uwELCiJ)}RRxOJ!R?e<=cp#RO**Qh#AdlFacuO zPLAvEL;$hun0A`&lA7@H=dP|*-$1tpV&OpWgm4asB_d-~`L+)bOUdPv8S+v1!{!Rjl2moazO{Qp5X@^hGya>cn zljPc8A_c@EHMjv;1;m2ylao5+om5J9?6Z_QBd`lfA0eS)=Af8_+J z&pR&57@oP%>nS7_Whbe>vuY~OPWowVkYzHm0{XEr29L8!;nUM80UN5T+q{8T5WYTG zOCT1**^^Rn#y~87ysa4$%D4@cyN_n%Dg&`>AaCr8l>%as()HuHW~>mmWtb@A%7Iv% z`fR1tjKZtiSm&}Pj45a!7K_1U`Q0}e*VyWHjl2IfD#9|^?-V_ixl3AmzDvjV>$5B8mD0x4UD>a8Rq~dW6 z>jcCytulBGhX-O|lobWmlmW3|=TWGtt3WJ1FD>Txp@3LWk=JpXfmqx}Y=dgbfmnP; zo}(sw(xRwZiw*Sk&`1Kr(vVQfIi~`|5=f{KHkoj!D*ptfZUhxTDnKl#L)%HQR3H}Ap&g_HR3H}A=k26;Di90G48>~j&${UBMe>v0rGbKc3KfXu z(&HWPuYYA^m7N+{55#i4Fpz~(F1cP^X5)3Cz+ir^3?eF(=aTo2>_Y*upuTuvY=Kx# zQzZg%l^p#(mChhsH4w`NW<5&R3y8&wPo|m#0Fa1;Y6TN|0wlF17cyAyUUw#0!I0%TvN7!Hz1jHh2Flh|}V)5bcqJ9YiV)0%H8mSEe zV!66%H~+j0h$ZM($F={E|*GBj<&eXj|`VpDU0-&_d9Vsm$#e2BGd^Ax*fKwJ|& z2E>AOsgkYO1H^KEc1iZ2EfC9|gxV`EIW*1))S0XYVli@gA^W!t5KDZw6W2}?h{ZOl zq8<84>W;DSnm^-!SmKdqNxNCl55Usb%#8zLiAR=@_ORg9N)|?QFb;@iBFROV7TIuU zJjp93(iVtiBFQ6XlPwU-xc{&Nyc4P}-U2HEL_AZ{T)n){X+Pj0z|$bsHd- zctQ#PS|JdNP0vL>G@2AR*tCq%&45^hu@Z_25R37^E7>n`KrHdoXQ7$|h(!}W#EZ#+ z_bGFVZ79`1Ec z;psRamiRxNB|t1Te>zK`nWFH4?dM^-Lz7fB5R2ot2_TlU++L_80b-eNtm3kfKrAW9 z^Yu}*A9dSk11ZQ7AQqd!07u$55R3gF$+36>hy@d0Uu`lE#4?`dId1F-#FAn^nznO% zq0VjmyrbU)5KGDOSatk75KC$25n-yN0*J*CJw%G30kJr~nxk(3V(CCuEGAR|u^4## zMCv31u`IkR6LSP&alF6q?mQ67=EY}@t;0YpRm=CQH>UuxqWvCWih>4ECq-q zMSj?EBM=L2F|XP%1&AeuHtc8*#B!3Rojg+s#F9dbOrE(2#BzeRDS74s5X&)|Cpe6X zur_6;)J5H0-Zr;fNGyw~$@d*MWAqzpZHItZ9P6t|-flhk38}CM#4^lJpbaJiu_WVaxcSIsW(Lx4foulE zl1htDwis#00>Uqr1Y((| zY+Bsp2*lzD;S&cS7W`liQac5RWrDwrrjP=}l0qrta*;qRDU|cvWh4+w3Z@AtY_@yCM$6BJQDri~wTUmE_&mE)K*Z zGvm)@i2||6%mlKZivqF8%mlEJ z9EgPgbtZc1KrB1iouaksKr9jLCedhhAeKn>1JSqYKrG?BhKw!hKrAT1fxcVfKrHI3 z_3T=4AQti8W`bjVKr9)*)mB@9Sen?MGfqfSvxh{b{woL}z+#9~Pz40FAJSS(&P&h0P)V)3d8?AS33#B%PD zXH6&&OF+lo;U``|EM>UPxdAVli?m-xC>b=A#z83D1BQSQtgFalyJBSa*J3*sSgnK&KlB1(DOWKPA3_46H#Ty3X0*n-E7%Td{Y~tB zsxV%H44dbNz|W<*BcR+QB&{QXFpmncztM36duJprq$YH4%2eIR`H-3@;e=ZrADR|g z4pYm`#9i}UAvIxcMaxa}%T$>r*qsS(Rc7cL9N33cE5fGk3TIriS7B3Es4%yfIfa}t zgS#js^;TV^Dx5DNQM=~tLf|(OnzpNaa}!WY*vp2Io)B1{n~?tVy=Jfyg~;DXPAG>l z*UTf>v10^Am-5(~$VevrcS`Ep)7bLU$iatN1_nC=JvHV2arP`Y9)F;I0`R-Dkp4QnR z8M`uq`V;#QGl*_Pl+r1wZBj8(CFDC<^sK8{!F!ClL!G0X^+$}xP%($3mOYy*qd6If zXe+)P`%lg)0m?0$o~6?=M}tq!Huk1uWA2&_x2lE zkBkyge0BbT|63r$kIdiXITOKwvL(cC3w|r&x7q{GnO&S3#CNJNz{RgcDCEKaEi?XG z#BcKu(NO;Yw>~9lnu-SsyK`HJ_+Z|C>6_-*&~0)7L473Y{isGuTA07bwq4=?HmcQE z*?Ytr@{UXUhCmKxA15_iPNn7w&wI*$HbQzLbb9;7&}rA0pzOuG6xEsr9wqw`LtNS_ zm|W6dz%-J!3Ly(qB*BC$m%B35q<2DuqPm@ra7f28W0>%60+-2P3fZ?3gfy=!U9?SG*9#_+gV zci1d6h^Q-=Qv7FKq+xTH)(aKnWq7#hiJw&c8Z;R#>u>Gfw_ZGxsVMHEDjQ@?DD>x5 zLJ~z+yjcdjCBYbgt+*5i#Kt8^D9teA`=|dxMWbSUf>lRb#uw4yrQX*?(CMLY8iZ6CG2LxH4YSr6g)R; z2@s}6AO>>Cf6Co0l2lKv3f{_mRCkH`&CIFsovG7}Pt6?#BP|@Y@_RSCedhn+$d@0x z+3mI0VQSZ2h4N!f-5y_q(^^M<2EVb=^k{b;3H6%QilvInUc(Wfj-!;@d)VNH% zOxP}R7h9IynXp==3%pAjWaiv zj`4FxwuJn;wsg4NYyMFAiQvzwKZ8GqtrU-laW0RznEeh}sog0R?&78@=aGO7L|&Us zkPERx8s6O3*fJX~XXol8H@L6y+LgQ5C>I^MBX-s6eV-SuS#y~zJ)H*@()QO;eCyPU(m^qeZUO>+O(rA|%; zq&}XGh*dAz#kABq{ES@1dd<|)`XX!(IaK$6rK|PjIl1`V4SrP7_kG>bxIg)heQ^67 zpoDV`p)=(Fb)O7umCM`+vE7Zb&#b-S(-F=Cv0Co?$%Mkk);k9@iRa$Q!)BU-xIXXoJ8_~}NEo46~qR7jx#|ke@jX+|=0zXCf<|Y@_QYSVx zPqqaPEOKPzqW23t2VQWr<(f7&Kj+;N_zry6IyKqm-u$f1ci_0(7V4iENoGzFu{7tA z*4>0vx7C}ZCe}f%dj)Sdex9TUalddBicare_2RxIRmK|hVAo)~)Be1RPP8Y&CKFOO&B^J*2Vt0z zyFtg~Bi8Nd6VlD`A5MPC4IdLPLqkbCHOFbF$GfoLZD$e57yl(AGJ~ zEOI*ZSVYI6;HcB9kMAzm*T1{k**Oj|6YBq>A(!qIY)XA|V;OzvZ`Wg&HmAP1wX8c8 zwD_0PmnUft$y*mQo%9|92R>hE8}~@Xnf-Y8uBpD3_KTQKJL9zOy@o~3JB^#Mm9$x6 z7Hu%K@F}0JaV_>Xl(4g?P+{`1@^UOEruu41$GAzVuhO3=$P*`+TaNfXaon}o$#J|i z@sYA^v4CTA$?9YEmCg++P@8hW$?nR>jj3agW2C)uZw*XgBK>ol2A=S#yd!I!oOMdb zAs)?!QzcaI!2?doI@zWlE9+xhXlYL73m*xWmycXJoab<)=UU2@aTr;g)ZY`Dmv`}N z#*LgRK2V}5mzP~G>AhbKr_`lf7@v3AqFVbtT$nb2GGeEmr~GnSE7QR#^nLg(ZY2K_ z?aPtX%dth_O|h3~|2myg+45cxF}O}gn`x|5btG|Wr9cp}7<%b->LK-q?`N*0G>k(# zvD5FTD;47pndnOoj(nc#3j6e=D|&H@6AUX(3h!6L7Ed2H+AXFUB8!Y+Zzq+Vsh113 zODb%PyL|sltlU#YJ2_L~V?R8jpYFYK(myrhD0% z>n2p>5kZ?d`}k> zyJU~5LVL(F_Q7?B)sJa(KVk1$_W&X%Cp5ZWvp3f@^qr7<6VE%vPw$hE!|#Em{U9$U zvsi;1{1r3#V(S*M|CnpU#N=dcAkmGne%fm-Ho%k=E7sK5$0lJ$dK1KAWeZigvV{)T zeZ7nUC@$F}%z_xyY!sCBK@P<5x>j{u()~W@kzH5oAjs%bXL_xRw#N?a-eI~fB-2)I zzcaL^RPOCnIWGdal-+kPi@ZpWXRFR?OUh=gGvMDmGrg6#UiHG67qWBfqyy}&2YU%( z%d(+k4)k6&bde|?)}oBEoih&?9Fs-c(5KlCIGZ!QWYKnl9zicVcik2n7;YJCo6J$U zQ)YPEw*FpEvDyMFS;{`Pxma|;VcBMzwpYFPIGe;EJJQ#y)&9KKN$jKCs_7{kUgsIP zb@o)Bb@tlRM4mdosqfahdzv%oQ?~R0wr}Rcf-SPY_CZ0(=>lPu$xLig>vVk5!@kq9 zx{4BVx;BE=zTbtOy~buYE3y5I5{{{|M^)o~jz+*E*CWb<7aR5WU&M~PS}NaR@{K=K zHXh(WFhJN&lE_gBn1FVYk5xFI>O&o8hHbrTP+AvpqjN8fwewlVRh5bRmyo2E;brfd z!yOmsy>r>dRyJwvbjI8@V=LRVu6(Mw_eO`QepfHjwfs=y78}wX2%KmsLtq3FAe?o7 z`b``z(udc2&se;=kZ`yzGoMyhIy`Rtpz@R1Axv7+@N4fk=+D+^4Z{z-k1GG|^UXT# z-tY-mztZm)yC>%Ulg?CS4a?bEMfA5>d@fV+F9eQky8O=gjDBqPDWF&5bWXc4?yx&2o zQqsqj6QbOn&u}%fQEt63J}KBx99#q#rq_ZnAJtp-!#h}rq&#?VDvDR|5XGln&&GJG%b zPX1x$G2>f&j;|BHTjdsoIJ0{gNbRacW8WcmCa`6Bjj zqF6zEI4)@8ZSr|$r=>lj$kM)j`YFeeLnv5`z`Z5flFl$?ETi`Di6lsmSVqP1lVw!H z!V?Z%$<98{@szn`LD)OHf9h4gom$JCq)YTS{c>ur!V}C7-A*bM52r`m?wow%EFq)g zW>Rp;@@V^X$PAGmT5vC-d)G`1-?!jc#3$qXk)LAthYG&u)u?^BL#}21Wd1MO!Gza1 z|MZ5;222e)N6Y+Y0lTw>>}~6z{@oZxzrW0Yg{-=v1uib;YW5)0RHojtu-!Veu+#cY z|5_Pk1@4A|QcHXciAsLV2~w-QD=4hoPrAr_uLg^A1mz7!MlD*!DuVKoBU@l}B|O-2 z{C1<%-*quD=f#7<)e>cB*)KEH2r@OLuYuT+96w99Wwvqn*GH-_yZ5KMMDU0sU$DvY7y4J5pn>_&;+N&oh?#?=V&-4OTIPRhzuU=Mk+;c1 zoQG<^JIzF1H#QQvlG+~B`b6Zx`kO_uu6Ekb4k8cK=QQr)VzuuiK2;lh-56WnQUm~O z@7qD@C<=1jtljXc`S9{a()~y4>?hjNt|ufcTU5orPTZjNS{N;Pl)~t;pXf~Mc@)PH ztx|v0J+#2&AgU@}b&oB~+ncqtP0hH=Xj?MN?RUPUkFcKA-6=YdNI1ZCBK=jQop`%_ zDs(1+YpmTy>P(2Q|Eoye-ihjKzv=ZSVrLVbxp2$=aA(>B`oO{^`!5|YeNyke99a0< z{)^J)gKrKj&T8GXMnde_qCR3?(G~lg zml|mIX~=wR*9K*=cYm!B8{4cAAKP6dWpAv@+F@_WMw3Q5`ECk2LMQoZD}_s@@l*eBWn)Wubk&Epgf&sog1lkZzCDZk2rN zzT5D&QhwM$hCM?2#!zB+oZ-EIypAGA`-NAP@ts6rqe=9=l^pVug-Cn#SCq)^*p!k- zOnc}ien}vv+<5ezDPh0m_Dabks3Nwncu6?49K^p+q-r1f3aW@>xL(@PFQND~HnzOz znSGDy+n0hjECu`Ld3WrezYIUj_28El)e5600f`-uI8m#5m=}HU(4DSF3rG1?k96#h zy@GDym}S4J(=S&eKXr7yUeK*N!NgbnJoqbiIUSp@eD=}lgTLa#M34FhFyXB!fE7pV zt8F0?04wr0H({RxtUT!>$?tarSUKiDxNVmNz=}@w9D9!hz>2)@k?^-dSjm+`*31H| zOwLBgPaFkUIXcaV3Caan(WzPFQE~xRWO47Y0Ukf+083m8m8Vup%rp zDbEC0k$8cnKNrG^gabB#3$QZP=OfPsSQ+1TLP7@#u!1~#bebuIm4Qf|Hw!{o8Q7%r zW*%VWb6f7f108@Br|uV!?EqN8QYL3z{-jRyjtcTJ0aheV>U>QBSV^*;>{~AZup;52 zBbx%SlEkabERhFTK^7jhW&x~>L&8KEDTEcI?@>Y40Km$hS`i_v4D6F{*aWb$$VVCCavoRW0N@&GHDz1!XatT^TAY>*Eu zaU?(5_8DMBA_X~i#Cif?C07R7^H~Th1NV^VRDhMiS%cgH9e|Y-I%7647hnZ>>K)H8 z7hna6PNB~TVMW3m*_;Zn(%<*L#E5{^XC~341F!OsQ@bjv$gWSlL1!dlkN>{mIqjo|GsH;5MX6;RzLT* z)9S>Hwceq@`o-&uRXj>ON<)U>OzCh+vxsncx}n(cKnct#dvrR`@bs|+myf%Eb9-v& zXY@fCw&eVE3{A#`I(?EF$$o14M&pZ$KKXEzFwz(+uC^o*l0Hd^)K1T$m(dI9)y?#b zMBlsXBdO-Yp>rW~&+6>b{Q_Z49+ zX53;NWB{OWd^za4s&~F$*MBztth71*ME=h_xMdib3>R0csVZ_MIg1kfRsZL?|Au}M z*EpU-HX2>vLzv1j9t_wg&UqRSP`Mk^8Hg4b(Q)2)AY^U2i{i;Wg z0R}V@cHK-zDdUyIL5(QT?Li zm3NH+_X)>^W5|i+OSKn<`%=vChGA#ThhdwUtC$O$gS|Idwd>+)D8=KH->&%R)eM!}@B?ATyqIrAKH2Rd`QN>O9u*5KG8PSZlMD_NnX_rr`KlA=$ zY+;Cy&SG6xy;eO|znn&!-=yRE=V2Fo3zmFT=YgqyR+USq; z=8sD-m5!DYYCbK&R62hH{ow)Dr}}f8g z>R>7rhJ=#gpPj+HhDN85x<;wy1p8wz`}e|BIvR>@g>IOHf0nc=Ufl2YC45<0S8U2hm8ntEM9dbe(A^@g{ib)91S@lv;Sp(2s8gT`$I@o{(INgB+|VCm`c^Z-=|M7 zVJcNuxKAE(gQ=7`fYv3gG`{0@w2q!a|11zeOWa4!CAVpCk{d^3(76RlXuTB2$MZ0i zj!vNm=}2={7xGJT#%O+S%(;SLbWIY*?N8Rn?*aT0j$j@!Q`KQACE&hr#MNOcC3Jrx z<`kUsCp4nt?oVcY?oVi0hikko}R&ix?d-n&MqZjDjZVO9x z6Z;4TERy*53&2ItZ<5<5ncyPo1q6$u{1);A6QS<4tJ?~n#J^Hd zOrTicN$q3IWOZ;6LQ7KD<9TqAis<{RBlGb4sAwXz99_937@z-+KAqI%-h8BMeExU^ zxCr6br{v60aFNkR=zDa5+v*2mQ^7&>-GaA-Pf6`hnBXGnr|%0!ncyM?f1|(Ar_H~0 zO@HC&se_BC=Mq*wa<<$#i3IkntOB@*?)XyD-CS^yTw`76sPU-&IEzzQ zJdt3G<|SRpEx-{h(EOwuxuZD3=XQ>C@uSUmdDDS}_w7(Mjf=9}d7_)ey`$Tz{NhA+ zU#xa9xQOoICCOrN5#84(KC#>D-_2d=%thm7i?8O6iL$)f|#3b5_sESB1)2VmP(@WNUA z%Y$uqAXgW^#Di^jz(!$oGn^#L^nR{qF>JfIQ}6la#jx$-T9)XqxUlUOzdfs0)4^Dom=OM=cpIb8>;kFLXs8O|(hyScY?@20_#vfMT>;Hv=JE^hTbw9vq|3w<@y z+?dws#h8b=r*TL{iOt9*UTf~}ZqHu4`Aq??e-X}=T92R2AC}=B@E--uAD7`=nJA^yd|HNcW$_07!%yM4Qh%n{mICL>59}y3&cL~XzbHufz=d#2}n2!uZaR2b?Q|19)A&@LcI^oj8Sua|JK)138xh=SoWBL<}Cz6};ZEFO%gc!-Esl8Y?8jmH;K)tUaoql zd)XuXf!K`U@pW{EUZ{vs|6hDZ1U@+@qASna-s|C3n|@O4o6K~ytR&W&eK|doc{bfe zOkM^{D3lb0_m+t0sn?6rk{NpoH~1Djn!ImK(`8eevPeYSr%feoV9Jb@z*u(@b};oBYs=VNhH2tq~e9#J8`&# zL^9K3dbf9LS<6?h^>$O^-APaDbuJSdm%=)*yVs~Li&;qWj;F&VB=SbO&ARtB=hB)2 zQ44uq3)!mo+!`-2xP&BmG3mvka0yAZS@bTgflElVMUrmAhDXHd(d$;t*e^I$=}n?7 z@`76eH2b^AX-T?!vm71G{*exskmxUDl1pz9F{t;E$i20232E&2(70-nYcC^uI;Et%XZSd|HoPyGDZ1wn^owO8%=a85QHGrKKjy$Cq_K4MtVu3hLNaOl^M2^VC6o)XqC!Qu zg!IRyNk{YC>qvRV`cP8iQLWACR$R=5OGuIwlULvjmyi@`U*4!QLAQfLJ^RSzuH?%= zf@Vj^P?IxULb=_>b9eON63Xq?-&1uKE+PHH()nlM64HO2`$-YzRh7DwJhVPsLe4W< zoV$u}2|3Sba#|GscKxon*?C5XqdR5G_)Gt@4C&$j;ycpt$vJs`5~O$F63Y6d-4P9g z>(-$&`V!f231uO*lEa5)iLOI~&NSH-x`t|U>JUbMkP0<5xz&ova0#hEH;suiTtYcy zGsY8rxP)B1WiXo=olqh(kxSNx;q{uLRc@tH-3DPb&ISFAS8}bL$EEj4n_ty=Eaez! zeO3Fh%nFs~vQB^CYzf&8!begUs1}d~T0Wa?np>@$d}t_5-)tc_T#@syBH+ABw<<9v z4{%;zEjt{^*~O2_GtxhwLyrCv3W7jTI0ZqB$eY*4tJY={l#=%j!8oh`PdR*f92lo#jM#aqAqKii{OzP* zM!jKZ5xKBYt^GJ7*6_KaouRIJdnn@-N04{K;CT@lUZmvgQ1YXZsg3IVr;`5~RvY^k zPp!{6v}x~ z5>iB}|4aTT9a8^a@<-`@{%57*uJld)o9p7UP8iOp&z|CM&5|(0sa%23;_R$su`8@N z`O^->hk5@o)APh28HEj(&Bt!!wB^AYD0#(hNKHNoW5#)x7i&OMh6vHBdw0F=|r zN&2}~8vIH6G#KIxue(>O#T?IqHAktYnCF}YIIk2%vo>_zVXE|0($+A+8JcmbRi{)! zT{qP!lkVBP4IoSeUBB#bKttC~f(3jQLFl&S0U0 z(^%;-*sKNPie=otm4hr8S6Dc7+^tF&S1=zL&Dt=o5XVZ1HAom&tQKxyJ`BRRqFcYM z(3S|}3JYrvg)=a&U@kHev|wC0RJcs6)`M|nJu4f7roySA>zE7c3Z8^p7(6ZJY>#%Ogxh2xm}3>a7PNe2pF z6JcD*BgGd!Ai}s3`!$brtnh^EgvNIk-HP=e=ZuB|i)`u+(IVCWMmHbE73I#uW2Z1M zu3#isNEH}Y5Ge^`F&G$EFnR@$G2#+~$EGlY3>a6KNWB#2F&I|@3x|bKA2*yP_l5?< zeB8Z@FqW*Z8ZfR9aeCZr4H#F5-N;z?!gKz_Mob)QGD~+L&(Zuks<{$kM+=F4m502o z7R>vLH5sz8?wn_pq_GWgW*wES%&}OtI%JCPSQloI*;uI;yRGm=Y`6ZKLVjprHs+1m z7e=^BEU6H19&?@%uEjmc-&Uv)OHtv*@^=*O#hf1$1XI*f+{Vf=ThyZ&t2PT^n1^a_ z83fgoBV&N`n1{tHE@O7YgJVZ9F9ufwR#lc4E47J-RH}%FXkA7uE6xjz$k@b!_Fb!L zu^xq&FsIZ{GX%D*Xv{CQ2gFabb`w@qyA|t~#%~68y*vx6!hNgpn^~T&92&fX?Ai}F zFIDUtZa0gm+u8p?s@Nyo<*7HB(7#jaKmk|zITY2*Lhd8W#0cj&i>iz24^n>Wmvj-k zd0Xb%_zl>gjowM28J)@-5k<-ycT8^VbL7mdT{OaO*=9@HmLa1|C}R^zJ2JeK3Fp~M zS*l?h5^LJr`pzoM!E$T7Afu==fPVsn;74%@28#gPb>XN8I+B#>eDl z>9{qly`-GFV=`ps9Q*m&dq(tKGbQZtwa1L!8Q(W~SHk|h7I0qbvrkfV2C#{gt~bv= z0|Ee$-NBvX1x%U5yv_O#0bg-O$a`Z=XLn;Z%uZ$vAmr5z5%}{-d-?~Njal`|v>n*c zLN6?SHY-b6Vc_c8Qe|ApZYtTPFG#A!msePMaraJWqp4AyFe2moWTc{0tc=KxeT8d{ zu59pE9`n8|wfRg*QmAuKxYR#czWpyVAb1qjAC^^Xyer5>tw`@A)4qpNzr4&u8k!r=E#;K~&IjsV>exRL>w2f`S&(j3M^58~C3@MN zz~^d%4b7z;heiPBQGT9j$s<9of0Z>{Z>;5PBps)&Lyd1IT~DawG^@lnBy7-XNH|xw zj|!rO8uucdqQ-GDRMfB2O_@w#IOsavf;o>eZ`uA5x3{1Sa2^$Tt=nw=S>2J51A$Rh zOHQPdBiey?cf1URzLo1`+tWG|DmqS%$fGtOQ_GD zQ$DGFQ-;jHQ2=Z~GgX`(6~K96a_5Q)yr!$7&_LcWUti}p(u4YW?UU8ItH{b^|B*i8 z_K_>7+-pd*`&Ej?{7@O-Jj$a1S(@CeQISj;kwOWP4Q1RwHejmo_fQ)GahEun7(3=| zl_xWVtzDXKUs$`5uRTV^Cm?KufYyfnjA9n3_Ua?-Jt9DnNRpw)^{(Q(*;xJsI^1MS5*#Xc*99Z0h&9%mm5 zI}n$od0;Q>Koj96v40oB4rE(X#-A&M9Z1vfk^hzf*nwo_Y^qCUVFwzB&>SC!9cX-V zgUb_H*nw=TD`r<^VFwbXg93732hv18N{WCl`h*fn+S6ESny>@e-W?%_uwVx&e6gq7 zo(nrrp~IfU(stN^xIUUCeAt0ve-`T4zz$@SIzDOS0y~f}^mBd|b|7Ksr~FFQyP7K= zIS*`M2a2OR3_v}My22i^^?-{e>_D2j!dkg7S7l{vx@%_PJ=vUlHWp*NyJVArI@to-Zu2*zZ5sw2 z%L2~ZP%3BZWdY}HEJoXK0}Y(cLUYIw3A+CO;+dO;O}3}ZdqxE(S$Z}gwYDvzbUe41 z(@?l2e&#Hxe!7g5!m`vnrSvdv`eICE&Oe-cvYTu^jqx4`3)xYuE_q56IeB^t?sPz95+nXMbSpmQpayP7ArkV4xLC#9M`Ig8|NlJ zABl)W)^_@i-*<#C*#Zu=IPB=B^lx&_#rjh;cd7alWI;Ts2w4c)qe4QaM;sUDs;g3>Xw(%)wK?+UL2b?>V_ zSuN)tZZ{wNj69JHxS#xKg6~HY|7sZ45~JVwB5z_?7oz7L`E3-AP4r9%G4NqP{rNZ`kyE~&ym?^n&f`Q)@?vE zUKRUbtKk%VYpJM@Ots;-Wm43cWtol2>MHcMUN>>HT7?SzcJE&3m=7np^(}};swARX zEZ{5gA>yrSw`8J%cn}4GY!7^t-ly*_hn3-=>e^T3_8{XQnWLuD0jXw_U0W+e;qi|6 zATp7EAF92ArzWn@3Sj2;2r!7u8glKmT&ECX5H1REPg%DLQzJH2C~7bj&j#N$;rW`|6o-$6G) zIAo4g0anB_Gh+I-cki@DSE_ish4SuXr1d$M$&910_t@Q{s>?DKro7{409M4`n6ukc?c58~N<~D&*!{i2AItJp8$UEGL9Js!qm9ulxPxr8%1G{-U@Lp8Ls&2O z!e~315a|BI22Pgqq$thBiYae`KP!7LA(#aNiw)Y6+hK#Y>ij=^oryn`i`)NC3$hJj z#x_Wul(D6vAyiZ4z;L_MH9LxM3)td)=H$#{Z+K@m zSW4VEb`wSx;*6blX%GBjc8=wamLP9J%7xu?Cl14dNcl+)#Z?&uOBbynt)&a*+uXsj zw3cqbi!N_s=e!ONUYnI=as&>_FtzMd*XP~X2|t>y&#petcf!k%H%d3m;TAlLc2aXp zs(Zk(+{(;0dDH_V?oR7mllwhDm|MK=0*4uRbFNJ5^j#*qv70nD$xu^zxHso?un+ z%)T?jlM;zZC1|NCk4TX%j29Az2|J#~8Il7;csM2Z_VtGKaYxa%H6AH>r`sDP?z`-; z>Z$f{PH506*fQgcukBbt>&sU2-$=Mu@gez~5bP{;PLO1lN+`MrFDBS9UP|N=XwNz# z4&LiPqjBg5=+^_d0iDPvYpwFoM09m^8|Ju3kjNR41@Y%`cjD^emJc}{ARRby0Ow4& zPPp#kkn{$fgLa07h-f0`w#2YA?Ra%9zf59Z4N{Agpo&9^MVN}{7^GMvNWtBOR@u>u z4w7@#8F%LFLAL&@CcjBEx$xtK^>M{GVKXvCqgS2V7tpJk->WBUCBD~qP=)1jS9FB6 zad4MGq(x`sWc$Q1yEcjQpUXq8BtRNT;=GQT1LpwOBjJLAdMV$5lGi~;tC!T(;7@QI zB{H2s>6G({32g0ZgAA9*xH!mz{CeDZV1K)|#Bp@FHr}mWS)xzDSKF^*GC(AqVIl#Y zBQ6KzDMfKhXwscKm45dj_aTWU-K}FWB@7%RD?2dUoe8-*&n!9FTwH=4y6HsZW50)W z=^afHwQ|F_XU8=an&gLd0&45xJ5(9z5~euX(e~cbtT%8NT?a!hSkzEX1kp_i6XS{U}0dLQNYzLH1Jm4E8Km=N$ zwUVy~KboK6j)x)1#0Md_I;JJw%b^o=6mSv*5u@EAPvaucN*zuf+QiB>OvYN5*z!qp zj?M;!YZ4QRH=yC9bQ8J?X*dx{AU>>fA`*nLLI(~~HBux02_18n?}gORpo1KQw~*h9 z#B(QMc1Jq*d_ulLkTdO20D)%5*&_jKFWSO0?H5HL8;f4OJLxL|gEV^e{$zy6m85;> z)q9h{B5=e*;yZy~3_q4!s5bk$eWdF3`szol7~HZ8QUma)2^vP!e?>vZ$h0O-22HIr zBTdWUKBa6SsNY-k%VsD2hRY;G6?Fa0jm+Lwfp=sc<(@@LI~A02ohY|cUJ(jPxVDs> zltf&g^Wc$!veNpLFv4!R*-E@SQv`QA#Rqr0isryPggZ~FMoL8<1Ze7?jB4)EaXO=2 zb7J&_w@x+^1{tdes9IE^JnZ^(wsGh;32=eG)){ULiaEI^@`t0MPL$8$BaWvZeNdTR(O zy4X$@HH+edF5dgH?H@!R`_l!9cR0aT)5(=*U5ZYv+eIs|6`|L=a=*5BIw4JV3nA@<;c2Flp)(VY<`HaVKFXS-egt5&4>z_c$8oKD5+iD?(yo|IZb(23?#)0lcB_wl~s-;~JMfA@^QctjB7 zr$EPu=65GSZpw24TSt@;R1`ZaKbE2gc{rqsZ&-vAGETt9CRa5F&HoMO} zs84`E!?a56K;%8zF)y3+hr8h`!uf!*`iHFZjK?aqufEzHAA?v()9}avo;1Wl)QraO za(o7l()Cn=-VB@?zh9D?IA%1i`P6^zKIgdfJEoh;{7^_{;-y0}(#z-ODnSW9FbP8FNH5H zB2UqER7B4!ZHC&C`<}E!IWp9is2o18w1wrbRPOmKQJh?wcof=3@;2Z1&%5vWG*Ov+ zKk<}`?|FD0`4^N^T3I4-D(-KDw^?xEN9Fx%VGb*Se1pYQc|SxusQcz;pkpB^g@sl@ zr$0zkxh?8OZ&8u*^sj?H5f4f0_^S3-{Gy)~C(;Y4N35zf2d}3PRCXdElIkI%f3-h6 ziYh4Stj8))2%xg>rLdj_mtIu%_^sOVU=JeQ-GB8M@37!8%T8s_`SHz7{^R!rs?wl$ zc1PEGwj{#o7nPWo{Dwq$4ynYv;LMr*r)2UCD}r13O!uCW@$s=1?COY%$HG(eO?F63)vsd zj{9DwJnNGWM?xW#M;W4~#|J47@OJDErhNlY$Y*11rRMv|@7KP;K_a`^^mfuxctI_1 zt(x7TT@d~tRq!?!8tD&3CHL7}Xp+tS@QgHzHmy}~`4CXqklhE1bA^H7O>d&oY0k43 z{%&oL+zag>UrqPUJv%U7QC60bA3m;Y`hj+c{m|4jbu2u5kkXQ|_n?c~yYTVKvKE>R z`J7&61kqV6$a;>B%>aRQmZPjuzvP}axb0HTZO(*1A zGxl3o@Gi@JNV@Q=>52Md*7vszJ0`s`wJ9VNY9z^As2OZ_{#i9*ZMtu0C0XbMog>8` zKW-r_hywgg;`+{*8(R3=86gj%<6=m?8N11T7<`F~b5Xw2b_J z8%Ekh&wus}xe^aEi|Bb(vw=yNS)kTMos7&Z?R05%PncO=PCANa8pF)u%O}ROm0@Pt z6B!pPFAXz`@jzRfwkXUj%J_F}%Azo{DEs+M2CRdbMN~xSQW^kqU7uGEEdQTdsI|Q709BJ=i~7-7RVBN3hG6GEGh-QA+7BqaiZ=< z!rSdY7SZ>+g*oj&7N`|@ZwzEHqV-IM#sgW}(W2=12eB%**Su?M5tZ^(sgtVS?IFp& zdcJKJkVSgEsO7qa)ED~$HW>n0M0c)hjH_${$Omj9(; z#2Cnu^(`OBvg6d)RIC<|CBNEhQ8o+6VyhmsxFHM3qLsNbsu5N%&9m6I&HBy)Svuuy z3;ei1mSj<_)Wm9ms2N8pLdqqS$Hi%#ISSLuGM4l zkYU@@{;YW0)<&5##)6dOb}e`_(n`Fg6hywJbnZWyk)(R9|71q;{K&s2cWT{Fs?@rF zD=&zgn0Jnx(>ba&o7Ai|d#eCa9(Ouj>{@-0o7{`7;ONdC!;#PTDy7D@9BOQjE+kD$++oI)TlLr{zHt&?nKVo|qnnrpK z29?TI+2v?eY3-BMxJ~j3eb@>85uKT~F#%BiS=lMAwQtu7Ea^_=YCYfC>=+#wn$_8> z^(5(`){|SSEC%da$-mDZ3F5*r=Hz1A)c`K6Rwci;T@B*O2+EV6+pY$Zp*Di?Tlfl@Txp{B??%y>pS<`dD3!7btL_)oQn+kaJyIQDbv zF_{fScWkg=z2vsj_@e~@C9w%7bqjFE8$zBY?9wgJ?t3&a&@qZpNeAbVhIHFB{kZ=xj-!)Iu=X9PorHN@mu3<)DW?e~}`~-y5+EZ}l$5UXJ8hnbh9dn8l zAP7vgVVw9+vLmB{Nb+>7uQ**%a%w>D+4*+a`y$=YDbf_8OT(B+E{G#FCyo5pBwYq( zqhuw~8}7BbZfE$X-eW#s$}p&yVIt`~rH~jb*pjN@ij)>n;+g%r@GYuNc#m1D3$M?v z<7X)iOg(ZH{F^~i z!Vg`jTh#sBRLAZ(Q>EpRZ}hdH`FSriUFAOxRQFrjM5*gFHI}K5s=)WmaEvqSPPt&u zEp5-Y9y2@qx*@8%mlrUk$R0CmXF)n-N9!sK>vh)n&phuL3xwAZs>XWx~x@CHhG`aD-(HDZj$X8lo0P5&0~XCv&#G*OU?tr{OFC1({V)nGWz z+Fv>2Y6%Q#SV>QZZ{1KeQvZ$xEk>&Q zMunsBY1>8h`Ka(6OM`2#dSz5N#)2w<^ecg@zFy@c-d<1A&5gglHCnS5ny%>os6o>e zzWNJ)KUFL_UTugHnMAi@dZJ_^BCpY(a*t50P0ZfHNA4oJmzw%)T`PWqBaS*A9`ZYW z^uRi?ay*_;Ff$o0=8MOlESSDi65pVWc`8jlW-dD}Dy0A0Bdp zuA;d28G~~V<(kxuJu}ajpt__~VVlB7x;T&`q8p>eOrISled3$?zJKfq6E6KLYA+rp zT*y1e=VM{QHK8%vAEIEw#fDQtTH;~C6?=yr|8S>;*4_RNszO3)=QxuVV&-b_DM4*~ zVR8)>CS0t=XTBU2CR~(8n2<(?377g0*5U(Sd+Z6m2B(rM=;?|JL+d%j7qDRT_onRn_A;&3<#-07aN^^;KH%!-rtPOeotxuoDS~SG zPgMx07DYGG_&lhVH;48pMmz=85=w!{00~qJjn;q2UJ+D_;&Mx58=_irWm=*ch-&F( zlPu;zwG4aj%e8+BswHBjvD=pfswHgYS+^nyR0|C|7ka%LQ7!ssD25}TT3$1EDAL_P zwYa5)8omY9GQ`9y^4vhRxY>ppj)H1=!`!X-$_-SDTTiIT7^oIV0pu>Z>C1+%Jm?-K zfocg^x!?VS1geFke{rR$8&pfTw|;I5s214zucSG-y&fRV=}%nn-0&)4ntV^v6|YZH zEJ|Izwy-%w?kT7ieKK6GcB&4&(fq;|u?>C5Q$vsc1xd5sqV&z76rW-W4UTAKg z_SFZ~LR)<~?V=B=#Vshbc@$L3n`z_R_cTx~v)oIHV{TCC8ZO96Ep`LdazT)v`rZvx zOSm96^|>3U7MgIFc|{*oi~hxy5gxhLRlbg`@-1roJ)&CJZ`Eb~1l2O)r3Tl?LAA^X zLNcu*LACrDd6Qjc4yq-}Y=}ou2i0O;G|4$A1FFRwI!IEepjy{DvPx zR11}*-uD-%mRU};jAA6HmMFh@j-`xZ6sQ)qjoF!xpjvu()9ST+P%T|vHkl89rUn9d z^$*#gTGVTozP^)q0`Y->h5C3lNL+Q(-)Nn5khsv(HvX}sf!1rKDD&hJNZh{$dg+Dg zAaQ?&24q0uLTF&%l@)A&)BjOd=Yzz3!K1^Oagewl1%{cKmq6nFHPX+^QU{4^Zt+E+ zoCy-w%wjArj8j-ogibhTjnV>!d$82Pmd_#hjvVc*l_$T$DDI%gV{B8khs(lE3PKG z%IvIqs7wO!OZ}h3zd+&+@KiF-n}ftPkC_s9WrD;t58?}Nu|eXhKmSvh%?62^nQs>K z6(sIov|ipGb&$AG!U_6C8IZWH9%9QpKS{r~70DQn7HcNzOD>Z>E0t!aTmQ&;fga4P&`%9hL6X`Y`3NL-7IzbFNn zAaOOe3C#v)LE;umM;CFIK;rhlx?BX6B9xr|M2r5fqeq}tq#kNTjOAum%-+w6Km~~p z1S6jf%pS7vCw=B^)X4qYPZJ3bfPN=w*DMacwARr8i3`mmexGle8>-&t~_JTKD5}IIu?CV5Y^HT&!OUX(GwpU;5jt=4*tKR;5oEp@oyK& z#KIC}2U4ifW}kU-8t@#lP+t?Mmcd_k$wq}QFy z0~TlAQVqUEGiN=D;W<=Hk47z4H(lzT>49}rrUUbH?$pPCxO*&`K7Q=I>_*L}r zWfQBOKTW!!WCWf=8cWeV%k}#Q*i{y1mIUw|V&_{-EY>?9|dOd~1K3@7&U5xs|lbS{5hkyL^9U4c=HB z(;#TTbLeulpnm9Q;fS?`#aFo2{TiA_idP`t%Hi2)xUY_6D8!T=^B)H zNCjTEjk6|WgM$lJd9vacYcc%c*ODOtS9wr9!WP48PA0Fh-?&XIK$d^=DvY>mi`4nh zDU#B#)>ta?V%uTp6-kF)k-B8O!Q{#sgAF52yCp}=CKJ{eNI3AgzvQl`?cOg*LDB;x zxyfS`!quELr$ylU22_tg;|Pqn>qv^ftO3=su2E({vrU@Ix~jHrLIDB<+LL-~f~7}J zjbrL;+Q!yRXg(GFjN$iP1=X@<0TpTAcIi;&)#ZQ7Nci8cLU_QzOktAqOXm8yE$Y&I z;#KxKGtIw6=M*O)2mn1LMJT@QRgk!Aie&hfR~4C8;WCGm1plQ)w8^B-8Zj~K=0s5} zCM2>=j(oPwagDVYb_+`ss~8g9CQc@|9bL0ctYGtf(E?1!Z*9tCAIJ;%h~;gW5zSNd zh`A~fx8{sA|MXQ*Eo&yE_|8`o$Vj`$1gh?!uJq5v|AnMuSJ7*3{^H-Ja(ch{WXl?9 zPtnG;HBx?(+wQ0iND7Pi>qOVkuP&r+_rUHD&r2b!X+XjT)hGwi3Q^&H=oYcN`uJz# zh=l~7aurkyRF4RC+u(EeFYcm3T-#y6g{yXJjKxMaH;Im5CiAa?YFQhj5pt;wRLhzx z62h=HP%Ue&ND42tfol0*ibhb5Ue9H}QQ^k>2FJaOSWVPMF=~Z8_qr^o7E7!;H%S&$ zi)MxdB|ORN;2*N=M7;jN%arIOdCMZD#a}6*FGMx%#iJhJ^`<^(_HT%~kJmX%zwS#7 z__a@b6hGChnV+;|S){QTcirw#1B3+5C@q%C8tJ1bN0V%<8f9}6Xl4wwW{rYNf}e6J zAqf_&WCJZ5r77t&W5xwd*-}{FJorsk&2oHGkkqK*l$~ZUrqFU?x|yi=VNAOfTCwPZm(=`0&)A#B)}J0`2H;0ZK*3G$f&qRl(Q5sNvtkOcx~0H`cl^Ch|W~k z^`;QZr<#9UUl8ASR(7Y@ln=0$v+9+oi&H95mm{BL=A44 ztOgY+qaj-*^;<^kw{8%4C)HX8D7aHc6v$yodgSv-fts!2kbu;mTGo6*McQc=%dhXF z!XYS?#V6OFQ&+84l^1KTU!jg`eivT}e(}Q+ZoFI0qnfu(P1I>kZZP_qT6g6doY&p( zcnemE17NvkDu+r&z|z5y%Sb}N!l~F`Z4F>a4M8eK04&z9QlBgbV6hI6g^CdX%jP0= zL0S@kWwUWwOx$EOHAOk@gG#3p9UxR<4J@fZ%(Q%s~V!Ka&FhmUKSx8hZTtmy<+~ChpIepNa+af3JdAMFz_LYDj-CWyA!7}b0W5M2Nl^^T z>I{d&6^9Y9c-mciEC+XXkw5|E8h}O4_#jd-!cE=aj#P{QSgzU08CwHb4v%0s`AGnl z%`qC3OAG*u+!Z8D0AP{3k}A9eIq71#A1T5J2AtPT`f+&GfC~*q-1($So9EO0*Jc1% zl)kY6EJ==gS+Rxy7IuX^@46Czg^X3_B`E<|3^P(F;b{OCB{mDMe;B}$CQmL>S^}{2 zh#J}_BVd{NVA!98fW?QN=gS7LB#+`}nhgOgUi&7xDF&zO5>jE^509X!xQ&J-XSkgufXY33CEaZueOG9}8 z7Pw*i%zOsG(i3F3kcNO|X+Ce60bogra_=cJ1h5z`WUlN_1F#sXX9$j@0a%nWnUQ8J z083hoVNn_a7TyP?eg*;--ind|1Hh7GxPx^^2?2{&>&CE<9smoHE?@&#SXgvecn|Eh z^iT}h?1D{NtOA9w&>jFw&pE?#Hh_gyfcCnW24G2xGMr%pSa@%gYMBUFc)d!&OaKd$ zh~<4z0GeuA9(;4AK6`L02VUK zFe?whvMco#8^EIEo#vkgU@;tG5LLYZEW9!$4K@N6FRQeEGJqvX;GI@W2C%S46uiRH z04!;NhONm6SZ0z>>TY-1CDBU?IIt;!MR+M-j06EEoZ>{NI9+*ES&malDNHmTh=VPO>^IumW@S4{IqKTmW{@?krz7vEZ5TIi}I0)g|i}Va0I|&Gh)U`5H~mmU|FC0 zD{c1y083oAJn059vBc$wpWX^!IX<9uU~D~rMV^%Vi#UKqzR`X_vmL;4O-+8{CNi;b z^u&Yh04zCmHXUQ~02Zy!IsCqB02c9u%?*j}0I+l%72mcEz@k-<6Y^UJfTiPXXi}-{s-#HGz62}yuI1XU3tJ7^?Tn}K0LyO;B@86+v+R~iUA};Nz)wr?77Qj-H zKVSo3krx%GUjwjs7VJ0*U=eS)7S&-{ecVB-LJPo>=V2H3SRBCe-vuMazY2BR0W9m? zwS>5K085;mxbZds3*-bQ^Z!j1gk1XX#t{IE_!S!=w2d%|#eW(b@o(dw#b{2`K;;g}@Vyb_iIQ7qqV<6U(7*3Th_+ zEViSD({|bbmJ<`dFAm)VuxPj3eP%ulVCf3dUa$qQWG~L&T)qonITGdGRiq7I(O$4$ z*`ESn(N;e$IFbTjQOLX(X~qPwq{L_!*#cOKxgQkz?;`8n+!Y0by8xCWhC7&d6cDhS zZ2dhnqzk~(<*7|A2e917W`~A%0a&^y+U#-w3v`c!hIRp1kjjyA0L%S?Z1Tkv082`g z_Dnf(Qy%w?LhU^O%MEU?LhwBR%ROQN_lp96MWOqIu04R|F7XDEI;bwW_xfaOK1?jf zbf&v*Hieu(G6w(_By#{@fvywC91ydi>!cjOqTrq4p8{afMoLFuVkzd9DQJ`fSZm6H8)_@@a1XOW%N&)tD53 zWfv)TjWU2`S7ZNxCLO?%rnYP1CNi<`^pt~p5wO(Tu*P-)SbRR`2>Q|hEXoUgk@hSA zOWLyc=UXtb6binlL52Xp0>vMhX8|lo@duQCX#VXpmy48sB!5@_S^5EBQ7-D|TP6Zn zq}J_{>iP*5o{qAZFM!1dn?s3Y0a#d$%GSOB79VUbC7K0bVI76q5de!%K@R0N7Jvm} z13tb07N5La&zMA!IOQ|F{L_g57UhW^zH=ggC6TE-(Fb7Zt<$A1NB(aJZa{8=ie z51aE^l%+j=8X*t>U@6NV>;|wv*9kohz~WV4*a=`!Zb*w_SyuNs_*D2HVDYp|e5?#$ zQMTyfQxaifQ8qp+)P;MesZ#DfLL42yl4z%F>i3MO${?R3jU;$Xte)y~!@Bl2l`Ls(r=X?Fv_5oP1-(~?UFC6!>V@&`o zvlUsq>sSCwD7J`~gaxpeWOPx&GXN}D_6%O%8o-hvA6k@$Oe}q(Cia~OSY|(%^uIvB z;!7{^odvLTj^byVO#m#RMa4^T1ptdtJ-`uw+D;%*+Bkz5wLh!W%P#vSY8OcGipNtEVCmwy}~j8EE$0&t(^#1W|vK# z%pzb}DlgbKi-2Y6Nx}131T0In1y^PPES)RCeLq40ETQv#&D&-to@xEhrjf^zSRyj9 z#F~nrv;i#DZ5)w|?EscjlND>N6#*{0S^3|aR3&HqBDFb z6-jB3FxHK{*bZQcONU|+m{@8jS40f916XuMo=ql*7~lXbYmpQI;#gw4B}tCR#1fk$ zavBX_f$kBg909OMl3acf0kB9m8Vr;mjwMb_a>5BFmdZ&zkzhRli*ub#`$CMek<@ z0KgK97P-0Bzg^|@fj=f&M5H}2jS@Awev;*PTnBUjERv!k^tgr83J=l_Z2*f%1Cl$a z)_1^EU;r#c54+gMKPyKpaC}NEfCaiogw=>+S?i7wK0q8xteuFlB7g-m;yjrj2Vjwi zaSyrF4qy?vf)j?d16V|0JNr^QfaQN_8qo)^Z2XoBVA<~2!ie1pV9BjG$-TZ2z;X3dtp~qtR67P>(H*UyvfBz^IW}=(ap-pd%hr~g&;CFhOJ~s51zi9O zVR8QV<(mMO?NRQXMOy(ZTNeyg_9p{awyNt3jwAzEHfCl=K0_Q!a?I8uT>y(a_ru2i zn*bIU?#f1kn*f&WhINcP8xgSBw4MkJ=>)KJdTyoW0$6gegwXI#081xjD?1m!l2hOm z8rlh9={&c!JQu){Q$QeJOa`zdM{S+SC2k^e-)yYS2Cz7Cdp8DW!^DzJByqoN1h8!E zKBj8`V7WQWZPV%s6U))6sjlCfLXJJ%`p56E;eXa)r!Iiy7;9_R@0CQt zPMur10G5s3$^OXzmaRiKiLNlQxO2-ui33F=};jKRmpXr5Ybc@%0V>zS_oJIbC7`D@{vFZ zK(nEI1nbTeClnEL-w^}y_80l}M&>#uoq17F_7A{4oL%Tvx| z&$>h%$c7&bF5e}}Gh311$m1gQA}?{Lle!Eu10$jy)gYuEL&d5(D z<KV7zm2!L^I1!N*-ego#t$5fqo~ z2$v}PZ1_Gzm~ip#$HA|LFyTJEj$pQ3{Q;MQQ?D$|jBHF{KZtymvhCn|+}{MjSu`+) zVCKf$MS#p}6?X-9rhoBLFxhb-eqBEcCC{M4H%3>#Kgu=IM{XOUaI3 zuHKfy`$x$LYIO3^xT>aBHBXspc54QxPAs1Rn zM>;xaW4_M#363keRjALfE zLSAaEjxJ7Dr#dAVH}1-Lf{xkNTFa+OT&d=76NDu|%;0W=B1}D33Z$Di}LPKJa@`cdKJY!Zj&J!c{u}!XkcE z{M`xVI%ZY{m34(34GKjM@32fH`UoKqJ7J1Q=sShnh(p>(^a%rJ4v0~1#dV`eCGfP% zj>~|C5r0a0+3lBpd@YL_S9joXG!JHdZ_1%H~Ol;DNVkmvnB5fpI=_Y=ZL9WIP23HOu2 zrya@U-#hj|BMBaYCV0E{e2M$1$`U@fynArEJI+L62A8*OMu&LEqa^MSxjt^IL=KLq zm7_x}@wgX9AU}>Xk{H4f%j;ObC8#O{Inj;=Kp2GMD)AlzKbtbXTf#zepu~F@VQj}2 zGPOen4V@r2ghY;hLh<%-y|3GaP6rOfZgvtW)*eup>u~VE2}!YA14J_73>0V`Cnq7E z60L11awCq|?y}kOI_`zS3?xnxpxc5FT^3T?5sds=T!ir*8tCu8#yAU8JKWLG8tWpY zc3hVDp>Rce)s?eP;s?QhZB>P%4KoJ!RZ*-0@4!5OLt!2hUOQ5fQea00hjO7@OWAVB z4gHtXYnQ?OyPXD)FdcCj7!x=4U?x-O$u-uBtB(2#xE=__H>{!bT^ z{nPui~$K~{;vee{7K656fq~@ z*9k*QCLO0Pm-!&lmdV2vL59Svl6*6z{5yRR$F9Iug%&8)E}qNouYmo$^2MlZgNkTJ zTZfh3yK)Jwo6itdZg&k6jw@t2wU&fFg7@+VUFUEUPKIth4u&V0y66TGLXV@N3)9Td zow*h*NzmPxO3-zzcVh0WCov_^I)v(tRfKBCdS|9-w$V`4_?^(QE<18Vmmco@iIt|2 z2a9Ui3lF0015U2AcF8yeDeiTOsR%9a%4D!{5tXY4r-SOPndLYfPNC$1&^F<|rDCGQ zN#-n07b!W3F6lrQNw^=n!?lrnkmSU`wQhuf%Axx^72T_JopNwhT-EGLgqW(uy@Efw zG@QOyEN&Atbh$fyf3Ubm(9*TfY5wF&xzJ|%J?@9&3gkw85IqSq zLSiSB*hEPq(p?VR^iH%fY8)OoK)TNfm3Hxi=iI%LGEqO;>7U&rg}cKs!{r*o zDRRjlDG>QwR$t4CDSggbqax;y{k3KoPvl9fP?J=RChk%ySe{&;bh^<*DaV2%8W_LIT1lJSgRMdy~Ns&&_HljYvzot~lIVxSY+w3omj3NAC z9xPG9hgcHvzj=d+fLV(w5{MoI)$>3fZ0ezh2)+3>ch zM5LOlSiEr~{dC9fJQa&!4{_J{GLEEl4SzRMapIohV0Bk>h@X9A_g$5p z>CM8^UM2OfiRkx!<5dFJ^!LdNM&IuUtkd@;7aM&q6}Y6sQ^@Fh38k2|kLj-RV@R0E zy7KsTo2mcnm-WRN|1j5>qFz(3XT%>0RJ#z4E#<*FRR;Y6wuE=Y(Z$+a?XL^FWu-6d zeo48RQN}bh&3F@4(8ptno7%sQ`n`|EJzH2l7f`z7H0b2JtFRY>BYGE_UXGsjRULGm zJ!fh^9F>Sa>ASbk_k)?Mb$VEMK^4zTZ8lXh9EqF2nNgc?(u1-;GEm%|YPp|imTcIs zQd7IIpZ$ibqBi_-?rvEF?YOUEq1y$x{(s@?!01U|2r&9;6kfgnRcjYe111i$n;?hM zZW=sba&Ww2NvqK60!n_*Cv43UM(s(;SonAi52Y57(h@%Ykf*M;oboLkiJhn+v6Hj` zL0yJ5#7@*-31}JH7)R9XKPuhhRsINR>@zfFf5dKMeTqIkyUP@gyMKs2Gb?H8t{@-Y zdUwgYur|r%LhF5K2$5(0S{OW6P#u<%_aN+c#-0ndio3#*pb0zRfEs*pcw=baGhDx) zZK;;ZAgkGg>%OVC)g7+4WFJ$jV)(0_53hbxf800yFVj79;}v09Bh_K4eNVz+ZuuB( zJ=Ua%qd8zR%5c-(a)r=RrYbMhfasg{j4>k`Dt3CG^*V zW|Yl{DOSyWE+nCxJ}fsUJaNr*vs!&d?ZMJ5Z*F|(VGf&07T)W7AUr&Krttm8)#K5% zOJarlKCT{JtzJ?reEtzyV{4XV3crQJ#PV|GN#B+5?^VO=jUCEzV*io)Y=GJkawj%k z3ajg{TH!S*yAIy4q_9Dg8`fE;JQe+@ToHUH*3a+Gjo30NtX_s;NOe0;O5Nc2j>yvX zEGc8mHdQmGhkSf_%6-v%<$$2N1iy22+3iE3V@XY-_PZjVw(nxx5*<@E>z*7H^)K+6 zB(0lO#=Da|CN0+~D$}5XWb(wiFVf+%xyrQhHnb?baw*UG{UTa}8CIeV?ccVpka7E6^j+pKQH&x>VCn`s_aiG10% zeVzS3)H3M|L;TPr-f&{_m1wacvdTHRB7joHC99vqeA=@qe>)uRK|Lfn`661+HPsYzqO|Ks=m~^`?3Af#17+0C1J-^9kX?qWH2`({)*DL zFW+8}+x}eoyBFWROsg(g<+1H&U-A{o)#P<{*t1*@$FYNI^3*)M6V`_tB5$3CKk?eg zvH##@d1;7^q}yV>xqg(iWG_2)-^EBuT&KKU(b-4!_|YkCXc#dg>^2(hIUAOxchDy5 zp>31#=;21AsN0>d?20t|c8~T=ozm(zcI08@XK&;C87q)EQM^ri!GN{^`P|z`p$M&q zF|Adzg958~5GC0`c}!z59&4H2n$xY>HOf5ek_AV3v&?P3?uKe2<5Bim_pE;oUddW# zi`s2+gcKZF*QsiYlG}6pS?82P7M;t3q^5Ny`c7YGqPW}~BN`9AJ&rUV>yiu`%I>RqS2{(xA>p4;QU8t0Fb z7xl7&Qy*%*H(aUdTxJ+&E#$AZOjI@t%l9R zQ%|&>L)2uoeJWGy%Eo>9tIVldEohC+U!_l3WL-&lZu>oOQ6uZeZKGJd*A?EL`y{`m zuG7W(aW+YcrM|^{B;pyclDtEb>Wc5^xKUb@@Ehi%tKTMQGCRxJE@_?WrHef;z^AfP z<#mhv1iz-HGdnT%@Q=W)EUjevVDeo9G*3fRr2a9}xbz|Om2OdxK#Fpoc@fk9D6%O* zWtNB^?uf$lDJl|a109Dja65!Zdu@41QVqf;`cFa<5|FTo>FKKpRZOISWT4a1mr}^o z$F!XOPp?RQ`Og*+3(U9E&{Xb<^`P8g9^n3#8m~9uK}n(wa_unG(^##bCj2;84-qvk<-bSJkP<1SZwI=Ywp39~KbDCT_~ev}Kl zz1-cB!Kd@|eTaGIJaZDDZe)g-ciyuoFW56SApwI)W@2)PM1Rj)#30Xvglvp+@*B($ zk$B#-kg4)XFQ^Xw^jh_!TnxleqFn1%V6HqC<_q6(H89JkR!^>a3xZRB z5LbsEhzFD%)>Hj9I^vG?;O$ox8^se}H#xFdn#QoCIf@@=$r)!TTnK#K;Q0RVy1{|N zTU4z_@u#s~JYqh3)a*=w$I`+4ZKDCY$D5*3dbN!0(RP8x4Wr{el7laObn+htHpvAB z7WpiN=ldq@3T$eK%IR$ge2r(bnvMG!D6O=%pHUO!+%xN*@!PSvP5KR#=5(C^*xGzF zI4BxG(yOVbJWM}sY?}Hh;9~vg8=pOc?|fwP%>$uHG4N~s=&;Ye!S^%2ucy?~`dR-NtJeu&)t9wF zRc0*mpqW3oKfXSamOj{H(V;(86m8a&5h{fvp}(hYZ_i zY>eHTR#GD0%$$ds5EDn+M=OcFoyO%!mB!_xk*`?chIE##YC}K|rqPhiBB?GhqE#gW zbQ|lB=r+|eSSM7w8KJ5=0o9H5$Eus^S*+8lp8|%6!C~>e>Bf7KdySurKn!J!T-6&9 z(0YE!yuNqloAJK7X1C9ED!#``vEjE#FQ$>+B))m4k%R*X1Y^XKhSy)EsgWNfQi zJXUutkX|oHkameQj#iCLH#p?C<&oT&@Yt`$Csc3slA;3}>Q21{$r3RCc;#X50%O_u zO5I9XZ!x3a_)7gsRquPog7J^K=Kh&tu0j5lu~qiW^NV&KVn+TZBmIbC5fd}Tzd#gA zyeSGr0mahO&Ov2_fMWS{rzrk!6et#yiV408Q7mF?Q?ouqu~4^}nsp+Ig?iP*+HAz!rT0#e(X)5KzCH}lxcqk~A*}7DE4GI(s>Jt9*2T&|u>N4rAC{QdYF+6q- z6w3t1gSrO~iX}WUi(Y~P#iE9ah)j(K#exco2%*MO;p>&Vpj0s$zq}L9tw zgd-e%YB3%Z3tss802368Sh2|fcCI7D!&Ep9ibX6V-T+JM2%Fry0Sk&n>_+?uC>F|f z(-&A!EYt-0OHeGlkXm}M8Yq?sVLV+!4HU~oVJh8S4HQd+kV?Od0>xsoI>ys^SPi{a?6{|-XV%5_Y0|LdNDxJ zsE_iWh6)4y5K2jEDhU(|g;JUNRUZ@!i8RlJBRVNnr10=HilA6Xq)$ly1lKgzjdbw> zD3*v(y0?8UD3;vIq^}o1u|$}(P4CYI#iB3P@~j41H}@u@_To#8z`0wE4RCc-9WLBS~;PQx5k*dXEt zTUW!NSl%<&DoVP6VsX=@)V~JBGR~Ax)NupF;s)V~A*N}r(NN3yWN2A8D3)$L{r91u zSUxVQzGBYm z>w30?!1^~R7RmRACPA@0O(HS$L9wL1J~RW0rA3#NqYsKD_mW%8Cq%JyYmmOrA&RBj zo%H=9qFB23k>228!jkfSLMcHYgUh zv%2dgP%OWh_3}QbgJStDU{;`;>3azj3pLCfiJMSjsgmZTDMYbQZOj{&L9xtmLSH3Yb#q_{nBrsST23$)z`8?v9N#UKz^-R#*gr% z)j_eCPfSrN*q~T4>E;t$M6poE%*Q#1VxhK}kMj`4Lj7iLHi;+}s+_v|S5Pdk>0avU z%b-|3aE?NF1W_zs@hl!O6BJ9P{iSFYqF6HfFGaT_iiMqP*7_MyEKn%XIuD9vN+6w? zWe$qP{P+_8J{1%Tv{Lv)<>}3NW>A~eL9wV~B8hXLSU!6uQ}3yRVo`UFBz^$J^2IZq z`dED^s=9uHCz`2h4vNJ*NFdB%gJQ{Ch>RXa6bpOVJZ2IU3rAp;xz`*N%U4=AZ<{(O z7Illi90YeZyrKK5LoQ{9S)RJZvWNIsJdYzOQ^PkCs;`xCPZ_Y9()NGeNPKf1easXM$oe|IQV-WP)Nb|IVQlvq7<_|M)7*WMBE?cAJI& z>&5!wOi(Nqs6Q#!GeNOvT!_YU5yg^8zl`O8Vi~qJ*ZAu)D3;5+7b!P0L9sx*B&whv z6pMxZpHaW}gJLNz7Y0DQWbn1`uHxRypja+9Esnkg#WFN|&cc2f6w52$y~VyOW@DgO zq69eOM3X+2{gDZZMI-kctpiak8fur@5XGV~{P)}xD3(57#bUS1 zpja+{<&VAs#WLorQGEF_D3;48zR3WhSo+&7g886W=9aXItuBLN`G%ce!f1eEi5{Ql zp)^3TM2~;rscV2@i5_3z{U>jd#uwCOf@1kMZ^AgDVgI+%^iRdoZ)m~(8x+g8|5MhT zxI?}8e;?1jFJmx_F&ZK}6EP%}CB>jfQj-`8QPyglrm`FRRuRflNywIjrZC37PMuRu zsTf-aWrmKXIGy{g@Av-Q*Y&&m54tYbc+Au1{drN}&YM_6V);(}FfU;ZiG})NX7zLl zB$g{L=2kD1Kw>EgR++YQfZ+zZ*s%u~?h)CO$!8S&B>>eM*JI@`Ktt{|rt`-cesHuL_hv zVkuGkG4a=CVi`SYy@v~lh0A@Il`3OjX>Gk)_I)gIs_QGaRR&&gj8Qwq3rn2u`i9^^ zVzE|NYsjjzV{81K^Boe4wM5DN(JEfgym!gRACOppePN^FdiVll~yc`68rMR9U%%OycbEDDofwtN$W#Dbdqx#d6SCO&pC$pVmAr2czu zvR|08qoZ=mW?~V@6eE~HVzHil#aAK*iN$bomG6`&Bo;q6ttn zbGHP==tE+WJ|?Iy0Eva~oA8DiB$oAmrpMWh^e#PhTF8|xv$q8`WsJOz(c1W;t`(4V*$p4>gDZ8%h z3>&P%N?)`SJ#W1ycuCm_YC_m161Jt;!nM4az>UX_98`hYOM8 zv{$ypxDiQ~A=3*tBJDL@oIRgpWi<1h9hZ$tqHE8eiWfc3HNDjWuQk z!3~5p3Ty0&7=Fi09?l_=y08bHIVaK@a6Ktp+#P8ykp_tccSnY6Lql~Z$QmV5ESP%# zxft2aw8QH#Rd8C#@Ya{40>zc#>{s3C*(SCl;;=+9Q=A=XJAz!bH56?%o>6Zg%;aV} z*%~4TZEHnacg*NESk4q?2iw+49QM0Hlfs!Ljc+qc9+zW3l7+;A`yj?WPy5DfmR(Nf zZDn3;(10_Ni%Gl#R1DdZ=4!K~c-iJEx>3#zakg(1z7k*XO)#~+ihU_}@xb;O6z=3y z;#Zi!XqWfx)Aq-#=;b63ZO`B$xEE;ZcFN_tFT@(mnNM*qkROwIIA#{^s@$Gro+h&n zcNN*5%+p}LkcGrzvjNZG$^M(f`}nW=>q>?2b{aGmb*vJ%lm?9jv1n(tja>#S4KO*S z&dw=6&-@Ls=n%3My+1^rTS|< z?)d&dg|Z6?X65G+Ua1#jrzFGABwR#{q=sE&qUY_b%-PY1Q54FKZp^kvz$(2R-ITme zgT{g|%A(jap|K!#?YCn>V?oqqjO&mC!cQg`BHH%D>oXlbA<6<0;AQX$)(E%_+5W>p zN!HgVVITX4XuQURtS_wv&u~U0?N`z|pJ{BxfD;sjr}p&SY*oav{3Uz(F7|H3vcfle z`X06b0^R~+k3n#!XwXm+FEuL9mXKS8&S8PVgM5oTB!#SA85uF40-%cMiVV`f3e{iyJF?=qs%|!5c77$ zD&cnu)WY{M6A`GCZbZHv2_1gx7&hFVc?02@@(1FL9Z3cLgXl$#5L!K+q19l)xG5TH zPdO2i!Es5rq(R@_Y){$45J5DjsUob?>rEK&e1hmRb*h~2({UvC73~Hs^^nF@#lvZp z5*p>o+*R7mdJRiNJMELWQMnqP#=9@7u^%x_JC(AHjK-&Br`2i5BL+l0xXuaB5ic@0 zw|UkK8N}6eP9D#KVT8Dv!71YHW~d?7?0N1R3RBCX|Jw1)tHdym_dhCF@18h?kzMU5 z5b3r$_Isw+_Bk9L-e%ZsWi^MuXUSjid!6RB=ZpZ$_{`k`hiQg1iGyI6Lb zskFU2P~Rmp+%a7X@1o%BXPU;c+AqP{vAQGcfCtn4b!5dhxEwsdc)K|@mgeWgW8nnb3-opU1^Kt zs1(-D{qQE|{aH%8{orK_eqU2q=KJ@s_VmOk!4$aMn_eZ%bdnM@7X0~CylIbdcx3t% z9v*3`ZZVkqj9;cjKWNdLdyijEiLPrgn)?T@M*Ch$(Vn9f$n;X}tkA0s1zEjRoAmVW z{_qFlk{>erdzFxWA?e{{YWf2|E95l1s5gDyk1jiHkH_^m;k`4tsg#{P!Fca1{9$4$ zcR%b_z#D?_72Q;mxhTBG+)%+|@`z%(#rC<0f>q{5fdttwz4d^8X6v@Omj#2&l>$9- zZF;L?O(twbzb#lK|9(sj4ni{UW+}t?W+p|i=N;UETq4s|rgt|O^sM8pX$>crXk1@d zjm65^&QpSCSqe&eYM4h7(5sINjuTHR;fD)KdYckdU!{DvCwfxw{}k*SGO|}P zNmcIMaNLG}Rp6CQqN?7d9qy@24J9vSaL=VfW5HjgasASvvEZ+!aD&sKvEYBwV2q-K zU8i)dm$$vAJ=}RRVdq)R38Sf>?3e@*Ei3J*&hm>4OEH;)6nOZ1YDz5Y0Oej~QN1_g zhnNa2NlZo4N_UFIE=cIrA{cgZ*$IN%wFo-<{K6yR+~rIUhE)X?mU_j_i5;hH9*G3r zF3kumnrsq$u$^FAe5>?i++(@o+x0ybVx!I!Zw53LF(V@7FasKkn2|FqPKbniF$R&| zIUIOslaVQw!#Nx7%m_wabR730<(E3e)r!HIjo5G7KHVyX2fIsM<6emUfU^+cZAt|` z*)7(clY-{kl+@1?!`$P_#FA2qkdm%pP7G))Vxg`S2L?2j|K6Q+UlCh&&XHckz> zvF)l02bL<38(LRgISz4G#D2Bdsrt+vG<;pS$DWjCTjaV0>1m84mvH* zy*Ri0V5+PX57t!tQQm_K^H%x8fV@rRzIK4o06JfMXLQ z!voR218HvF`i2Lig9g@gcF_jiyzi`}UWdlwc9EEJ9U2QPR5ZE|jIi6}Pr5}p|7Uo@ ze)QMO#1*%%&MeojuK2Cp9rDmvbc7$Y;JRhQ{ZfnEB%SfZiDEJ(^@Q68-SbY-mj`ya zeSqbP=t~2a-9G4jaf-e+AmRou405zkpY767Lu6IJCdIT)iL8dZ^{t2_0}p=2vvhjy zDHV~6mHxlO6XF4`TWY>LUESN6dpz}xJ5fH*jh;h}NQH+rQ-gFq5-m;+9k4KgSqj*r z7&&NRK5K>FGaGkZ!rf3V&E4=|tLf~H2CLcB>w50Ba@Fp&2V2c%+vIn?elVU*_Dgl% zJOR=9;K*$rFJ#}AU#_6IhB$Grrpma5pmw=Y3&;Vf&{%XX-sf%WKHso?_JNL`jzU?* zhW`DVZm9i^CbRQ82EGrX+^*)lRPc6$qm&gVL`6CrrL@Z(956k7Qyq2cfwtRqoho*b z{AG7)5qlT+N~)UMMgbchkV`$~wo%A7=Uz_T0Bz zOFLofbgZ(>0Hwb2c7scg?GCK2^dN=Oqp<_4vv8&+zu{_+aJg^Ke&wtIN)thAxAa&@xA08h&9dku#q!fhN0pBU1ZuoJLon$2OL;^#;7nlt zRE=anKux$yPo&A6d@eP7Uyp=|_jQ=2KiK1B(mXo8k93o}lJ4nGcO%_s@Z0ZCKTgWy zzD>Wq81 zo=tiKI}*O-1=puCB=bz*Wl|=@LEf33zsUQ0ZYc+xE;cBD|8A@De9Cuals~B;Jm*+3 z{BQ3o=cmjnd-{`Zh2J=KJKVYFiSnD%l?~QiNs=liA#z0~A^mWH^0fQ4xSOm1^vF9Y4amabQP<@?*(Xo~3P0+0Y?{kN)FDQ|~09v6yURPxVSd zV=>vtoq8h)jm2am$M;kcG#2HbBOKSHYmW;%EP{U@XgxIojm1J{DEiPyy#43m3lZ4d zRd}3tq$?7evwGmLdBF4Hza!0Lr6SD+qK}L~W3k8@4ku+oW3g}<3U|xwVb5S5%mn66 zA7Pm~JjM*ey@)|%(*VoTL4{@38)V@!8lFTjad?Iam?i8A$cT)%Q?yDihQ<=Hd1X=z zjU~dn@bQ%qX?A0A{ybfM{sz0MST$08{ttFx@te7&>yIC|7&_!*PDa8`R^-<{mWczb zVmXvxu0(!C{uBA@3}kEgmCR0y^L==`iGt}0jBTVO(tKX&&h*Ve`Ao{_b{O|1;u`;1%-acN6aH#P$m&0 zLl%1TA)Uo=buw(>IyBn4b)HcS58xI%MHmjZ67@%0)!}%gnz>LM98o*mYUq&hw}siv zc!6Kuc(I>T=FwaUA z3P&kbJgwdjn2&Qj7i!6LGG@;l@8Cx9bUvnij_15_XF3bB9&dD)oqcE%cVwEaajW_FF787NNHa zgzAnzg@TLSITLa73Rb5`hd7Kllmh)2$&K@Jb9O>DUx7}aR1ks}VJa*glZBm%H}dpm zp9gW!ScKZ-KOKizPw!HP*gHb-l0u;$@}EwWmV7$pvo{tV5rO*?Ny9MPSZFN#<_b#3 zeF|DyJ2?DL<>?+1M>%V8m*na06CRu|ao^I+G==)v*S=W1YPc z*0}wO)W*rmIM;tf?n=R2XXbP5Tf6iM3@Q6#sppiXEk zoeDypiX_dWD3W{Fxy}nhXKRe=%Q|}ap|S9v5T|=go@;p0u}kQK0S6w{p8SVjO@7?) zqCBs@V}u_Xi^P&H=U^-}mQGiRqvEGzv_sUAJ(W zzPi4WLb}9N&D^-4&YHc3FANXKYfBvIHWyk`mD z4UMH@=85L68_-zvSoiJT$ib|q+-}FO`p{SmSr69G0Tu2#w`t z%R}av95fcqr2Llqn~f#+X+Hh#W@E{{luv)K*;sPFvGn&kT6;rd zX`IQ^bhxqESWc3ODGi#?STye#a)~L>STye#acxqdvB(+SptNo_md%Ti=g?RlGF3FS z@}aTZEI7I{oC}R5*TiW=*WyI|OmS|c6EqelL+#djXe=!=xw%eG&{&*mwObpZv9!$; z<_0^}>Oo^kkQN64jPN*2OaMD6lg4R%lW)gXe@W(jO5}?Xe=bMXNs#M zG#1CJ#~a`^-8`Xe{}>yUZ7I&{!NdPO?kn{=K;~wz}qU z^}s(>@U{8SY^(?N$Siynx^bb(>fR=AJ+R)h2oE63QLnmzA0F^)s5<1x>|qc1XnD9Y zuA5oC`QxIn{b;zG0#p`{VmJFdUHIl)9eC7!6|O~YdNfs)Ur+$y>bg|3zv_Zoj6HWUeaOT6_~~_BFm4r{KA=|xZt*zXt9y7Bei!N1 zJ!*8n0iQ1(Q4h<572sz^_qQXg$ElE76s}Z#I$^(u)t<`dAz4GtZh4W~=`nF=ler4! z9{8qIpo?x2texC|Uq@B+=L5T{Ha}u!-+O#;pM@VA-Q~KKSY4h*b$r&( z!`gcmz7^eYJ!(<)OS&HE56K}4pcmacwd`BE@Egg4OI0vB!i7(nZYK{qsT+QBbiJ$D zf9pbU@t_~>p3#Lb(H_L=m0?}@V&pMW6LIv$W1;F|_t^n?VrbuQ&%Kkh>}J9c(S(gX(C6J^7s1MB&wy@i6iwkj!kV>Phs2>&(k3qx@i+18sA{J z+Mi7flCSV6aG_QhKW9jao1_xOP4xQ8w^sL6Y<(7o5w}KJD9pHJpV)`Z@!V%E^j$$- zR!D(bR+=5hD5u4hi>K;n8#-!hKP~qucoxVyz*XsBco2? zsz#M|O;vp%W1&2WQ7B$kf3xyYeI}z#d@%)%ShRUZ_t~-z?DG?HmN!)xmN)erv)Sjw za@-dzbWXlW;j?^`=h(h|4y=Rw0)$S=M=89QkMiW)W-QywB<@yNa~nUgQOa>h+@rJM zwNd`R4p5Y8!f3zlPNu}KdGLxDAOB6>BcFcA*sW6=f4oyk$WH;jgDVgZaUR6AGe3&| zJZyZpa~s~PQ(nkjfrxyiFm%Y+y;C25q7#!CsQ{lv3WkSxb&SL~ag+|JChY(2k$BQz zT}OxXC@kYxb=Ymj-{R)93~_UviJGvqxRyua%{t>SB{Tle=hrLNJvwfjDu#|m5LCRi3A;ZJ;Fud8< zokg;jP%yC9Q+VXjbeMIt$(`kGFQTAozg^+BN7P}KN0dA3WcMZAcGT`8Ye)WcIvvW5 zA5io7d2c0vn>JvryIutk=2c@#BT5JMk?RL=y2(`-t)%J=kJ(V&YyYbKJOno-2Ndoy z0~P#^qa1F@Xka}ylpp8;W0BrA@azG2Qb*P zb?qDc9I_rT+n?}J+alSYT{nR=Aj6?xoW(E{47s1kREO_VC*BLxgMu1n%02BKEnwfnNd3(+G9y4IDn-zh#B;bNv*zw>eq+`~8Y zWbM4Fo*PK4!r$tdQBUl-j&={)ICAfG!G6Ev8GkVY(654rO7VC5?%^LN>7yG{L(q-E zePvUTIqtTGQms2@5YanlUS*%Mklw0U`IPZhd4sK)SYV9i4WIymJmdNHu3y5UY`U0G~W z=uT7l3{%tBV|KRgETV0sl!0lJ{3Fw**JJx_Pq18UFH5PKM#;l@$?GwDTThm=?Io#? zK^xv*$Hv|Lj53~g%b>6Q&dcj5={7>I2QF1Nlzgg#9hO+MNJ=LP8*M=)TEF*iE?>6VeQhR%M^dIOH#^Y z=aAgvokJ{w=QJD=P4j2`dzPeK8NN9_0Z>j zmEn0k!|11hw0Bj$MML|Ub15^^k|OVIyCQ~o-@MywPSIh+~nbizhz#UyrugB@&2lN&hY^B?Vt-4$dbQ^ zbtds6>FRkSLza!7`g9xr>Z{dA&K$LOQEiQyd7a}h^GA-W!&OxwyM1t`pndOdy>|KC z%1p9lYMPv-dbSN&5Pr>iwooT{hns-Cf8+Nv0Ph(Y3K#Q zz3KW)4R&RwpJnDE0q3$nxIf*LX~M1RffGf4GnKYEeiEN~AWOtdDZj(V1`ZDFgpn#6!kYvS9fXEr+=be(m01;sml|C{+#;9|rw5eugE>y|01@Fql^P#_h<|-!XPp#4M5*R%ePJg+ zq_a@i7a(%3KC`n;7$72yR^q+hYs>1}>nGqW2@sJqH6LT_bz%+d4Hh^j2@sKNG9T;Q z>%i*Y8z68}5+EWOWzM_2!lTMop#YIUPHyL*FhE4OPMLllAQHs6+4)WwAR@f1Ob-Q!1afjZdxZfa z!X?Vo^8k^c$s4h6gaIOa-7#kYBC$_};jE-yl5XOo5|*$zdGaB~h+m*ARz_G|13T_d z;?EOv28i4uhOGGI0YtR&3W;X`BECEB_aD&$h&Wf^Z@u;hi1dHeGARIv+!_i|PfG!a zq%3Jg6aYj_a|Z3sBLO1F@5C8%fQTtKF~vp;AflC8sC@@aw!6c6j}wW(E9-*{Z3j&MF5dP`k9rset<~-39Zl~fJh-dVC6+W zK&0P6ORWeXQb;|s(wYJgN%7S3E&_;X$s?=8oIh!OC^RM(Fm`Z963zic#73EykuOAv z30l?#v>}R5TDX|LRue$PAK%cOWdIO4Mm&qZ)eR8ozOLx*1rTX2=pi3Z1BfWTIx*w} z5b?)9PSjTfh$uFm=<@}Lys|NrXf*(c7|cA*{%FH@c#MHk6!!v%9Me8qR*(n~Nz7Gr z^wK^D5b17IY;OXH1T@gQ#T5Y}ih3tD-m(EA-BpU!UI39}H9-iNK6wj^CpEzUkx*DO z2{izSz;?+P^S_fN2KoRI{if!z9-9-ak_$#te z8bIXNtwT3JL~;GtQZPUyG(4s|RuLegn0yR#79et->zUT>wF!|fhNy&w0YJncy}|f3 z1J2ACf)dw`14PbEslksXK*WI9VEhNe6#t$fDPdv&5HT2PAiQD2Uq$L{Zv{5CRZ6J5`W4tq2fNbnzmb1Be83i@R490V0Yu-(C0YrwGW_y~yryL-gJRJ5n1BjTpza^dq zh`c?6E($sa5c!iBY7##T5E&jaYa9oN{MmP2BRK;gqS6{N(+m)Kk~6yhstQ0v<Zqsnf4zt0x8!zVoBEu_YdgB0*KWjo(gNpznX4P+N&HzMw zA6Gb!14Q1{_&u&228d(`o4t4f5NTwM??~53Gl0nWlHcRHVSvamypH%SK;#{U99cOG z5E(vo?&q0b5K++xNpAv(G~;`Ej;R1dRFEMXL7NcCxE4Ze0*Ex@yL*nP07T#qIYUi& zr=ezi?;NQJAW|d|@-rA9GHh>#`hOcD0Fhx2GtYMbk+-CP$mhcVkzr>u^LGF0ElGxnPt8Mh`c47o^Hwjhzv)XjlKhjyruiE zHVy+shJ(%O-T_43(oe584+BJoFPkmD1Bkq(`mZ(&14M=c%u3z?MBY+PPd~{hp!nG~ zl)W8GI6wVJsw+uHUu8LD+zd1ImJ>pMI+fP3KX5ueqc2nBB0$8~CK(_ijoh^{zF9=N z?n&oa07MAHzl?8m0Ys96FclX7B6G&iyCkInBGM+i`q%)GxvkIQG134L)tQLwcThy$ z?6Vg6t_l!QrB0P*#{)#-%cWB-03rnKze=yizm%RHAm}e{E?uNVNDuC+nFNT;)Hin3 zsRBd@H4F8HT>z1;Lg^`h$mjaZt~P0ah%}nOdksaTZ=autGX@}nF}*a#fFd%mFIeOp z1|WiIx-`}aMWlaUfXGP9tsb*+=wiy2c5c$g|KmNTmKty`YVtg4O@@Xl05c!L9v+JESKty`kg1!t8`NYZT>XinFNS9bpzW_x3 zn!FMJMj9X@&>gz~5Q%>(Eh$n_kD*8U>MBBtWE< zxU}Mz4-hfTYa`A9M5cD!A3S0R5II>9e(UvS5gGhyXwm`@X&qWpPfG)cq%9dnv;ah+ zat7_sqW~hP@1&Va0FfweVw#O1K*TV$O?wU?GF5&r&CSs9S-ReIW6hw^Nq|VJ{&Hj7 zAV6erz_9%yK!n{`HF(w#AYvHYmM{nq8MHK9Yy*fq3!G%1KmkNh87ED{0V0vC33~|? zKm_&ZWK%dmB$D;PUIYaYf#dk7aDYf8>wWhnLx2crE$Yv~%_1_WcJk-UCPW6U4cFTM zBF`|B5v79wkwKhcavMP8*~TBUp@y3f34;@m11Nw9%8!(O0U&Y--`QPa2oQP3>+QBf z0Yp&lB;o~t$R&JNHxUI8K}C{AE&xO>wRX(58=}P9NHY-)|1%Z>5NRXLMm!k=hzwpZ zeDe$-(l#*@v3b|QZf+Z&BR@+6h@}16?9f~5Fj#m!Z7q1K%|X6zw%-bATsD+sP+sX(ng(I zX-xx&q8KLdyu%A=|zoMF18&6rfch~SPSeg=q0jCNl}y$~fO7+SZ`CMZ5x;bK-= zUjZUB_=euB-2f2^aRGm;7a-Dm9qS$q5Lqke@jIRc5W&8>JTwUqnZZ9!*2e-wu#K1d zrZypBYbevY8z8cKW;OdG6p^tm6jnSMAVSezC@V+?h$QD?9iz2B14Mcoe@$9w&o|I} z#jyYptls5~x0^+zw+dSw4G^K!{Dr^*M6ipLnni%fa#?J0=x%_BmFe0T6N<=?ZKRBW z6+pzQX>F_ripYrVWf@f~fQVJp+E_Ofkzw0QG9RM>BAoG8enwdU5$v_Eyu4n32zH&a zvP7-E@A;9*wASDXMo5e_fGFR79fIE zi>59CL>9QWdgrkK5v+GK=mjqn_howDlYexYhS;E#Y{sxGw zu*MybY5)E$U24{Svd+289jC7=iD!dsA*8szXC+o@IAfoT5Bdn z4N2YjYZD?_*Qms=0FgC(ckdB3fQZ_!6Aq`LHGJ-2WdX?`6xhS)Y;m6eS!@T$vS750FfWmnbn3-fXHZob;&wFYxSc9%V-my@=zoU>30T5BG zcj;*31BmdU1$kyrM6~x#Z*k@ai13@rjHyEr(cQba5bVV1Q3y{ckOt=2StQWL=d9{MFbOeprecriU{8|Dc*^W zO^EQPiyH5MB4XK5u*FXRAR<5%E01EZ*3@MQ`SumBO|$s;kZ_y8h&qe50{01+jQ zTgMit_D07Z>yyF9i@GT3`)c_(&oWmWxd;k%?5+N!UAfi0!5%Y$R%y!sn&i*Jk!U-?!n!JibDEoKW4NhD%crn3Pe zxV$8yB0vPO<8*?p0X zw)-Ia-DXAxAR@`Nr`g~DBDmBfZAE|xqTHY6hI33#*F!dj^cmR#L=yGY8g2Sg%ia10 zaP86n5xK^*eP?k15nOPRUElHYi+z^3#Uy}8a-b~xovf7oUD*selkEL ziKGy5p>HSoT;B!Un`D4U(u90OWS;?!-JCS8NWM%{B?r(>;w~mb5lI?Hk+0DN*giB{ zoOCikB#EHNrqGaVf0{eaB^e-+WQAIx_DQhGeNH%|WPnH#U2)~NJ{9(vz7x1mxLNs< zL`SV$?Gs`9_Br6xk^v$~RK=BO8j5|I=85x828iIU%bpc={)GFGWNcHw5a${us4@`e zMU9y2vM)q!5^&ZDGzMjo7A~f)l@1V*$A>esGyo!YL^R%uIiqgRypC|U2Z&@AXp!I2 z03wK24nwj45qW%Mf<6Kuf@pN;LjXifY~W-?10bR?la{>)iij}-g%Gy~h}dbP%Z??m z%k2_!5svoSs`y$Wvk}pr4n+jjKxK*}03rxIhYd?8B96=|M72FY#I8mefdGiW4M~j( zKt!!nSOh=>k!*)S14OXgH?(&9O^9ePXrdY# z01=J!6k{_eBG!xq(QEqwBC1m*qVVz+xjYSGit$b;BD)#sq9z&u5sje~f;kkCJ?uD` z(jaLog&$_Zl?LrTBHx~b1&AnzdobZjgZ2dR#-5}G5K#(uXAUEt+QEYWz9$ksBLE_Z zi}vHH01*}6Qwjed03rx!djbX^f}T2-FpU6+AYAMTssIrcE{VAcLn~@oB%Njg&n(Tg_EH0I!_Pi||m-PbSyaoOpw0E(qwegAjKUj-30RR91 literal 0 HcmV?d00001 diff --git a/student-tests/fixtures/sample_progressive.jpg b/student-tests/fixtures/sample_progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa39b16d02eab6be1b6e12b07f7e98592b8c0c29 GIT binary patch literal 8520 zcmeHMXH*o~y6tYdfkr|%86@W%MFoK-BOsvx$%r5b5)@D*ND#?6V?c?L1tm((APp)a zC@3gdk|H@tc-4)Jv);PSjC1F{Klc@^UhCAU{e^w@_f^%Y{lWcF5JFu=O$7u7gFs;5 z541l7QUt;9pin3t4ETY;VE6>Ygap7rcH{`07)eG>jwC}ODX1A}C@ATvkVslqT6#uC zW@csz8a56#CJqKBW+to=FaRaMCm%e;x?*j3A#=$Rpk)t_`rm*^QxVFkbM9lZ!&Lfd)Q?* zsn9H=Oe(_4%)Ic_^QnY2F5%8~c5g4R=a7_=+`P@y<{cLwr5g-6$9isNYg0tHN!1MB z6F z_!7)6$qtcy%Pry&kD(=dVQ@7*=1s*TsL|qhRl)GlR~Jt6D;3HG z2~a#Tw^XQwGg^eq*(~l=4qf%TKIBvh{D15St=mu}%@NG2w@smvZk9$~o1h%?=zY|o zdgiG0QLvCq=oULCeKs+)PNBYcW%0bl&FCuPWfG#I=6d{Nx60Ma33Z#L+TT+0*G*m; z@%*x-s$ALi-S!B|=<7Rz!YP#UGF$E)?!hB}kDlt(l>1wPADkNbXTBT8hHgLHc(!X} zPhWopx(}*=^+oChO{kR7F^V|FIzp?~*(W0t?Gr`Fmxwl0dz$@yAyT8_oyYi|;;+)a zChFyT#i{|Fr!mqgZc+WR$E&$hn?7_UJ_X0f(3R-XKR)I#6cjDOXS_0ck8M|NS%!NnA-13KgqKok(?oNZC#KVZ7}rl2Hl$2u&1diq==w7SV>9pKL#Q?wEsU zTp#{B{3eYUE7{W+m3&38uoG69j?q;l<$O;G>|PVSZ3+@_atk_Q?dJU9;oPhB6h-|( zOPI`q@wRu2N_y6O$m?h-)OqE;N}8ynKo!;B)QG7(jWHzwdLjn`1hhE_1cGgTxYma+ z0fHfbArvV>PC>~cehlb;guoma1_{7)r>}T3>zEP@O+n(T<%W3llhmoJmGW}X5?vBD zRYAOxFfcm}MEDdEVQw zm0mjC!)rj^W#A!wzlQ8Y^BU}lPlFCGe-=%A_v*5BD2|{0xPeBwcbR zrh{CRvMwctW3t($vyC)(jYpSVXQW36`l*}Ox8xoc>uqg5`6gq5lHvSA8Jyh*HGW>* z2YvcOGLo2N9c6nApFKnWp--KP>4~MOHP4laXW^;7H)J4TDt)|E3F<1I>Q9n&%bF*r zUU@c<rwRZp@FfbmhM~`7I{epN{L&9-VmwF?7!MB*ifv-wAK$>hNK70lA18WD!3DuW zPQl7%iECuQB`9!+$0t{VmT}s0hrFj$@Fkt($OUw)@3>}`ha}4u@rHE5GIu89#9fT( zkx;AN;>=-v?sf$e@oE+MZ(+xyMkfO%#fO)*J5xj5qis1Ig;*n0Y9d^8(x~8eFQ@1T z7C7FY2#6<(?YW9Uv3aq6l)Rtxq$*gDeon9@vUT8_QRMLyqHc#0GjocaD6V^n*34B2 zb{|Pk%PPwBaO{N5{601$-HJuwi}}6SESBB26qQ9+`DMBVPwq)=rBlzu<11+p?OEoU z5|8C6l=?o0WxXdGRx7_kz`@L#Ivww;w5nKlRg&M*h1f)*Ka0<2Kq}^^)26|NJ2M0G zx_i9&D(Ii*=-j;zDtNr9XJT+Kx2r(+UcTNwsKoC9k=V~2n}&z;7sSlL#meW-iWLPF z7tFp2b^qaM$YlUDd8U?vQnvSb`=e>zNtbXOlZ>)uh@%0wZkF1AK<#TdBawj zOA@_L@#!P_&=bYT>w;%EhC{eplCQ)uIVABfQCL-uF|@-bByqU9-jn5fx*x$LwMpo$ZuKG{_OLHVy`_Eo)cf? zJ_zlHijP<$2%N=G#9`)wGgtdP<^#g#_ZwMC3`>sC! z^0?2^a3^0@@tg2u|G9fHjLs1y_Lf|elCqR#-k~dI`4*umR8Prbre5>kRzfcrR+}}d z3ukBwzlnA2~rKIYAJ8 z5UeQh<_UuVZ=_HtKHh<1!Fc|+poo&vW(ZQ0oK-}gO$_Zq!NM+{k!cwh|NW5-9|a-Q zxIp=ORKesyZ*m83c2i`M;uyYjMCsu^Je=}f~Jy>5S+<`3MM za8GBB%~KDzL;|bGz0Ki{R`<*ZHS}u42HTHSA(L~jSzwJGK!KUxL20m1HY`+BTL_v) za4n4saT}r?ks%nVr{NE`cULjRdKc?oi1>^(dH~hHdOd_@;-D5-DDPBL!stT6WDEJI z+R`=7lJHB;tb$rI-B|A`KuSJXqX*DSSg*f9EwIoCEVNx3$bj*+_Zt%H?Z#x92% zb|vAmq_eVTSONBF6mVpd_b)`^^0vwUK!aOBZ^nq$rh4y4F8`g#`dyUY^kV*|k93`n zPs7AO+pgnb`~YizG4(4JYTbqi4D zujNCF63+4E{_Mw+=Kn2ji{=~cFY%uCWh#HyV(^#5uer3Qz41#?QY4ej?TzCq4Okz* z88^1l0y&!!#%F#~q+6{d-E%(me~l#@L$rmBNULOanM2j9Ky=W&7o-icw^y z=^ht;ZlU_hKYwb$@~}StdTF<1+O9BB5lqZ3`KI-ZhP2O4Pk&Z?_2}2-1r#L`1?}wa z85r1sXtYHpqDo>gQ~MyrZ$Nr8yQ-|9icZ8s?sAt0EFo4!5! z`HN*$QWUWPh6l0u-o!A<}czV_^a5YP~AxvLer~Kp$k5evnx?SB}aT_|x^>H{UbRh^UO9spne;HTa3v~f?HA|U^&=XD>(DHG8TPm=ot0K$>F!(tz;couX&FwFj1OY zl%W%x+$Hk$B;_>OUcvxx3ech=}KW*!ZDbgV_nNX>ffjhr>f_CGw|1E_%uPy@Cl z|JYN(sR1c(S+a;=MQ8ZO^|FlNyE#`6fCG}zvfF3 zcxlVs8=0MHlA3T)yF=py_ZT;xfvTyPx|wmcgn8%ewz8hh-RG0rqKV}idaoTZo0yA} zI~=3-o%0vfG$KxL6;?B%P8T?qN#0LW5larU_tLzJq7xWS3%pNa%;<>WbzWT5z#j^% z4HrEwTq7Qo8pCkRZh^jc4{ulx*ou`GygoQtQ?o7lG~9XM+RxmC44u3#bA?kmTG&k5 zPf$_X%LDy2ILFn8w#$SwfA+Zbj&+%b0L?gaf$G;B>{+Ho$WVfe8+gV9&h47Dl8TP#HFj0weit@w{f7vQ!R8m;zGtw=7 z_g}^rf1CQUrlG5pZ=C+Si&rIOBW(o9`~RV32K2>;zN z0+O8R3)GYHj(x$CWXOH{$()vk@UH7+n%kFT_<37r2Yw&Emj83E+?ai_Trh6Wb zM8i;AjbHYzaZ2j)D5+b!pX|%B?gRzT*5^2@b<-dVQuw=fz`u`^R<>H}KQrEc_dl?z BO#A=< literal 0 HcmV?d00001 diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..bf6951e1 --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,93 @@ +"""Синтетические изображения для тестов вложений (только стандартная библиотека). + +PNG собирается вручную (zlib + CRC32), поэтому можно получить и корректные +файлы заданного размера, и намеренно испорченные варианты. JPEG — небольшие +fixtures из student-tests/fixtures (синтетические, сгенерированы Pillow). +""" +import os +import struct +import zlib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +PNG_SIG = b"\x89PNG\r\n\x1a\n" +MAX_UPLOAD = 10 * 1024 * 1024 + + +def chunk(ctype: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF) + + +def ihdr(width, height, depth=8, color=2, interlace=0) -> bytes: + return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)) + + +def png(width=16, height=16, color=2, depth=8, fill=b"\x40", level=6, extra=b"") -> bytes: + channels = {0: 1, 2: 3, 4: 2, 6: 4}[color] + row = b"\x00" + (fill * (width * channels * depth // 8 + 1))[: (width * channels * depth + 7) // 8] + raw = row * height + return PNG_SIG + ihdr(width, height, depth, color) + extra + chunk(b"IDAT", zlib.compress(raw, level)) + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Несжимаемый RGB PNG размером около target_bytes (шум, zlib level 0).""" + width = 1024 + row_bytes = 1 + width * 3 + height = max(1, (target_bytes - 200) // row_bytes) + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + return PNG_SIG + ihdr(width, height) + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"") + + +def png_of_exact_size(size: int) -> bytes: + """Корректный PNG ровно size байт: маленькая картинка + вспомогательный чанк-заполнитель.""" + base = png(8, 8) + pad = size - len(base) - 12 + assert pad >= 0 + filler = chunk(b"zzPd", b"\x00" * pad) # строчная первая буква: вспомогательный, игнорируется + data = base[:8 + 25] + filler + base[8 + 25:] + assert len(data) == size + return data + + +def huge_dimensions_png(width=8000, height=6000) -> bytes: + """48 Мп 1-битная картинка: файл ~50 КБ, распаковка была бы огромной (pixel bomb).""" + row = b"\x00" + b"\x00" * ((width + 7) // 8) + return PNG_SIG + ihdr(width, height, depth=1, color=0) + chunk(b"IDAT", zlib.compress(row * height, 9)) + chunk(b"IEND", b"") + + +def bad_crc_png() -> bytes: + data = bytearray(png(16, 16)) + data[8 + 8 + 13] ^= 0xFF # CRC чанка IHDR + return bytes(data) + + +def truncated_png() -> bytes: + data = png(64, 64, fill=b"\x13\x37\x99") + return data[: len(data) // 2] + + +def wrong_length_png() -> bytes: + """Все CRC верны, но распакованных данных меньше, чем требует IHDR.""" + raw = b"\x00" + b"\x10" * 3 * 4 + return PNG_SIG + ihdr(4, 10) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def sample_png() -> bytes: + # В compose-сети (ops.py in-network) монтируется только student-tests/, + # поэтому копия публичного fixture лежит и в student-tests/fixtures. + public = ROOT / "tests" / "fixtures" / "sample.png" + return (public if public.is_file() else HERE / "fixtures" / "sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (HERE / "fixtures" / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (HERE / "fixtures" / "sample_progressive.jpg").read_bytes() + + +def truncated_jpeg() -> bytes: + data = sample_jpeg() + return data[: len(data) - 400] diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 00000000..b2772f90 --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,169 @@ +"""Фоновая нагрузка для сценариев отказа/обновления (только стандартная библиотека). + +Писатели: POST сообщения с Idempotency-Key. Неопределённый результат (обрыв, +timeout, 5xx) повторяется с ТЕМ ЖЕ ключом и телом, как требует контракт, пока +не будет 201. Читатели: GET истории и /users/me. Параллельно опрашивается +страница состояния HAProxy (UP/DOWN реплик), чтобы измерить время обнаружения. + +Каждая попытка записывается как событие (время от старта, тип, успех, статус, +X-Instance-Id, задержка). По событиям считаются RPS/ошибки/перцентили в окнах. +""" +import csv +import io +import math +import threading +import time +import uuid +from urllib.parse import quote +from urllib.request import urlopen + +import common +from common import actor, create_chat, request + + +def percentile(values, p): + if not values: + return None + values = sorted(values) + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) + + +class Load: + def __init__(self, base_url, writers=4, readers=2, timeout=2.0, stats_url=None): + common.BASE_URL = base_url.rstrip("/") + self.writers, self.readers, self.timeout = writers, readers, timeout + self.stats_url = stats_url + self.events = [] # (t, kind, ok, status, instance, latency) + self.lb = [] # (t, tuple(sorted UP servers)) + self.confirmed = [] + self.same_key_retries = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._threads = [] + self.alice = actor("lda") + self.bob = actor("ldb") + self.chat = create_chat(self.alice, self.bob) + self.t0 = None + + def now(self): + return time.monotonic() - self.t0 + + def _record(self, kind, ok, status, instance, latency): + with self._lock: + self.events.append((self.now(), kind, ok, status, instance, latency)) + + def _call(self, kind, method, path, token, payload=None, headers=None): + started = time.monotonic() + try: + r = request(method, path, token=token, payload=payload, headers=headers, timeout=self.timeout) + ok = r.status < 500 + self._record(kind, ok, r.status, r.headers.get("X-Instance-Id"), time.monotonic() - started) + return r + except OSError as exc: # обрыв соединения / timeout — результат неизвестен + self._record(kind, False, type(exc).__name__, None, time.monotonic() - started) + return None + + def _writer(self, wid): + n = 0 + path = "/api/v1/chats/%s/messages" % self.chat["id"] + while not self._stop.is_set(): + key = "lg-%d-%d-%s" % (wid, n, uuid.uuid4().hex[:8]) + body = {"text": "load %d/%d %s" % (wid, n, key)} + for attempt in range(200): + r = self._call("w", "POST", path, self.alice["_token"], body, {"Idempotency-Key": key}) + if r is not None and r.status == 201: + with self._lock: + self.confirmed.append(r.json) + break + if r is not None and r.status < 500: + raise AssertionError("unexpected %s on write: %s" % (r.status, r.json)) + with self._lock: + self.same_key_retries += 1 + time.sleep(0.05) # повтор с тем же ключом и телом + n += 1 + + def _reader(self, rid): + i = 0 + while not self._stop.is_set(): + if i % 2: + self._call("r", "GET", "/api/v1/users/me", self.bob["_token"]) + else: + self._call("r", "GET", "/api/v1/chats/%s/messages?limit=20" % self.chat["id"], self.bob["_token"]) + i += 1 + + def _lb_poller(self): + last = None + while not self._stop.is_set(): + try: + data = urlopen(self.stats_url, timeout=1).read().decode("utf-8", "replace") + rows = csv.DictReader(io.StringIO(data.lstrip("# "))) + up = tuple(sorted(r["svname"] for r in rows if r.get("pxname") == "api" + and r.get("svname") not in ("BACKEND", "FRONTEND") + and r.get("status", "").startswith("UP"))) + if up != last: + with self._lock: + self.lb.append((self.now(), up)) + last = up + except Exception: + pass + time.sleep(0.1) + + def start(self): + self.t0 = time.monotonic() + targets = [(self._writer, i) for i in range(self.writers)] + [(self._reader, i) for i in range(self.readers)] + if self.stats_url: + targets.append((self._lb_poller, None)) + for fn, arg in targets: + t = threading.Thread(target=fn, args=(() if arg is None else (arg,)), daemon=True) + t.start() + self._threads.append(t) + return self + + def stop(self): + self._stop.set() + for t in self._threads: + t.join(timeout=30) + + def window(self, a, b): + ev = [e for e in self.events if a <= e[0] < b] + ok = [e for e in ev if e[2]] + dur = max(b - a, 1e-9) + lat = [e[5] for e in ok] + return { + "from_s": round(a, 2), "to_s": round(b, 2), + "requests": len(ev), "ok": len(ok), "errors": len(ev) - len(ok), + "error_ratio": round((len(ev) - len(ok)) / len(ev), 4) if ev else None, + "ok_rps": round(len(ok) / dur, 1), + "write_ok_rps": round(sum(1 for e in ok if e[1] == "w") / dur, 1), + "latency_ms_ok": {"p50": percentile(lat, .5), "p95": percentile(lat, .95), "p99": percentile(lat, .99)}, + "instances": sorted({e[4] for e in ok if e[4]}), + } + + def errors_between(self, a, b): + return [e for e in self.events if a <= e[0] < b and not e[2]] + + def verify(self): + """Все подтверждённые сообщения на месте, дублей (в т.ч. от повторов) нет.""" + items, cursor = [], None + while True: + path = "/api/v1/chats/%s/messages?limit=100" % self.chat["id"] + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=self.bob["_token"], timeout=10).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + by_id = {m["id"]: m for m in items} + missing = [m for m in self.confirmed if by_id.get(m["id"]) != m] + return { + "confirmed_writes": len(self.confirmed), + "stored_messages": len(items), + "missing_or_altered": len(missing), + "duplicate_ids": len(ids) - len(set(ids)), + "duplicate_texts": len(texts) - len(set(texts)), + "same_key_retries": self.same_key_retries, + "ok": not missing and len(ids) == len(set(ids)) and len(texts) == len(set(texts)), + } diff --git a/student-tests/race.py b/student-tests/race.py new file mode 100644 index 00000000..655b54aa --- /dev/null +++ b/student-tests/race.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Конкурентная запись: 10 клиентов x 10 сообщений (перенесено из lab1, Bearer).""" +import argparse +import base64 +import json +import threading +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import urlencode + + +def request(base_url, method, path, token=None, body=None, headers=None): + headers = dict(headers or {}) + headers["Accept"] = "application/json" + + if token: + headers["Authorization"] = "Bearer " + token + + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + base_url + path, + data=data, + headers=headers, + method=method, + ) + + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + +def create_user(base_url, name): + username = "race_%s_%s" % (name, uuid.uuid4().hex[:8]) + password = "Race-password-" + uuid.uuid4().hex + + status, user = request( + base_url, + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "Race " + name, + "password": password, + }, + ) + assert status == 201 + + credential = base64.b64encode((username + ":" + password).encode()).decode() + status, session = request( + base_url, + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert status == 201 + user["token"] = session["token"] + return user + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-url", + default="http://127.0.0.1:8080", + ) + args = parser.parse_args() + + base_url = args.base_url.rstrip("/") + + users = [ + create_user(base_url, str(i)) + for i in range(10) + ] + + owner = users[0] + + status, chat = request( + base_url, + "POST", + "/api/v1/chats", + token=owner["token"], + body={ + "title": "Concurrent race test", + "member_ids": [user["id"] for user in users], + }, + ) + + assert status == 201 + + chat_id = chat["id"] + + barrier = threading.Barrier(10) + + def worker(worker_id): + user = users[worker_id] + created = [] + + barrier.wait() + + for message_id in range(10): + text = "race-%d-%d" % (worker_id, message_id) + + status, message = request( + base_url, + "POST", + "/api/v1/chats/%s/messages" % chat_id, + token=user["token"], + body={"text": text}, + headers={"Idempotency-Key": "race-%d-%d-%s" % (worker_id, message_id, uuid.uuid4().hex[:8])}, + ) + + assert status == 201 + created.append(message) + + return created + + with ThreadPoolExecutor(max_workers=10) as executor: + results = list(executor.map(worker, range(10))) + + created = [ + message + for worker_messages in results + for message in worker_messages + ] + + assert len(created) == 100 + + created_ids = {message["id"] for message in created} + created_texts = {message["text"] for message in created} + + assert len(created_ids) == 100 + assert len(created_texts) == 100 + + messages = [] + seen_cursors = set() + cursor = None + page_limit = 13 + + while True: + path = ( + "/api/v1/chats/%s/messages?limit=%d" + % (chat_id, page_limit) + ) + + if cursor is not None: + path += "&" + urlencode({"cursor": cursor}) + + status, page = request( + base_url, + "GET", + path, + token=owner["token"], + ) + + assert status == 200 + + page_messages = page["items"] + messages.extend(page_messages) + + next_cursor = page["next_cursor"] + + if next_cursor is None: + break + + assert next_cursor not in seen_cursors, ( + "Cursor повторился: возможен бесконечный цикл пагинации" + ) + + seen_cursors.add(next_cursor) + cursor = next_cursor + + assert len(messages) == 100 + + history_ids = {message["id"] for message in messages} + history_texts = {message["text"] for message in messages} + + assert len(history_ids) == 100 + assert len(history_texts) == 100 + + assert history_ids == created_ids + assert history_texts == created_texts + + user_ids = {user["id"] for user in users} + + assert all( + message["sender_id"] in user_ids + for message in messages + ) + + print( + "PASS: concurrent write test; " + "10 clients x 10 messages = 100 messages; " + "no lost or duplicate messages" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/revocation_partition.py b/student-tests/revocation_partition.py new file mode 100644 index 00000000..1193a246 --- /dev/null +++ b/student-tests/revocation_partition.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Минимальное воспроизведение замечания ревью №1 (отзыв токена при разрыве с Redis). + +Прогрев кэша -> разрыв сети API<->Redis без рестарта Redis -> logout -> +восстановление связи -> 8 с опроса старого токена (дольше circuit breaker). +Ожидается только 401. По умолчанию — команды docker compose-стенда: + python student-tests/revocation_partition.py --base-url http://127.0.0.1:8080 +""" +import argparse, shlex, subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common +from common import actor, request + +p = argparse.ArgumentParser() +p.add_argument("--base-url", default="http://127.0.0.1:8080") +p.add_argument("--partition", default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1") +p.add_argument("--heal", default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") +args = p.parse_args() +common.BASE_URL = args.base_url.rstrip("/") +ROOT = Path(__file__).resolve().parents[1] +def run(cmd): + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True) +a = actor("repro"); t = a["_token"] +for _ in range(2): request("GET","/api/v1/users/me",token=t) +run(args.partition) +try: + print("logout during partition:", request("DELETE","/api/v1/auth/sessions/current",token=t).status) +finally: + run(args.heal) +seen=set(); end=time.monotonic()+8 +while time.monotonic() healthy/UP -> graceful stop старой, и так N раз). +Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые +реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS, +ошибки и задержки до/во время/после. + + python student-tests/rolling_update_load.py --output evidence/rolling_update.json +""" +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from failover_series import observe_instances # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=8) + p.add_argument("--output") + args = p.parse_args() + cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("rprobe") + before = observe_instances(probe["_token"]) + print("replicas before: %s" % before, flush=True) + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + t_upd = load.now() + proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200) + t_done = load.now() + print(proc.stdout.strip()[-3000:], flush=True) + time.sleep(args.post) + load.stop() + t_end = load.now() + after = observe_instances(probe["_token"]) + tail_instances = load.window(t_end - 3, t_end)["instances"] + errs = load.errors_between(t_upd, t_done) + report = { + "scenario": "rolling update of all API replicas under continuous load", + "update_command": " ".join(cmd[1:] if cmd[0] == py else cmd), + "update_exit_code": proc.returncode, + "update_seconds": round(t_done - t_upd, 2), + "replicas_before": before, + "replicas_after": after, + "all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)), + "errors_during_update": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "windows": { + "before": load.window(0, t_upd), + "during_update": load.window(t_upd, t_done), + "after": load.window(t_done, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0}, + } + report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"] + w = report["windows"] + print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f" + " | p95 during %s ms | data ok=%s" + % (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-", + w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"], + w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"])) + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_all.py b/student-tests/run_all.py new file mode 100644 index 00000000..4eb24ef6 --- /dev/null +++ b/student-tests/run_all.py @@ -0,0 +1,49 @@ +"""Запуск собственных тестов, которые идут по HTTP через балансировщик. + + python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +Входит: API/auth/конкурентность (lab2), идемпотентность через реплики (lab3), +race.py, вложения lab4 (test_attachments.py). Тест согласованности БД<->S3 +(test_attachment_consistency.py) здесь пропускается: нужен доступ к БД и S3 — +python scripts/ops.py in-network consistency. Отдельно (меняют состояние стенда или нужен доступ к БД): + python scripts/ops.py in-network storage # тесты хранилища внутри compose-сети + python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl) + python student-tests/failover_series.py # остановка каждой реплики под нагрузкой + python student-tests/rolling_update_load.py # rolling update под нагрузкой + python student-tests/dependency_failure.py # отказ Redis/PostgreSQL (останавливает контейнеры) +""" +import argparse +import os +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--base-url", default="http://127.0.0.1:8080") +parser.add_argument("--expiry-url", default=os.environ.get("EXPIRY_BASE_URL", "")) +parser.add_argument("--with-failures", action="store_true", + help="run student-tests/dependency_failure.py (stops redis/postgres)") +parser.add_argument("--failure-args", default="", + help="extra arguments for dependency_failure.py (custom stop/start commands)") +args = parser.parse_args() + +env = os.environ.copy() +env["BASE_URL"] = args.base_url.rstrip("/") +if args.expiry_url: + env["EXPIRY_BASE_URL"] = args.expiry_url.rstrip("/") + + +def step(title, cmd): + print("\n== " + title, flush=True) + subprocess.run(cmd, env=env, check=True) + + +step("API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env)", + [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_*.py", "-v"]) +step("Concurrent write test", [sys.executable, "student-tests/race.py", "--base-url", args.base_url]) +if args.with_failures: + import shlex + step("Dependency failure scenario", + [sys.executable, "student-tests/dependency_failure.py", "--base-url", args.base_url] + + shlex.split(args.failure_args)) + +print("\nALL STUDENT TESTS PASSED", flush=True) diff --git a/student-tests/s3_outage.py b/student-tests/s3_outage.py new file mode 100644 index 00000000..c32964c3 --- /dev/null +++ b/student-tests/s3_outage.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Недоступность S3: деградация только операций с файлами, восстановление без рестарта API. + + python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json + +По умолчанию останавливает/запускает сервис s3 через docker compose и ждёт +прогрева чтения/записи (python scripts/ops.py wait-s3). Для другого стенда — +--stop-cmd / --start-cmd. Тома не удаляются. + +Ожидаемое поведение (S3 НЕ входит в readiness, см. REPORT.md): + * /health/ready = 200, обе реплики остаются в LB; + * регистрация, вход, сообщения и история — работают; + * GET метаданных вложения — 200 (это PostgreSQL); + * POST вложения и GET /content -> 503 unavailable (+ Retry-After) за ограниченное + время (таймауты клиента S3); pending-строку удаляет компенсация + (проверяется в test_attachment_consistency.py через api-fault-s3); + * после возврата S3 тот же токен загружает и скачивает файлы, старый файл цел. +""" +import argparse +import hashlib +import http.client +import json +import os +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import urlparse + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 +from common import actor, create_chat, request # noqa: E402 + +ROOT = HERE.parent +MAX_FAIL_SECONDS = 15.0 # connect 2 с x 2 попытки + backoff; ответ клиенту не позже + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, ok, detail=""): + self.results.append({"check": name, "ok": bool(ok), "detail": detail}) + print(("PASS " if ok else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + + +def run(cmd): + print("$ " + " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=180) + + +def upload(token, chat_id, data): + return request("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat_id, token=token, raw=data, + headers={"Content-Type": "image/png"}, timeout=40) + + +def download(token, att_id): + u = urlparse(common.BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=40) + t = time.monotonic() + conn.request("GET", "/api/v1/attachments/%s/content" % att_id, headers={"Authorization": "Bearer " + token}) + r = conn.getresponse() + body = r.read() + conn.close() + return r.status, body, time.monotonic() - t, r.headers + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + py = sys.executable or "python3" + p.add_argument("--stop-cmd", default="docker compose --env-file .env stop s3") + p.add_argument("--start-cmd", default="docker compose --env-file .env start s3") + p.add_argument("--wait-cmd", default="%s scripts/ops.py wait-s3" % shlex.quote(py)) + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + check = Check() + + alice, bob = actor("s3o_a"), actor("s3o_b") + chat = create_chat(alice, bob) + original = imagegen.sample_png() + r = upload(alice["_token"], chat["id"], original) + check("baseline upload 201", r.status == 201, str(r.status)) + att = r.json + + run(shlex.split(args.stop_cmd)) + try: + ready = request("GET", "/health/ready") + check("readiness stays 200 while S3 is down (S3 is not a readiness dependency)", ready.status == 200, + str(ready.status)) + m = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "S3 лежит, а чат работает"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("text message 201 during S3 outage", m.status == 201, str(m.status)) + h = request("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], token=bob["_token"]) + check("history 200 during S3 outage", h.status == 200, str(h.status)) + meta = request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]) + check("attachment metadata 200 (PostgreSQL only)", meta.status == 200, str(meta.status)) + ref = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("message referencing existing attachment 201", ref.status == 201, str(ref.status)) + + for i in range(2): + u = upload(alice["_token"], chat["id"], imagegen.png(10 + i, 10)) + code = (u.json or {}).get("error", {}).get("code") + check("upload #%d -> 503 unavailable" % (i + 1), u.status == 503 and code == "unavailable", + "%s %s in %.2fs" % (u.status, code, u.elapsed)) + check("upload #%d failed fast (<= %.0fs)" % (i + 1, MAX_FAIL_SECONDS), u.elapsed <= MAX_FAIL_SECONDS, + "%.2fs" % u.elapsed) + check("upload #%d has Retry-After" % (i + 1), bool(u.headers.get("Retry-After"))) + status, _, elapsed, _ = download(bob["_token"], att["id"]) + check("download -> 503 during outage", status == 503, "%s in %.2fs" % (status, elapsed)) + check("download failed fast", elapsed <= MAX_FAIL_SECONDS, "%.2fs" % elapsed) + me = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("session still valid", me.status == 200, str(me.status)) + finally: + run(shlex.split(args.start_cmd)) + t0 = time.monotonic() + run(shlex.split(args.wait_cmd)) + recovered = round(time.monotonic() - t0, 2) + + check("S3 read/write warm-up after start", True, "%.2fs" % recovered) + status, body, elapsed, headers = download(bob["_token"], att["id"]) + check("old file downloads byte-for-byte after recovery", + status == 200 and hashlib.sha256(body).hexdigest() == att["sha256"], "%s %.2fs" % (status, elapsed)) + u = upload(alice["_token"], chat["id"], imagegen.png(12, 12)) + check("new upload 201 after recovery (same token, no API restart)", u.status == 201, str(u.status)) + + report = {"scenario": "S3 outage and recovery", "max_fail_seconds": MAX_FAIL_SECONDS, + "s3_warmup_seconds": recovered, "checks": check.results, + "pass": all(c["ok"] for c in check.results)} + if args.output: + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS" if report["pass"] else "FAIL") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..59439012 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,138 @@ +"""Поведенческие тесты API, перенесённые из lab1 и переведённые на Bearer.""" +import base64 +import json +import unittest +import uuid +from urllib.parse import urlencode + +from common import actor, create_chat, request + + +class ApiBehaviorTests(unittest.TestCase): + + def test_malformed_json_returns_400(self): + r = request("POST", "/api/v1/users", raw=b'{"username": ') + self.assertEqual(r.status, 400, r.json) + self.assertIn("request_id", r.json["error"]) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def test_registration_without_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "nopass_" + uuid.uuid4().hex[:10], + "display_name": "No password", + }) + self.assertEqual(r.status, 400, r.json) + + def test_short_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "short_" + uuid.uuid4().hex[:10], + "display_name": "Short", + "password": "elevenchars", + }) + self.assertEqual(r.status, 400, r.json) + + def test_password_never_returned(self): + r = request("POST", "/api/v1/users", payload={ + "username": "secret_" + uuid.uuid4().hex[:10], + "display_name": "Secret", + "password": "Very-secret-password-42", + }) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"id", "username", "display_name", "created_at"}) + + def test_duplicate_username_returns_409(self): + payload = { + "username": "duplicate_" + uuid.uuid4().hex[:10], + "display_name": "First User", + "password": "Duplicate-pass-123", + } + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 201) + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 409) + + def test_empty_message_returns_400(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], + token=owner["_token"], payload={"text": " "}) + self.assertEqual(r.status, 400, r.json) + + def test_sender_id_in_body_rejected(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=owner["_token"], + payload={"text": "hi", "sender_id": owner["id"]}) + self.assertEqual(r.status, 400, r.json) + + def test_outsider_cannot_access_chat(self): + owner, member, outsider = actor(), actor(), actor() + chat = create_chat(owner, member) + chat_path = "/api/v1/chats/%s" % chat["id"] + self.assertEqual(request("GET", chat_path, token=outsider["_token"]).status, 404) + self.assertEqual(request("GET", chat_path + "/messages", token=outsider["_token"]).status, 404) + r = request("POST", chat_path + "/messages", token=outsider["_token"], + payload={"text": "Unauthorized message"}) + self.assertEqual(r.status, 404) + self.assertEqual(request("GET", chat_path, token=member["_token"]).status, 200) + + def test_naive_timestamp_cursor_returns_400(self): + owner = actor() + chat = create_chat(owner) + payload = { + "resource_id": chat["id"], + "created_at": "2026-10-02T12:00:00", + "item_id": str(uuid.uuid4()), + } + encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat["id"], urlencode({"cursor": encoded})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_cursor_from_other_chat_returns_400(self): + owner = actor() + chat_a, chat_b = create_chat(owner), create_chat(owner) + for i in range(3): + request("POST", "/api/v1/chats/%s/messages" % chat_a["id"], token=owner["_token"], + payload={"text": "m%d" % i}) + page = request("GET", "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], token=owner["_token"]).json + self.assertTrue(page["next_cursor"]) + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat_b["id"], urlencode({"cursor": page["next_cursor"]})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_pagination_returns_all_messages_without_duplicates(self): + owner = actor() + chat = create_chat(owner) + path = "/api/v1/chats/%s/messages" % chat["id"] + expected = [] + for index in range(8): + r = request("POST", path, token=owner["_token"], payload={"text": "page-%d" % index}) + self.assertEqual(r.status, 201, r.json) + expected.append(r.json["id"]) + received, cursor = [], None + while True: + query = {"limit": 3} + if cursor: + query["cursor"] = cursor + page = request("GET", path + "?" + urlencode(query), token=owner["_token"]).json + received.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(received, expected) + + def test_chat_list_pagination(self): + owner = actor() + created = [create_chat(owner)["id"] for _ in range(5)] + got, cursor = [], None + while True: + q = "?limit=2" + ("&" + urlencode({"cursor": cursor}) if cursor else "") + page = request("GET", "/api/v1/chats" + q, token=owner["_token"]).json + got.extend(c["id"] for c in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(got, created) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_attachment_consistency.py b/student-tests/test_attachment_consistency.py new file mode 100644 index 00000000..682ae559 --- /dev/null +++ b/student-tests/test_attachment_consistency.py @@ -0,0 +1,306 @@ +"""Согласованность PostgreSQL <-> S3 для вложений (нужен прямой доступ к БД и S3). + +Запуск внутри compose-сети (БД и S3 наружу не публикуются): + python scripts/ops.py in-network consistency +Без DATABASE_URL(_FILE) и S3-настроек тесты пропускаются. + +Что доказывается: + * сбой между S3 PUT и COMMIT (pending + объект) обнаруживается dry-run'ом, + исправляется --apply, повторный запуск ничего не меняет (идемпотентность); + * свежие pending (моложе grace) и ready/используемые вложения не удаляются; + * объект-сирота (нет строки) удаляется только после grace; + * ready без объекта — только отчёт (missing_objects), без авто-удаления; + * обрыв загрузки клиентом не оставляет ни строки, ни объекта; + * внедрённые отказы в API (если заданы FAULT_*_URL — экземпляры api-fault-*): + ошибка S3 PUT -> 503 и компенсация; ошибка БД после PUT -> 503, объект удалён; + падение процесса после PUT -> остаётся pending+объект, их убирает сборщик. +""" +import hashlib +import json +import os +import socket +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +sys.path.insert(0, str(HERE)) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +HAS_S3 = bool(os.environ.get("S3_ENDPOINT")) +GRACE = 900 + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class AttachmentConsistencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool + from src.objectstore import ObjectStore + from src.security import hash_password + from src.storage import Storage + from src import reconcile + + cls.settings = load_settings() + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.store = ObjectStore(cls.settings) + cls.rec = reconcile + user = cls.storage.create_user("cons_" + uuid.uuid4().hex[:16], "Consistency", hash_password("Consistency-pw-1")) + cls.user = user + cls.chat = cls.storage.create_chat("consistency", [user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + # helpers + + def pending_with_object(self, age_seconds=0, put=True): + att_id = uuid.uuid4() + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + data = b"\x89PNG fake bytes " + uuid.uuid4().bytes + ok = self.storage.create_pending_attachment(att_id, self.chat.id, self.user.id, "p.png", "image/png", + len(data), hashlib.sha256(data).hexdigest(), 1, 1, key) + self.assertTrue(ok) + if put: + self.store.put(key, data, "image/png", hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest()) + if age_seconds: + self.backdate(att_id, age_seconds) + return att_id, key + + def backdate(self, att_id, seconds): + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - make_interval(secs => %s) WHERE id = %s", + (seconds, att_id)) + + def row(self, att_id): + with self.pool.connection() as conn: + return conn.execute("SELECT state FROM attachments WHERE id = %s", (att_id,)).fetchone() + + def run_gc(self, apply, grace=GRACE, sweep=True): + report = None + for _ in range(50): # фоновый GC реплики мог держать lock + report = self.rec.run_once(self.pool, self.store, grace, apply, orphan_sweep=sweep) + if report is not None: + return report + time.sleep(0.2) + self.fail("reconciler lock is busy") + + # tests + + def test_crash_between_put_and_commit_dry_run_then_apply_then_noop(self): + att_id, key = self.pending_with_object(age_seconds=GRACE + 60) + dry = self.run_gc(apply=False, sweep=False) + self.assertIn(str(att_id), dry["stale_pending_ids"]) + self.assertIsNotNone(self.row(att_id), "dry-run must not change anything") + self.assertIsNotNone(self.store.head(key)) + applied = self.run_gc(apply=True, sweep=False) + self.assertGreaterEqual(applied["stale_pending_rows_deleted"], 1) + self.assertIsNone(self.row(att_id)) + self.assertIsNone(self.store.head(key)) + again = self.run_gc(apply=True, sweep=False) + self.assertNotIn(str(att_id), again["stale_pending_ids"]) + + def test_fresh_pending_is_kept(self): + att_id, key = self.pending_with_object(age_seconds=0) + self.run_gc(apply=True) + self.assertEqual(self.row(att_id), ("pending",)) + self.assertIsNotNone(self.store.head(key)) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + def test_ready_and_referenced_are_never_collected(self): + att_id, key = self.pending_with_object() + self.assertIsNotNone(self.storage.mark_attachment_ready(att_id)) + msg, _ = self.storage.create_message(self.chat.id, self.user.id, "", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.assertEqual(msg.attachment_ids, [att_id]) + self.backdate(att_id, 10 * GRACE) + self.run_gc(apply=True, grace=GRACE) + self.assertEqual(self.row(att_id), ("ready",)) + self.assertIsNotNone(self.store.head(key)) + # Даже прямое удаление строки запрещено БД, пока на неё ссылается сообщение. + from psycopg import errors + with self.assertRaises(errors.ForeignKeyViolation): + with self.pool.connection() as conn: + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_orphan_object_deleted_only_after_grace(self): + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + self.store.put(key, b"orphan", "image/png", hashlib.md5(b"orphan").digest(), "0" * 64) + report = self.run_gc(apply=True, grace=GRACE) + self.assertGreaterEqual(report["orphan_objects_within_grace"], 1) + self.assertIsNotNone(self.store.head(key), "young orphan must survive (grace period)") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, False, report, verify_ready=False) + self.assertIn(key, report["orphan_objects_sample"]) + self.assertIsNotNone(self.store.head(key), "dry-run must not delete") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertIsNone(self.store.head(key)) + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertNotIn(key, report["orphan_objects_sample"]) + + def test_missing_object_is_reported_not_deleted(self): + att_id, key = self.pending_with_object() + self.storage.mark_attachment_ready(att_id) + self.store.delete(key) # внедрённая потеря байтов + report = self.run_gc(apply=True) + self.assertIn(str(att_id), report["missing_object_ids"]) + self.assertEqual(self.row(att_id), ("ready",)) + with self.pool.connection() as conn: # убрать искусственную запись, на неё нет ссылок + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_pending_cannot_be_attached(self): + att_id, key = self.pending_with_object() + from src.storage import AttachmentNotFound + with self.assertRaises(AttachmentNotFound): + self.storage.create_message(self.chat.id, self.user.id, "x", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + +# ------------------------------------------------------------------ HTTP + DB + +def _http(base, method, path, token=None, body=None, headers=None, timeout=20): + h = dict(headers or {}) + if token: + h["Authorization"] = "Bearer " + token + data = body + if isinstance(body, dict): + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = Request(base.rstrip("/") + path, data=data, headers=h, method=method) + try: + with urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def _actor(base): + import base64 + name = "flt_" + uuid.uuid4().hex[:16] + pw = "Fault-" + uuid.uuid4().hex + s, user = _http(base, "POST", "/api/v1/users", body={"username": name, "display_name": "F", "password": pw}) + assert s == 201, (s, user) + cred = base64.b64encode((name + ":" + pw).encode()).decode() + s, sess = _http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, (s, sess) + s, chat = _http(base, "POST", "/api/v1/chats", sess["token"], {"title": "faults", "member_ids": [user["id"]]}) + assert s == 201, (s, chat) + return user, sess["token"], chat + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class UploadFailureTests(unittest.TestCase): + """Отказы на пути загрузки, наблюдаемые со стороны БД и бакета.""" + + @classmethod + def setUpClass(cls): + AttachmentConsistencyTests.setUpClass.__func__(cls) + import imagegen + cls.png = imagegen.sample_png() + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def rows_of(self, owner_id): + with self.pool.connection() as conn: + return conn.execute("SELECT id, state, object_key FROM attachments WHERE owner_id = %s", + (owner_id,)).fetchall() + + def object_count(self): + total, token = 0, None + while True: + items, token = self.store.list_page("originals/", token) + total += len(items) + if not token: + return total + + def upload(self, base, token, chat): + return _http(base, "POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], token, + self.png, {"Content-Type": "image/png"}) + + @unittest.skipUnless(os.environ.get("BASE_URL"), "BASE_URL is not set") + def test_aborted_upload_leaves_no_row_and_no_object(self): + base = os.environ["BASE_URL"] + user, token, chat = _actor(base) + before = self.object_count() + u = urlparse(base) + sock = socket.create_connection((u.hostname, u.port or 80), timeout=5) + head = ("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: %s\r\n" + "Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n" + % (chat["id"], u.hostname, token, 5 * 1024 * 1024)).encode() + sock.sendall(head + self.png + b"\x00" * 200000) # ~0.2 MiB из заявленных 5 MiB + time.sleep(0.5) + sock.close() # обрыв + time.sleep(2) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before) + s, att = self.upload(base, token, chat) # сервис продолжает работать + self.assertEqual(s, 201, att) + + @unittest.skipUnless(os.environ.get("FAULT_S3_URL"), "FAULT_S3_URL (api-fault-s3) is not set") + def test_s3_put_failure_returns_503_and_compensates(self): + base = os.environ["FAULT_S3_URL"] + user, token, chat = _actor(base) + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), [], "pending row must be removed by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_DB_URL"), "FAULT_DB_URL (api-fault-db) is not set") + def test_db_failure_after_put_returns_503_and_deletes_object(self): + base = os.environ["FAULT_DB_URL"] + user, token, chat = _actor(base) + before = self.object_count() + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before, "object must be deleted by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_CRASH_URL"), "FAULT_CRASH_URL (api-fault-crash) is not set") + def test_process_crash_after_put_is_collected_by_gc(self): + base = os.environ["FAULT_CRASH_URL"] + user, token, chat = _actor(base) + try: + self.upload(base, token, chat) + self.fail("instance was expected to crash") + except (URLError, OSError, ConnectionError): + pass # процесс умер между S3 PUT и COMMIT + rows = self.rows_of(user["id"]) + self.assertEqual(len(rows), 1) + att_id, state, key = rows[0] + self.assertEqual(state, "pending") + self.assertIsNotNone(self.store.head(key), "S3 PUT happened before the crash") + # Через grace сборщик убирает и строку, и объект (ускоряем, сдвигая created_at). + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = %s", + (att_id,)) + from src import reconcile + report = None + for _ in range(50): + report = reconcile.run_once(self.pool, self.store, GRACE, True, orphan_sweep=False) + if report: + break + time.sleep(0.2) + self.assertIn(str(att_id), report["stale_pending_ids"]) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertIsNone(self.store.head(key)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..d9bbf989 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,299 @@ +"""Вложения lab4 через балансировщик (HTTP, только стандартная библиотека). + + BASE_URL=http://127.0.0.1:8080 python -m unittest discover -s student-tests -p test_attachments.py -v + +Покрывает уровень 4: слишком большой файл (с Content-Length и chunked без него), +повреждённые изображения, подмену MIME, pixel bomb, опасные filename, чужое +attachment_id, привязку файла другого чата, неизвестный вариант. Во всех +негативных случаях дополнительно проверяется: ответ < 500, общий формат ошибки. +""" +import hashlib +import http.client +import sys +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote, urlparse + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +import imagegen as img # noqa: E402 +from common import BASE_URL, actor, create_chat, reachable, request # noqa: E402 + +MAX = 10 * 1024 * 1024 + + +def upload(token, chat_id, data, content_type="image/png", filename="photo.png", headers=None): + path = "/api/v1/chats/%s/attachments" % chat_id + if filename is not None: + path += "?filename=" + quote(filename, safe="") + h = {"Content-Type": content_type} + h.update(headers or {}) + return request("POST", path, token=token, raw=data, headers=h, timeout=60) + + +def message(token, chat_id, text, attachment_ids=None, key=None): + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, + headers={"Idempotency-Key": key or uuid.uuid4().hex}) + + +def raw_get(path, token): + """GET, возвращающий байты (common.request декодирует только JSON).""" + u = urlparse(BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=30) + conn.request("GET", path, headers={"Authorization": "Bearer " + token}) + resp = conn.getresponse() + data = resp.read() + conn.close() + return resp.status, resp.headers, data + + +def chunked_upload(token, chat_id, total, piece=256 * 1024): + """Transfer-Encoding: chunked без Content-Length: лимит должен сработать по факту чтения.""" + u = urlparse(BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=60) + path = "/api/v1/chats/%s/attachments?filename=big.png" % chat_id + conn.putrequest("POST", path) + conn.putheader("Authorization", "Bearer " + token) + conn.putheader("Content-Type", "image/png") + conn.putheader("Transfer-Encoding", "chunked") + conn.endheaders() + first = img.PNG_SIG + b"\x00" * (piece - 8) + sent = 0 + try: + while sent < total: + part = first if sent == 0 else b"\x00" * min(piece, total - sent) + conn.send(b"%x\r\n" % len(part) + part + b"\r\n") + sent += len(part) + conn.send(b"0\r\n\r\n") + except (BrokenPipeError, ConnectionResetError): + pass # сервер мог закрыть приём раньше; ответ всё равно читаем + resp = conn.getresponse() + status = resp.status + resp.read() + conn.close() + return status + + +@unittest.skipUnless(reachable(BASE_URL), "API is not reachable at %s" % BASE_URL) +class AttachmentTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("att_a") + cls.bob = actor("att_b") + cls.eve = actor("att_e") + cls.chat = create_chat(cls.alice, cls.bob) + cls.eve_chat = create_chat(cls.eve) + cls.other_chat = create_chat(cls.alice) # чат Алисы без Боба + + def assertError(self, r, status, code=None): + self.assertEqual(r.status, status, (r.status, r.json)) + self.assertLess(r.status, 500) + self.assertIsInstance(r.json, dict) + self.assertIn("error", r.json) + if code: + self.assertEqual(r.json["error"]["code"], code) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def upload_ok(self, data, content_type="image/png", filename="photo.png", who=None, chat=None): + who = who or self.alice + chat = chat or self.chat + r = upload(who["_token"], chat["id"], data, content_type, filename) + self.assertEqual(r.status, 201, (r.status, r.json)) + att = r.json + self.assertEqual(att["status"], "ready") + self.assertEqual(att["variants"], []) + self.assertIsNone(att["error_code"]) + self.assertEqual(att["size_bytes"], len(data)) + self.assertEqual(att["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(att["content_type"], content_type) + self.assertEqual(att["chat_id"], chat["id"]) + self.assertEqual(att["owner_id"], who["id"]) + self.assertNotIn("object_key", att) + return att + + # ------------------------------------------------------------- happy path + + def test_png_roundtrip_member_reads_outsider_404(self): + data = img.sample_png() + att = self.upload_ok(data, filename="Отпуск 2026 (1).png") + self.assertEqual(att["filename"], "Отпуск 2026 (1).png") + meta = request("GET", "/api/v1/attachments/" + att["id"], token=self.bob["_token"]) + self.assertEqual(meta.status, 200) + self.assertEqual(meta.json, att) + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(body, data) + self.assertEqual(headers.get_content_type(), "image/png") + self.assertEqual(int(headers["Content-Length"]), len(data)) + self.assertEqual(headers.get("X-Content-Type-Options"), "nosniff") + self.assertIn("no-store", headers.get("Cache-Control", "")) + for suffix in ("", "/content"): + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix, token=self.eve["_token"]), + 404, "not_found") + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix), 401, "unauthorized") + + def test_jpeg_baseline_and_progressive(self): + for data in (img.sample_jpeg(), img.progressive_jpeg()): + att = self.upload_ok(data, "image/jpeg", "photo.jpg") + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.alice["_token"]) + self.assertEqual((status, body, headers.get_content_type()), (200, data, "image/jpeg")) + + def test_content_type_with_parameters_is_accepted(self): + self.upload_ok(img.png(8, 8), "image/png", "x.png") + r = upload(self.alice["_token"], self.chat["id"], img.png(8, 8), "image/png; charset=binary", "x.png") + self.assertEqual(r.status, 201, r.json) + + def test_exactly_max_size_is_accepted(self): + data = img.png_of_exact_size(MAX) + att = self.upload_ok(data, filename="max.png") + status, _, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(hashlib.sha256(body).hexdigest(), att["sha256"]) + + def test_original_variant_and_unknown_variants(self): + att = self.upload_ok(img.png(4, 4)) + base = "/api/v1/attachments/%s/content" % att["id"] + status, _, _ = raw_get(base + "?variant=original", self.bob["_token"]) + self.assertEqual(status, 200) + self.assertError(request("GET", base + "?variant=thumbnail", token=self.bob["_token"]), 404, "not_found") + self.assertError(request("GET", base + "?variant=../../etc", token=self.bob["_token"]), 400, "invalid_request") + + # ------------------------------------------------------------- limits + + def test_too_large_with_content_length(self): + r = upload(self.alice["_token"], self.chat["id"], img.png_of_exact_size(MAX + 1), filename="big.png") + self.assertError(r, 413, "payload_too_large") + + def test_too_large_chunked_without_content_length(self): + status = chunked_upload(self.alice["_token"], self.chat["id"], MAX + 512 * 1024) + self.assertEqual(status, 413) + + def test_too_many_pixels(self): + data = img.huge_dimensions_png(8000, 6000) # 48 Мп при размере файла ~50 КБ + self.assertLess(len(data), MAX) + self.assertError(upload(self.alice["_token"], self.chat["id"], data), 413, "payload_too_large") + + # ------------------------------------------------------------- content checks + + def test_corrupted_images_are_415(self): + cases = { + "truncated png": img.truncated_png(), + "bad crc png": img.bad_crc_png(), + "png header + random": img.PNG_SIG + uuid.uuid4().bytes * 64, + "png with wrong data length": img.wrong_length_png(), + "truncated jpeg": img.truncated_jpeg(), + "empty body": b"", + "plain text": "не картинка".encode(), + "gif": b"GIF89a" + b"\x00" * 64, + } + for name, data in cases.items(): + ctype = "image/jpeg" if "jpeg" in name else "image/png" + with self.subTest(name): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_mime_spoofing_is_415(self): + cases = [ + (img.sample_png(), "image/jpeg"), + (img.sample_jpeg(), "image/png"), + (img.sample_png(), "image/gif"), + (img.sample_png(), "application/octet-stream"), + (img.sample_png(), "multipart/form-data; boundary=x"), + (img.sample_png(), "text/html"), + ] + for data, ctype in cases: + with self.subTest(ctype): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_dangerous_filenames_are_400(self): + for name in ("../etc/passwd", "a/b.png", "a\\b.png", "nul\x00.png", "bell\x07.png", "new\nline.png", + "", " ", ".", "..", "x" * 256, "rtl‮gnp.exe"): + with self.subTest(repr(name)): + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=name), + 400, "invalid_request") + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=None), + 400, "invalid_request") + self.upload_ok(img.png(2, 2), filename="x" * 255) + + # ------------------------------------------------------------- access + + def test_outsider_and_unknown_chat_upload_404(self): + self.assertError(upload(self.eve["_token"], self.chat["id"], img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], str(uuid.uuid4()), img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], "not-a-uuid", img.png(2, 2)), 400, "invalid_request") + self.assertError(upload(None, self.chat["id"], img.png(2, 2)), 401, "unauthorized") + + def test_invalid_and_unknown_attachment_ids(self): + self.assertError(request("GET", "/api/v1/attachments/not-a-uuid", token=self.bob["_token"]), 400) + self.assertError(request("GET", "/api/v1/attachments/%s" % uuid.uuid4(), token=self.bob["_token"]), 404) + self.assertError(request("GET", "/api/v1/attachments/%s/content" % uuid.uuid4(), token=self.bob["_token"]), 404) + + # ------------------------------------------------------------- messages + + def test_message_with_attachments_visible_to_members(self): + a1 = self.upload_ok(img.png(3, 3)) + a2 = self.upload_ok(img.sample_jpeg(), "image/jpeg", "b.jpg") + r = message(self.alice["_token"], self.chat["id"], "", [a2["id"], a1["id"]]) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(r.json["text"], "") + self.assertEqual(r.json["attachment_ids"], [a2["id"], a1["id"]]) # порядок сохранён + # Любой участник может сослаться на вложение чата повторно. + r2 = message(self.bob["_token"], self.chat["id"], "и снова", [a1["id"]]) + self.assertEqual(r2.status, 201, r2.json) + page = request("GET", "/api/v1/chats/%s/messages?limit=100" % self.chat["id"], token=self.bob["_token"]) + by_id = {m["id"]: m for m in page.json["items"]} + self.assertEqual(by_id[r.json["id"]]["attachment_ids"], [a2["id"], a1["id"]]) + self.assertEqual(by_id[r2.json["id"]]["attachment_ids"], [a1["id"]]) + text_only = message(self.alice["_token"], self.chat["id"], "без вложений") + self.assertEqual(text_only.json["attachment_ids"], []) + + def test_attachment_from_other_chat_is_404_and_nothing_saved(self): + foreign = self.upload_ok(img.png(2, 2), chat=self.other_chat) # чат Алисы без Боба + key = uuid.uuid4().hex + r = message(self.alice["_token"], self.chat["id"], "чужой файл", [foreign["id"]], key=key) + self.assertError(r, 404, "not_found") + # Отказ не «сжёг» ключ: тот же ключ с корректным телом создаёт сообщение. + ok = message(self.alice["_token"], self.chat["id"], "чужой файл", [], key=key) + self.assertEqual(ok.status, 201, ok.json) + # Посторонняя Ева не может прикрепить файл Алисы в своём чате. + a = self.upload_ok(img.png(2, 2)) + self.assertError(message(self.eve["_token"], self.eve_chat["id"], "x", [a["id"]]), 404, "not_found") + self.assertError(message(self.alice["_token"], self.chat["id"], "x", [str(uuid.uuid4())]), 404) + + def test_message_attachment_validation(self): + a = self.upload_ok(img.png(2, 2)) + cid, tok = self.chat["id"], self.alice["_token"] + self.assertError(message(tok, cid, ""), 400) + self.assertError(message(tok, cid, " "), 400) + self.assertError(message(tok, cid, "dup", [a["id"], a["id"]]), 400) + self.assertError(message(tok, cid, "many", [str(uuid.uuid4()) for _ in range(11)]), 400) + self.assertError(message(tok, cid, "bad", ["not-a-uuid"]), 400) + self.assertError(message(tok, cid, "bad", "not-a-list"), 400) + r = message(tok, cid, " ", [a["id"]]) # пробельный текст допустим с вложением + self.assertEqual(r.status, 201, r.json) + + def test_idempotency_includes_attachment_ids(self): + a1 = self.upload_ok(img.png(2, 2)) + a2 = self.upload_ok(img.png(3, 3)) + key = uuid.uuid4().hex + first = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual(first.status, 201) + replay = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a2["id"], a1["id"]], key=key), 409) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a1["id"]], key=key), 409) + # Отсутствующее поле == [] для идемпотентности. + k2 = uuid.uuid4().hex + m1 = message(self.alice["_token"], self.chat["id"], "без поля", None, key=k2) + m2 = message(self.alice["_token"], self.chat["id"], "без поля", [], key=k2) + self.assertEqual((m2.status, m2.json), (201, m1.json)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..2a7540a3 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,109 @@ +"""Аутентификация: Basic только для входа, Bearer для остального, logout.""" +import base64 +import unittest +import uuid + +from common import actor, basic, login, register, request + + +def assert_challenge(test, r, scheme): + test.assertEqual(r.status, 401, r.json) + test.assertTrue(r.headers.get("WWW-Authenticate", "").startswith(scheme), r.headers.get("WWW-Authenticate")) + test.assertEqual(r.json["error"]["code"], "unauthorized") + + +class AuthTests(unittest.TestCase): + + def test_wrong_password_and_unknown_user_look_the_same(self): + user = register("wp") + wrong = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], "Wrong-password-000")}) + unknown = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic("nouser_" + uuid.uuid4().hex[:10], "Wrong-password-000")}) + for r in (wrong, unknown): + assert_challenge(self, r, 'Basic realm="messenger"') + self.assertIn('charset="UTF-8"', r.headers.get("WWW-Authenticate")) + self.assertEqual(wrong.json["error"]["message"], unknown.json["error"]["message"]) + + def test_malformed_basic_headers(self): + for header in (None, "Basic", "Basic !!!notbase64", "Bearer abc", + "Basic " + base64.b64encode(b"no-colon-here").decode()): + headers = {"Authorization": header} if header else {} + r = request("POST", "/api/v1/auth/sessions", headers=headers) + assert_challenge(self, r, "Basic") + + def test_session_shape_and_token_entropy(self): + user = register("shape") + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"token", "token_type", "expires_at", "user"}) + self.assertEqual(r.json["token_type"], "Bearer") + token = r.json["token"] + # 32 случайных байта в base64url = 43 символа (256 бит) + self.assertGreaterEqual(len(token), 43) + self.assertNotIn(user["username"], token) + self.assertNotIn(user["id"], token) + self.assertTrue(r.json["expires_at"].endswith("Z")) + self.assertNotEqual(token, login(user), "each login must issue a new token") + + def test_password_with_colon_and_unicode(self): + password = "пароль:с:двоеточием-🙂" + user = register("colon", password=password) + token = login(user) + self.assertEqual(request("GET", "/api/v1/users/me", token=token).json["id"], user["id"]) + + def test_login_with_body_rejected(self): + """Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет.""" + user = register("body") + auth = {"Authorization": basic(user["username"], user["_password"])} + cases = [ + dict(payload={"unexpected": True}), + dict(payload={}), + dict(raw=b"not json at all"), + dict(raw=b"username=x&password=y", headers={"Content-Type": "application/x-www-form-urlencoded"}), + ] + for case in cases: + headers = dict(auth) + headers.update(case.pop("headers", {})) + r = request("POST", "/api/v1/auth/sessions", headers=headers, **case) + self.assertEqual(r.status, 400, (case, r.json)) + self.assertEqual(r.json["error"]["code"], "invalid_request") + self.assertNotIn("token", r.json) + # Без тела тот же заголовок работает. + self.assertEqual(request("POST", "/api/v1/auth/sessions", headers=auth).status, 201) + + def test_basic_on_business_endpoint_rejected(self): + user = register("basic") + r = request("GET", "/api/v1/users/me", + headers={"Authorization": basic(user["username"], user["_password"])}) + assert_challenge(self, r, 'Bearer realm="messenger"') + + def test_x_user_id_gives_no_access(self): + user = actor("xuid") + other = actor("xuid") + assert_challenge(self, request("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}), "Bearer") + r = request("GET", "/api/v1/users/me", token=user["_token"], headers={"X-User-Id": other["id"]}) + self.assertEqual(r.json["id"], user["id"]) + + def test_unknown_and_garbage_tokens(self): + for token in ("x" * 43, "", "a b", "z" * 1000): + r = request("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert_challenge(self, r, "Bearer") + + def test_logout_revokes_only_current_session(self): + user = register("logout") + t1, t2 = login(user), login(user) + # Прогреваем кэш Redis для t1, чтобы проверить инвалидацию кэша. + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + r = request("DELETE", "/api/v1/auth/sessions/current", token=t1) + self.assertEqual(r.status, 204) + self.assertIsNone(r.json) + assert_challenge(self, request("GET", "/api/v1/users/me", token=t1), "Bearer") + assert_challenge(self, request("DELETE", "/api/v1/auth/sessions/current", token=t1), "Bearer") + self.assertEqual(request("GET", "/api/v1/users/me", token=t2).status, 200) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..04114909 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,80 @@ +"""Гонка регистрации одинакового username и rollback составной операции.""" +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import actor, basic, request + +PARALLEL = 16 + + +class RegistrationRaceTests(unittest.TestCase): + + def test_same_username_parallel_registration(self): + username = "race_" + uuid.uuid4().hex[:12] + barrier = threading.Barrier(PARALLEL) + + def attempt(i): + password = "Race-password-%02d-%s" % (i, uuid.uuid4().hex[:6]) + barrier.wait() + r = request("POST", "/api/v1/users", timeout=30, payload={ + "username": username, "display_name": "Racer %d" % i, "password": password, + }) + return r.status, r.json, password + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(attempt, range(PARALLEL))) + + statuses = sorted(s for s, _, _ in results) + self.assertEqual(statuses.count(201), 1, statuses) + self.assertEqual(statuses.count(409), PARALLEL - 1, statuses) + for status, body, _ in results: + if status == 409: + self.assertEqual(body["error"]["code"], "conflict") + + # Победитель входит своим паролем; пароли проигравших не подходят. + winner = next(r for r in results if r[0] == 201) + loser = next(r for r in results if r[0] == 409) + ok = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, winner[2])}) + self.assertEqual(ok.status, 201) + self.assertEqual(ok.json["user"]["id"], winner[1]["id"]) + bad = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, loser[2])}) + self.assertEqual(bad.status, 401) + + +class ChatRollbackTests(unittest.TestCase): + + def _chat_ids(self, owner): + r = request("GET", "/api/v1/chats?limit=100", token=owner["_token"]) + self.assertEqual(r.status, 200) + return [c["id"] for c in r.json["items"]] + + def test_unknown_member_rolls_back_whole_chat(self): + """Чат вставляется первым, затем участники по порядку (сначала сам владелец). + + Неизвестный участник вызывает нарушение внешнего ключа на 2-й строке + chat_members. Без транзакции у владельца остался бы чат с одним + участником; с транзакцией не остаётся ничего. + """ + owner, member = actor("rb"), actor("rb") + before = self._chat_ids(owner) + r = request("POST", "/api/v1/chats", token=owner["_token"], payload={ + "title": "Must roll back", + "member_ids": [owner["id"], member["id"], str(uuid.uuid4())], + }) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), before, "partial chat is visible after failed creation") + self.assertEqual(self._chat_ids(member), [], "partial membership is visible to other member") + + def test_duplicate_and_missing_caller_rejected(self): + owner, member = actor("rb"), actor("rb") + for ids in ([owner["id"], owner["id"]], [member["id"]]): + r = request("POST", "/api/v1/chats", token=owner["_token"], + payload={"title": "bad", "member_ids": ids}) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), []) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..c5601401 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,49 @@ +"""Истечение сессии при коротком SESSION_TTL_SECONDS. + +Нужен отдельный экземпляр API с коротким TTL (compose-профиль ttl, порт 8081): + docker compose --profile ttl up -d api-short-ttl + EXPIRY_BASE_URL=http://127.0.0.1:8081 python -m unittest discover -s student-tests -p test_expiry.py +Без EXPIRY_BASE_URL тест пропускается (основной стенд работает с TTL >= 300). +""" +import datetime as dt +import os +import time +import unittest + +from common import basic, register, request + +EXPIRY_BASE_URL = os.environ.get("EXPIRY_BASE_URL", "").rstrip("/") + + +@unittest.skipUnless(EXPIRY_BASE_URL, "set EXPIRY_BASE_URL to the short-TTL API instance") +class ExpiryTests(unittest.TestCase): + + def test_session_expires_even_when_cached(self): + user = register("exp", base_url=EXPIRY_BASE_URL) + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + token = r.json["token"] + expires_at = dt.datetime.fromisoformat(r.json["expires_at"].replace("Z", "+00:00")) + ttl = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds() + self.assertLess(ttl, 30, "EXPIRY_BASE_URL must point to a short-TTL instance") + + # Два запроса подряд: первый кладёт сессию в Redis, второй читает из кэша. + for _ in range(2): + self.assertEqual(request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL).status, 200) + + time.sleep(max(ttl, 0) + 1.5) + r = request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, r.json) + self.assertTrue(r.headers.get("WWW-Authenticate", "").startswith("Bearer")) + r = request("DELETE", "/api/v1/auth/sessions/current", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, "logout with expired token must be 401") + + # Новый вход после истечения работает. + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..0242026d --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,172 @@ +"""Idempotency-Key через балансировщик (lab3). Только стандартная библиотека. + + BASE_URL=http://127.0.0.1:8080 python -m unittest discover -s student-tests -p test_idempotency.py -v + +Главная проверка — одновременные повторы с одним ключом, которые LB +раскладывает по РАЗНЫМ репликам (это проверяется по X-Instance-Id), +создают ровно одно сообщение. Остальное — семантика из contracts/README.md. +""" +import json +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +from common import actor, create_chat, login, request + +PARALLEL = 24 + + +def history(token, chat_id): + items, cursor = [], None + for _ in range(100): + path = "/api/v1/chats/%s/messages?limit=100" % chat_id + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=token).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + raise AssertionError("pagination did not finish") + + +def post(token, chat_id, body, key, raw=None): + # None -> заголовок не отправляется (см. common.request). + headers = {"Idempotency-Key": key} + if raw is not None: + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, raw=raw, headers=headers) + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, headers=headers) + + +def parallel(n, fn): + barrier = threading.Barrier(n) + + def run(i): + barrier.wait() + return fn(i) + + with ThreadPoolExecutor(max_workers=n) as pool: + return list(pool.map(run, range(n))) + + +class IdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("ida") + cls.bob = actor("idb") + cls.eve = actor("ide") + cls.chat = create_chat(cls.alice, cls.bob) + cls.chat2 = create_chat(cls.alice, cls.bob) + + def test_concurrent_same_key_across_replicas_creates_one_message(self): + key = "par-" + uuid.uuid4().hex + text = "одновременно " + key + results = parallel(PARALLEL, lambda i: post(self.alice["_token"], self.chat["id"], {"text": text}, key)) + statuses = [r.status for r in results] + self.assertEqual(statuses, [201] * PARALLEL, statuses) + ids = {r.json["id"] for r in results} + self.assertEqual(len(ids), 1, "same key produced different messages") + bodies = {json.dumps(r.json, sort_keys=True) for r in results} + self.assertEqual(len(bodies), 1, "replays must return the identical Message") + instances = {r.headers.get("X-Instance-Id") for r in results} + self.assertGreaterEqual(len(instances), 2, "requests did not reach >= 2 replicas: %s" % instances) + stored = [m for m in history(self.bob["_token"], self.chat["id"]) if m["text"] == text] + self.assertEqual(len(stored), 1, "history must contain exactly one message") + replayed = sum(1 for r in results if r.headers.get("Idempotency-Replayed") == "true") + self.assertEqual(replayed, PARALLEL - 1) + print("\n %d parallel POSTs, one key -> 1 message; replicas: %s" % (PARALLEL, sorted(instances))) + + def test_concurrent_same_key_different_bodies_one_winner(self): + key = "race-" + uuid.uuid4().hex + results = parallel(12, lambda i: post(self.alice["_token"], self.chat["id"], {"text": "вариант %d" % i}, key)) + winners = [r for r in results if r.status == 201] + losers = [r for r in results if r.status == 409] + self.assertEqual(len(winners) + len(losers), 12, [r.status for r in results]) + self.assertGreaterEqual(len(winners), 1) + self.assertEqual(len({r.json["id"] for r in winners}), 1, "two bodies won with one key") + for r in losers: + self.assertEqual(r.json["error"]["code"], "conflict") + texts = [m["text"] for m in history(self.alice["_token"], self.chat["id"]) if m["text"].startswith("вариант ")] + self.assertEqual(len([t for t in texts if t == winners[0].json["text"]]), 1) + + def test_sequential_replay_and_conflict(self): + key = "seq-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual(first.status, 201) + again = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual((again.status, again.json), (201, first.json)) + other = post(self.alice["_token"], self.chat["id"], {"text": "два"}, key) + self.assertEqual(other.status, 409) + self.assertEqual(other.json["error"]["code"], "conflict") + + def test_json_formatting_and_key_order_do_not_matter(self): + key = "fmt-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{"text":"format"}') + spaced = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{ \n "text" : "format"\n}') + self.assertEqual(first.status, 201) + self.assertEqual((spaced.status, spaced.json), (201, first.json)) + + def test_missing_or_invalid_key_is_400(self): + token, chat_id = self.alice["_token"], self.chat["id"] + # Не-ASCII ключ передаём сырыми UTF-8 байтами (urllib пропускает только latin-1). + non_ascii = "ключ".encode("utf-8").decode("latin-1") + bad_keys = [None, "", " ", "a" * 129, non_ascii, "has space", "semi;colon", "slash/key"] + for key in bad_keys: + with self.subTest(key=key): + r = post(token, chat_id, {"text": "x"}, key) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(r.json["error"]["code"], "invalid_request") + ok = post(token, chat_id, {"text": "x"}, "A" * 128) + self.assertEqual(ok.status, 201) + ok2 = post(token, chat_id, {"text": "y"}, "a.b_c:d-1") + self.assertEqual(ok2.status, 201) + + def test_auth_checked_before_key(self): + r = post(None, self.chat["id"], {"text": "x"}, None) + self.assertEqual(r.status, 401) + + def test_scope_is_user_and_chat(self): + key = "scope-" + uuid.uuid4().hex + a1 = post(self.alice["_token"], self.chat["id"], {"text": "scope"}, key) + b1 = post(self.bob["_token"], self.chat["id"], {"text": "scope"}, key) + a2 = post(self.alice["_token"], self.chat2["id"], {"text": "scope"}, key) + self.assertEqual([a1.status, b1.status, a2.status], [201, 201, 201]) + self.assertEqual(len({a1.json["id"], b1.json["id"], a2.json["id"]}), 3) + # Другое тело другим пользователем с тем же ключом — не конфликт. + b2 = post(self.bob["_token"], self.chat2["id"], {"text": "другое"}, key) + self.assertEqual(b2.status, 201) + + def test_key_survives_new_session_of_same_user(self): + key = "sess-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "до перелогина"}, key) + token2 = login(self.alice) + replay = post(token2, self.chat["id"], {"text": "до перелогина"}, key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + + def test_failed_access_does_not_burn_key(self): + key = "burn-" + uuid.uuid4().hex + denied = post(self.eve["_token"], self.chat["id"], {"text": "чужое"}, key) + self.assertEqual(denied.status, 404) + # Тот же ключ у участника и даже у того же постороннего позже — не занят. + mine = post(self.alice["_token"], self.chat["id"], {"text": "своё"}, key) + self.assertEqual(mine.status, 201) + invalid = post(self.alice["_token"], self.chat2["id"], {"text": ""}, key) + self.assertEqual(invalid.status, 400) + valid = post(self.alice["_token"], self.chat2["id"], {"text": "после 400"}, key) + self.assertEqual(valid.status, 201) + + def test_replay_rechecks_session(self): + key = "revoked-" + uuid.uuid4().hex + token = login(self.bob) + first = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(first.status, 201) + self.assertEqual(request("DELETE", "/api/v1/auth/sessions/current", token=token).status, 204) + replay = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(replay.status, 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_session_cache.py b/student-tests/test_session_cache.py new file mode 100644 index 00000000..e097270a --- /dev/null +++ b/student-tests/test_session_cache.py @@ -0,0 +1,149 @@ +"""Отзыв сессии при сбое инвалидации кэша (регрессия замечания ревью №1). + +Сценарий ревью: прогретый кэш -> связь API с Redis разорвана -> logout (204) +-> связь восстановлена с ТЕМ ЖЕ Redis, без его рестарта, положительная запись +отозванного токена ещё жива. Ожидание: токен отклоняется сразу, а не через +SESSION_CACHE_TTL_SECONDS. + +Redis здесь заменён управляемой заглушкой с тем же интерфейсом, что и +SessionCache (get/put/revoke): «разрыв» = операции падают, но данные заглушки +сохраняются — ровно как у живого Redis за сетевым разрывом. PostgreSQL — +настоящая (нужен DATABASE_URL/DATABASE_URL_FILE, иначе тест пропускается): + make test-storage +""" +import os +import sys +import time +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class PartitionableCache: + """Кэш, который «помнит» данные во время разрыва, но недоступен для API.""" + + def __init__(self): + self.data = {} + self.partitioned = False + + def get(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return None + value = self.data.get(digest) + if value is None: + return None + if value == REVOKED: + return REVOKED + user, expires_at = value + return (user, expires_at) if expires_at > datetime.now(timezone.utc) else None + + def put(self, digest, user, expires_at): + if not self.partitioned: + self.data.setdefault(digest, (user, expires_at)) # SET NX + + def revoke(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return False + self.data[digest] = REVOKED + return True + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class SessionRevocationCacheTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + cls.settings = load_settings() + wait_and_migrate(cls.settings) + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.user = cls.storage.create_user("rvk_" + uuid.uuid4().hex[:12], "Revoke", + hash_password("Revoke-test-password")) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def service(self, cache): + from src.sessions import SessionService + return SessionService(self.storage, cache, self.settings) + + def pending(self, digest): + with self.pool.connection() as conn: + return conn.execute("SELECT count(*) FROM session_cache_invalidations WHERE token_hash = %s", + (digest,)).fetchone()[0] + + def warm(self, svc, cache, token): + from src.security import token_digest + self.assertEqual(svc.resolve(token).id, self.user.id) + self.assertIsInstance(cache.data.get(token_digest(token)), tuple, "cache was not warmed") + + def test_revoked_token_rejected_after_partition_heals(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + digest = token_digest(token) + self.warm(svc, cache, token) + + cache.partitioned = True + self.assertTrue(svc.revoke(token), "logout must succeed (PostgreSQL is the source of truth)") + self.assertIsInstance(cache.data[digest], tuple, "precondition: stale positive entry survived") + self.assertEqual(self.pending(digest), 1, "invalidation debt must be committed with revoked_at") + self.assertIsNone(svc.resolve(token), "during partition") + + cache.partitioned = False # тот же «Redis», данные не потеряны + self.assertIsNone(svc.resolve(token), "immediately after partition heals") + time.sleep(svc.RECHECK_SECONDS + 0.1) + self.assertIsNone(svc.resolve(token), "after reconciliation") + self.assertEqual(cache.data[digest], "revoked", "reconciliation must write the tombstone") + self.assertEqual(self.pending(digest), 0, "debt must be cleared after the tombstone") + + # После погашения долга кэшу снова доверяют: другой токен обслуживается из кэша. + other, _ = svc.create(self.user) + self.warm(svc, cache, other) + + def test_debt_survives_api_restart(self): + """Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу.""" + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + cache.partitioned = True + self.assertTrue(svc.revoke(token)) + del svc # «рестарт API»: состояние процесса потеряно + + cache.partitioned = False + restarted = self.service(cache) + self.assertIsNone(restarted.resolve(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + + def test_normal_logout_leaves_no_debt(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + self.assertTrue(svc.revoke(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + self.assertEqual(self.pending(token_digest(token)), 0) + self.assertIsNone(svc.resolve(token)) + self.assertFalse(svc.revoke(token), "second logout must fail") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_storage_idempotency.py b/student-tests/test_storage_idempotency.py new file mode 100644 index 00000000..b4334207 --- /dev/null +++ b/student-tests/test_storage_idempotency.py @@ -0,0 +1,144 @@ +"""Идемпотентность на уровне хранилища (нужен прямой доступ к PostgreSQL). + +* сбой ПОСЛЕ вставки ключа и сообщения внутри транзакции -> не остаётся ни ключа, + ни сообщения, и повтор с тем же ключом создаёт сообщение (5xx не фиксируется); +* 16 потоков с одним ключом -> ровно одна вставка, остальные — повтор; +* истёкший ключ можно занять заново; очистка удаляет только истёкшие строки и + выполняется одной репликой за раз (advisory lock). + +Запуск внутри compose-сети (БД наружу не публикуется): + python scripts/ops.py in-network storage +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import hashlib +import os +import sys +import threading +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +DAY = 86400 + + +class InjectedFailure(RuntimeError): + pass + + +def fp(text): + return hashlib.sha256(text.encode()).digest() + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageIdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Idempotency-test-password") + cls.user = cls.storage.create_user("idm_" + uuid.uuid4().hex[:12], "Idem", h) + cls.chat = cls.storage.create_chat("idem-" + uuid.uuid4().hex[:8], [cls.user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def keys(self, key): + return self.count("SELECT count(*) FROM idempotency_keys WHERE user_id = %s AND idem_key = %s", + (self.user.id, key)) + + def test_failure_inside_transaction_stores_nothing(self): + key = "fail-" + uuid.uuid4().hex + text = "сбой " + key + original = self.storage._insert_message + seen = {} + + def insert_then_fail(conn, message_id, chat_id, sender_id, body): + original(conn, message_id, chat_id, sender_id, body) + seen["keys"] = conn.execute("SELECT count(*) FROM idempotency_keys WHERE idem_key = %s", (key,)).fetchone()[0] + seen["messages"] = conn.execute("SELECT count(*) FROM messages WHERE id = %s", (message_id,)).fetchone()[0] + raise InjectedFailure("crash after both inserts, before COMMIT") + + self.storage._insert_message = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + finally: + self.storage._insert_message = original + self.assertEqual((seen["keys"], seen["messages"]), (1, 1), "inserts were visible inside the transaction") + self.assertEqual(self.keys(key), 0) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 0) + # Клиент повторяет неподтверждённый запрос с тем же ключом — теперь успешно. + msg, replayed = self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + self.assertFalse(replayed) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_threads_same_key_single_insert(self): + key = "thr-" + uuid.uuid4().hex + text = "потоки " + key + barrier = threading.Barrier(16) + + def call(_): + barrier.wait() + return self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + + with ThreadPoolExecutor(max_workers=16) as pool: + results = list(pool.map(call, range(16))) + self.assertEqual(sum(1 for _, replayed in results if not replayed), 1) + self.assertEqual(len({m.id for m, _ in results}), 1) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_conflict_on_different_body(self): + from src.storage import IdempotencyConflict + key = "cnf-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "a", key, fp("a"), DAY) + with self.assertRaises(IdempotencyConflict): + self.storage.create_message(self.chat.id, self.user.id, "b", key, fp("b"), DAY) + + def test_expired_key_can_be_reused_and_is_cleaned(self): + key = "exp-" + uuid.uuid4().hex + first, _ = self.storage.create_message(self.chat.id, self.user.id, "старый", key, fp("старый"), 1) + time.sleep(1.2) + # Истёк: другое тело уже не конфликт, создаётся новое сообщение. + second, replayed = self.storage.create_message(self.chat.id, self.user.id, "новый", key, fp("новый"), DAY) + self.assertFalse(replayed) + self.assertNotEqual(first.id, second.id) + self.assertEqual(self.keys(key), 1) + # Очистка удаляет только истёкшие ключи. + k2 = "exp2-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "x", k2, fp("x"), 1) + time.sleep(1.2) + while self.storage.delete_expired_idempotency_keys(): + pass + self.assertEqual(self.keys(k2), 0) + self.assertEqual(self.keys(key), 1) + + def test_cleanup_runs_on_one_replica_at_a_time(self): + from src.storage import IDEMPOTENCY_CLEANUP_LOCK_ID + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,)) + self.assertIsNone(self.storage.delete_expired_idempotency_keys()) + self.assertIsNotNone(self.storage.delete_expired_idempotency_keys()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_storage_rollback.py b/student-tests/test_storage_rollback.py new file mode 100644 index 00000000..305f0ecb --- /dev/null +++ b/student-tests/test_storage_rollback.py @@ -0,0 +1,101 @@ +"""Rollback на уровне хранилища с внедрённым сбоем (нужен прямой доступ к БД). + +Сбой вызывается ПОСЛЕ того, как строки chats и chat_members уже вставлены +внутри транзакции. Проверяем, что в БД не осталось ни чата, ни участников. + +Запуск внутри compose-сети (БД наружу не публикуется): + docker compose run --rm --no-deps -v "$PWD/student-tests:/app/student-tests:ro" \ + api python -m unittest discover -s student-tests -p test_storage_rollback.py -v +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import os +import sys +import unittest +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class InjectedFailure(RuntimeError): + pass + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageRollbackTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Rollback-test-password") + cls.users = [ + cls.storage.create_user("rbk_" + uuid.uuid4().hex[:12], "Rollback", h) + for _ in range(3) + ] + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def test_failure_after_inserts_leaves_nothing(self): + title = "rollback-" + uuid.uuid4().hex + original = self.storage._insert_members + seen = {} + + def insert_then_fail(conn, chat_id, member_ids): + original(conn, chat_id, member_ids) + # Внутри транзакции строки видны — значит вставка действительно была. + seen["chats"] = conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone()[0] + seen["members"] = conn.execute("SELECT count(*) FROM chat_members WHERE chat_id = %s", (chat_id,)).fetchone()[0] + seen["chat_id"] = chat_id + raise InjectedFailure("simulated crash between statements") + + self.storage._insert_members = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_chat(title, [u.id for u in self.users]) + finally: + self.storage._insert_members = original + + self.assertEqual(seen["chats"], 1) + self.assertEqual(seen["members"], 3) + self.assertEqual(self.count("SELECT count(*) FROM chats WHERE title = %s", (title,)), 0) + self.assertEqual(self.count("SELECT count(*) FROM chat_members WHERE chat_id = %s", (seen["chat_id"],)), 0) + + def test_no_orphan_chats_in_database(self): + orphans = self.count( + "SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)", ()) + self.assertEqual(orphans, 0) + + def test_migrations_are_idempotent(self): + from src.config import load_settings + from src.db import apply_migrations + self.assertEqual(apply_migrations(load_settings()), []) + self.assertEqual(self.count("SELECT count(*) FROM schema_migrations WHERE version = %s", ("001_init",)), 1) + + def test_passwords_stored_as_argon2id(self): + with self.pool.connection() as conn: + stored = conn.execute("SELECT password_hash FROM users WHERE id = %s", + (self.users[0].id,)).fetchone()[0] + self.assertTrue(stored.startswith("$argon2id$"), stored[:10]) + self.assertNotIn("Rollback-test-password", stored) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..cebdff41 --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Эксперимент: N одновременных загрузок ~9.5 MiB и пик памяти API-реплик. + +Гипотеза: память реплики ограничена сверху + baseline + MAX_CONCURRENT_UPLOADS x (~2 x размер файла) + разбор изображения, +и не растёт с числом одновременных клиентов: лишние ждут слот +(UPLOAD_QUEUE_TIMEOUT_SECONDS), а не читают тело в память. + +Источники памяти: + всегда VmHWM процесса API (пик RSS, считает ядро), обнулённый перед + экспериментом через /proc/PID/clear_refs — главный показатель; + --docker + docker stats (опрос ~1 с) по контейнерам сервиса api + + cgroup memory.peak (ядро >= 5.19) до и после; + --pids 123,456 + /proc//status: VmRSS каждые 20 мс (запуск без Docker). + + python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 \ + --concurrency 10 --docker --output evidence/upload_memory.json + +Изображения синтетические (шум, PNG без сжатия). Токены не сохраняются. +""" +import argparse +import hashlib +import json +import os +import platform +import re +import statistics +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 + +MiB = 1024 * 1024 +UNITS = {"b": 1, "kib": 1024, "mib": MiB, "gib": 1024 * MiB, "kb": 1000, "mb": 1000 ** 2, "gb": 1000 ** 3} + + +def parse_mem(text): + m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", text) + return float(m.group(1)) * UNITS.get(m.group(2).lower(), 1) if m else 0.0 + + +class Sampler(threading.Thread): + def __init__(self, args): + super().__init__(daemon=True) + self.args = args + self.stop = threading.Event() + self.samples = {} # name -> [bytes] + + def add(self, name, value): + self.samples.setdefault(name, []).append(value) + + def run(self): + while not self.stop.is_set(): + if self.args.pids: + for pid in self.args.pids.split(","): + try: + status = Path("/proc/%s/status" % pid).read_text() + rss = int(re.search(r"VmRSS:\s+(\d+) kB", status).group(1)) * 1024 + self.add("pid-" + pid, rss) + except (OSError, AttributeError): + pass + time.sleep(0.02) + else: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"] + docker_api_ids(self.args), + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, usage = line.split("\t", 1) + self.add(name, parse_mem(usage.split("/")[0])) + + +def docker_api_ids(args): + out = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + args.project, + "--filter", "label=com.docker.compose.service=api"], capture_output=True, text=True).stdout + return out.split() + + +def proc_hwm(args, reset=False): + """Пик RSS процесса (VmHWM, считает ядро — без пропусков между опросами). + + --docker: PID 1 контейнера api (python, exec-форма CMD); --pids: заданные PID. + reset=True обнуляет пик (echo 5 > /proc/PID/clear_refs) перед экспериментом. + """ + result = {} + if args.docker: + for cid in docker_api_ids(args): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + if reset: + subprocess.run(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"], + capture_output=True) + out = subprocess.run(["docker", "exec", cid, "cat", "/proc/1/status"], capture_output=True, + text=True).stdout + m = re.search(r"VmHWM:\s+(\d+) kB", out) + if m: + result[name] = int(m.group(1)) * 1024 + else: + for pid in args.pids.split(","): + try: + if reset: + Path("/proc/%s/clear_refs" % pid).write_text("5") + m = re.search(r"VmHWM:\s+(\d+) kB", Path("/proc/%s/status" % pid).read_text()) + result["pid-" + pid] = int(m.group(1)) * 1024 + except (OSError, AttributeError): + pass + return result + + +def cgroup_peaks(args): + """cgroup v2 memory.peak (или v1 max_usage_in_bytes) каждого контейнера api.""" + peaks = {} + for cid in docker_api_ids(args): + for path in ("/sys/fs/cgroup/memory.peak", "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"): + r = subprocess.run(["docker", "exec", cid, "cat", path], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip().isdigit(): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + peaks[name] = int(r.stdout.strip()) + break + return peaks + + +def one_upload(base, token, chat_id, data): + req = Request(base + "/api/v1/chats/%s/attachments?filename=mem.png" % chat_id, data=data, method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) + t0 = time.monotonic() + try: + with urlopen(req, timeout=120) as r: + body = json.loads(r.read()) + ok = body["sha256"] == hashlib.sha256(data).hexdigest() + return r.status, time.monotonic() - t0, r.headers.get("X-Instance-Id"), ok + except HTTPError as e: + return e.code, time.monotonic() - t0, e.headers.get("X-Instance-Id"), False + except (URLError, OSError): + return 0, time.monotonic() - t0, None, False + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--concurrency", type=int, default=10) + p.add_argument("--rounds", type=int, default=2, help="повторы волны одновременных загрузок") + p.add_argument("--size-mib", type=float, default=9.5) + p.add_argument("--docker", action="store_true") + p.add_argument("--project", default=os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab4")) + p.add_argument("--pids", default="") + p.add_argument("--label", default="") + p.add_argument("--output", default="") + args = p.parse_args() + if not args.docker and not args.pids: + p.error("choose --docker or --pids") + base = args.base_url.rstrip("/") + common.BASE_URL = base + + owner = common.actor("mem") + chat = common.create_chat(owner) + files = [imagegen.noise_png(int(args.size_mib * MiB)) for _ in range(args.concurrency)] + assert all(len(f) <= 10 * MiB for f in files) + + before = cgroup_peaks(args) if args.docker else {} + proc_hwm(args, reset=True) + sampler = Sampler(args) + sampler.start() + time.sleep(1.5) # baseline + baseline = {k: max(v) for k, v in sampler.samples.items()} + results = [] + started = time.monotonic() + for _ in range(args.rounds): + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results += list(pool.map(lambda d: one_upload(base, owner["_token"], chat["id"], d), files)) + duration = time.monotonic() - started + time.sleep(1.0) + sampler.stop.set() + sampler.join(timeout=5) + after = cgroup_peaks(args) if args.docker else {} + hwm = proc_hwm(args) + + statuses = {} + for status, _, _, _ in results: + statuses[str(status)] = statuses.get(str(status), 0) + 1 + lat = sorted(r[1] for r in results if r[0] == 201) + + def pct(q): + return round(lat[min(len(lat) - 1, int(q * len(lat)))], 3) if lat else None + + memory = {} + for name, values in sampler.samples.items(): + memory[name] = { + "baseline_mib": round(baseline.get(name, values[0]) / MiB, 1), + "peak_mib": round(max(values) / MiB, 1), + "growth_mib": round((max(values) - baseline.get(name, values[0])) / MiB, 1), + "samples": len(values), + } + report = { + "label": args.label, + "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "host": "%s %s, %d CPU" % (platform.system(), platform.machine(), os.cpu_count() or 0), + "source": "docker stats + cgroup peak" if args.docker else "/proc VmRSS (20 ms)", + "concurrency": args.concurrency, + "rounds": args.rounds, + "file_mib": round(len(files[0]) / MiB, 2), + "uploads": len(results), + "statuses": statuses, + "sha256_ok": sum(1 for r in results if r[3]), + "instances": sorted({r[2] for r in results if r[2]}), + "duration_s": round(duration, 2), + "latency_201_s": {"p50": pct(0.5), "p95": pct(0.95), "max": lat[-1] if lat else None, + "mean": round(statistics.mean(lat), 3) if lat else None}, + "memory": memory, + } + report["process_peak_rss_mib"] = {k: round(v / MiB, 1) for k, v in hwm.items()} + if args.docker: + report["cgroup_peak_mib_before"] = {k: round(v / MiB, 1) for k, v in before.items()} + report["cgroup_peak_mib_after"] = {k: round(v / MiB, 1) for k, v in after.items()} + text = json.dumps(report, ensure_ascii=False, indent=2) + print(text) + if args.output: + Path(args.output).write_text(text + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/wait_url.py b/student-tests/wait_url.py new file mode 100644 index 00000000..ddf59a40 --- /dev/null +++ b/student-tests/wait_url.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Ждать HTTP 200 по URL (до 90 с). Используется внутри compose-сети: ops.py in-network.""" +import sys +import time +from urllib.request import urlopen + +url = sys.argv[1] +deadline = time.monotonic() + float(sys.argv[2] if len(sys.argv) > 2 else 90) +while time.monotonic() < deadline: + try: + if urlopen(url, timeout=2).status == 200: + print("ready: " + url) + sys.exit(0) + except OSError: + pass + time.sleep(0.5) +print("not ready: " + url) +sys.exit(1)