diff --git a/.env.example b/.env.example index 2b32655..cad08de 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,118 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Секреты сюда НЕ пишутся: их создаёт scripts/init_secrets.py в каталоге secrets/. APP_HOST=0.0.0.0 +# Публичный порт LB на 127.0.0.1 (единственная точка входа API) APP_PORT=8080 +# Страница состояния HAProxy (UP/DOWN реплик) на 127.0.0.1, только чтение +LB_STATS_PORT=8404 APP_ENV=lab +# Тег образа API и worker (один образ) +APP_VERSION=lab6 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +# TTL экземпляра api-short-ttl (профиль ttl) для автотеста истечения +SHORT_SESSION_TTL_SECONDS=2 +# Файлы с секретами внутри контейнера (Compose secrets) DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# Источник истины для сессий — PostgreSQL; Redis — общий кэш проверок токена SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url +SESSION_CACHE_TTL_SECONDS=60 +# Число реплик API за балансировщиком (HAProxy держит до 6 слотов) API_REPLICAS=2 +# Хранение результата Idempotency-Key, не меньше 24 ч по контракту IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# Пул соединений и таймауты (на реплику) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +CACHE_TIMEOUT_SECONDS=0.25 +CACHE_RETRY_AFTER_SECONDS=5 +# Readiness и остановка (согласованы с HAProxy: inter 1s, fall 2, timeout check 1.5s) +READY_DB_TIMEOUT_SECONDS=1 +DRAIN_SECONDS=3 +GRACEFUL_TIMEOUT_SECONDS=10 +KEEP_ALIVE_SECONDS=30 +# Одновременных хешей Argon2id на реплику (19 MiB каждый, лимит памяти 256M) +PASSWORD_HASH_CONCURRENCY=4 + +# lab4: S3 и вложения +# Закреплённая версия образа chrislusf/seaweedfs (не latest) +SEAWEEDFS_VERSION=4.50 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments +# Ключи — файлы в secrets/ (создаёт scripts/init_secrets.py), внутри контейнера: S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Таймауты клиента S3: недоступность S3 -> быстрый 503 только для файловых операций +S3_CONNECT_TIMEOUT_SECONDS=2 +S3_READ_TIMEOUT_SECONDS=10 +S3_MAX_ATTEMPTS=2 +# Общий бюджет вызова S3 из запроса, включая DNS (скачивание < timeout server HAProxy 10 с) +S3_DOWNLOAD_DEADLINE_SECONDS=5 +S3_UPLOAD_DEADLINE_SECONDS=10 +# После отказа S3 файловые запросы N секунд сразу получают 503 (circuit breaker) +S3_CIRCUIT_OPEN_SECONDS=5 +# Лимиты контракта (можно только уменьшить) MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Одновременных загрузок на реплику (остальные ждут слот, затем 503 + Retry-After) +MAX_CONCURRENT_UPLOADS=4 +UPLOAD_QUEUE_TIMEOUT_SECONDS=5 +# Максимальное время приёма тела загрузки (медленный/зависший клиент -> 400) +UPLOAD_READ_TIMEOUT_SECONDS=60 +# Одновременных разборов изображения на реплику +IMAGE_DECODE_CONCURRENCY=2 +# Сборщик незавершённых загрузок (pending) и объектов-сирот, см. src/reconcile.py +ATTACHMENT_GC_INTERVAL_SECONDS=300 +ATTACHMENT_GC_GRACE_SECONDS=900 +ORPHAN_SWEEP_INTERVAL_SECONDS=3600 + +# lab5: брокер, задания, worker. Логические имена из skeleton .env.example: +# BROKER_URL_FILE, WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS, STAGING_RETENTION_SECONDS +# URL брокера с паролем — файл secrets/broker_url (создаёт scripts/init_secrets.py) BROKER_URL_FILE=/run/secrets/broker_url +# Поток Redis Streams и consumer group +BROKER_STREAM=attachments.jobs +BROKER_GROUP=attachment-workers +BROKER_TIMEOUT_SECONDS=2 +# Версия pipeline вариантов (thumbnail/crop/optimized v1) +PIPELINE_VERSION=1 +# Контейнеров worker и потоков-consumer в каждом (prefetch = 1 на поток) +WORKER_REPLICAS=1 WORKER_CONCURRENCY=2 +# Попыток на задание (временные ошибки), потом failed retry_exhausted + DLQ JOB_MAX_ATTEMPTS=5 +# Аренда (lease) задания; должна покрывать обработку + S3 (>= WORKER_IMAGE_TIMEOUT_SECONDS + 20) JOB_TIMEOUT_SECONDS=60 +# Запас после истечения аренды до перезапуска задания сборщиком +JOB_LEASE_GRACE_SECONDS=10 +# Через сколько секунд простоя сообщение упавшего consumer забирает другой (XAUTOCLAIM), > JOB_TIMEOUT_SECONDS +JOB_CLAIM_IDLE_SECONDS=75 +# Доставок одного сообщения до признания его «ядовитым» (poison_message) +JOB_MAX_DELIVERIES=10 +# Backoff: d = min(MAX, BASE * 2^(attempt-1)), задержка равномерно в [d/2, d] +JOB_RETRY_BASE_SECONDS=2 +JOB_RETRY_MAX_SECONDS=60 +# Лимиты обработки одного изображения в подпроцессе worker +WORKER_IMAGE_TIMEOUT_SECONDS=30 +WORKER_IMAGE_MEMORY_MB=1024 +# Staging failed-вложений хранится столько (для replay), затем удаляется сборщиком STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +# Outbox relay: период sweeper и задержка, дающая сначала сработать быстрому пути +RELAY_POLL_INTERVAL_SECONDS=1 +RELAY_FAST_PATH_GRACE_SECONDS=2 + +# lab6: наблюдаемость (src/telemetry.py, infra/otel, infra/prometheus, infra/loki, infra/tempo, infra/grafana) +# true — выключить OTLP-экспорт (останутся JSON-логи в stdout); для сравнения off/on +OTEL_SDK_DISABLED=false +# Collector внутри сети telemetry; API и worker шлют OTLP/HTTP (protobuf) OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +# service.name задаётся в compose: messenger-api / messenger-worker OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Порты UI на 127.0.0.1 (пароль Grafana — secrets/grafana_admin_password, логин admin) +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d72808b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# Скрипты и конфиги, которые выполняются внутри Linux-контейнеров или sh, — всегда LF, +# даже при checkout на Windows (иначе HAProxy/sh спотыкаются о CRLF). +*.sh text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.py text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf +# Пакетные файлы Windows — CRLF. +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..0455655 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Попов Константин Дмитриевич, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: — -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: весь минимум lab5 без изменений бизнес-API; OTel SDK в API и worker (traces/metrics/logs по OTLP), W3C traceparent через outbox → Redis Streams → worker (producer/consumer spans, retry в той же трассе), Collector (memory_limiter, batch, ограниченные очереди) → Prometheus / Loki / Tempo → Grafana (provisioning + dashboard), SLI/SLO, alert rules ApiHighErrorRate / QueueLagGrowing / TelemetryExportFailing, runbooks, сценарии инцидентов (scripts/incident.py). +Команды запуска из чистого clone: `make up`, `make test`, `make obs-urls`. +Результаты проверок и ссылка на REPORT.md: REPORT.md, evidence/. +Ограничения / известные ошибки: см. REPORT.md «Ограничения». +Источники и использование ИИ: REPORT.md. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..26c5e6b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# Lab6: API мессенджера и worker обработки вложений (один образ) + OpenTelemetry SDK: FastAPI + PostgreSQL + Redis + S3/SeaweedFS + Redis Streams. +FROM python:3.12.7-slim-bookworm + +# lab4: настройки glibc malloc — меньше арен на потоки и возврат больших буферов +# ОС сразу после загрузки (замер: базовый RSS 124 -> 88 MiB, см. REPORT.md). +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + MALLOC_ARENA_MAX=2 \ + MALLOC_MMAP_THRESHOLD_=1048576 \ + MALLOC_TRIM_THRESHOLD_=1048576 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --disable-pip-version-check -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +# Готовность (а не только жизнь процесса): БД доступна, схема мигрирована, +# реплика не в фазе осушения. Интервалы в compose.yaml переопределяют эти. +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" || exit 1 + +# exec-форма: Python — PID 1 и получает SIGTERM напрямую. +# src/server.py: SIGTERM -> осушение DRAIN_SECONDS (readiness 503) -> graceful stop. +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src.server"] diff --git a/Makefile b/Makefile index 5d86d17..91c1811 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,20 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover \ + secrets up wait ps down migrate restore rolling-update logs-lb \ + test-student test-idempotency test-failover-series test-failover-series-kill \ + test-rolling test-storage test-expiry test-failures test-revocation explain evidence \ + test-attachments test-consistency test-s3-outage test-upload-memory s3-check wait-s3 reconcile reconcile-apply \ + test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay \ + obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch destroy help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up/wait/ps/down/restore/rolling-update: stand; test-*: own checks (see REPORT.md)" + @echo "lab4: test-attachments test-consistency test-s3-outage test-upload-memory s3-check reconcile" + @echo "lab5: test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay ID=..." + @echo "lab6: obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +38,158 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# ---------------------------------------------------------------- Решение +COMPOSE ?= docker compose --env-file .env + +secrets: + $(PYTHON) scripts/init_secrets.py + @test -f .env || cp .env.example .env + +up: secrets + $(COMPOSE) config --quiet + $(COMPOSE) up -d --build + $(PYTHON) scripts/ops.py wait + $(PYTHON) scripts/ops.py wait-s3 + $(PYTHON) scripts/ops.py worker-start + +wait: + $(PYTHON) scripts/ops.py wait + +ps: + $(PYTHON) scripts/ops.py ps + +# Безопасная остановка: volumes (pgdata, s3data, brokerdata, promdata, lokidata, tempodata, grafanadata) сохраняются. +down: + $(COMPOSE) --profile ttl --profile faults down + +# НАМЕРЕННОЕ удаление всех учебных данных (БД, S3, брокер, история телеметрии). +destroy: + $(COMPOSE) --profile ttl --profile faults down -v + +migrate: + $(COMPOSE) run --rm -T migrate + +restore: + $(PYTHON) scripts/ops.py restore + +rolling-update: + $(PYTHON) scripts/ops.py rolling-update + +logs-lb: + $(COMPOSE) logs --tail 50 lb + +test-student: + $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" + +test-idempotency: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_idempotency.py -v + +test-failover-series: + $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover_series.json + +test-failover-series-kill: + $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --mode kill --output evidence/failover_series_kill.json + +test-rolling: + $(PYTHON) student-tests/rolling_update_load.py --base-url "$(BASE_URL)" --output evidence/rolling_update.json + +test-storage: + $(PYTHON) scripts/ops.py in-network storage + +test-expiry: + $(PYTHON) scripts/ops.py in-network expiry + +test-failures: + $(PYTHON) student-tests/dependency_failure.py --base-url "$(BASE_URL)" --output evidence/dependency_failure.json + +test-revocation: + $(PYTHON) student-tests/revocation_partition.py --base-url "$(BASE_URL)" | tee evidence/revocation_partition.txt + +explain: + $(COMPOSE) exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 < scripts/sql/explain_history.sql > evidence/explain_history.txt + @echo "saved evidence/explain_history.txt" + +# Все проверки на compose-стенде с записью результатов в evidence/ (~6-8 минут). +evidence: + $(PYTHON) scripts/collect_evidence.py + +# ---------------------------------------------------------------- lab4: вложения и S3 +test-attachments: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_attachments.py -v + +test-consistency: + $(PYTHON) scripts/ops.py in-network consistency + +test-s3-outage: + $(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" --output evidence/s3_outage.json + +test-upload-memory: + $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" --concurrency 10 --rounds 3 --docker --output evidence/upload_memory_c10.json + +s3-check: + $(PYTHON) scripts/ops.py s3-check + +wait-s3: + $(PYTHON) scripts/ops.py wait-s3 + +reconcile: + $(PYTHON) scripts/ops.py reconcile + +reconcile-apply: + $(PYTHON) scripts/ops.py reconcile --apply + +# ---------------------------------------------------------------- lab5: очередь и worker +test-pipeline: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_pipeline.py -v + +# Внедрённые отказы (outbox, crash до/после ack, retry/backoff, DLQ, replay, lease). ~2-3 мин. +test-pipeline-faults: + $(PYTHON) scripts/ops.py pipeline-faults + +# Сценарий защиты уровня 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready. +test-worker-outage: + $(PYTHON) student-tests/worker_outage.py --base-url "$(BASE_URL)" --output evidence/worker_outage.json + +worker-stop: + $(PYTHON) scripts/ops.py worker-stop + +worker-start: + $(PYTHON) scripts/ops.py worker-start + +jobs-stats: + $(PYTHON) scripts/ops.py jobs stats + +jobs-failed: + $(PYTHON) scripts/ops.py jobs failed + +# make replay ID= (или ID=--all-failed) +replay: + @test -n "$(ID)" || (echo "usage: make replay ID=|--all-failed"; exit 2) + $(PYTHON) scripts/ops.py jobs replay $(ID) + +# ---------------------------------------------------------------- lab6: наблюдаемость +obs-urls: + @echo "Grafana: http://127.0.0.1:3000 (логин admin, пароль: secrets/grafana_admin_password)" + @echo "Prometheus: http://127.0.0.1:9090/alerts" + +# Контролируемый upload: печатает X-Request-Id и готовые запросы TraceQL/LogQL +obs-probe: + $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" probe + +# Инцидент 1: worker остановлен ~150 с под upload'ами -> QueueLagGrowing firing -> resolved +incident-worker: + $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" worker-stop + +# Инцидент 2: таблица messages заблокирована 180 с под нагрузкой -> ApiHighErrorRate firing -> resolved +incident-db: + $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" db-lock + +# Инцидент 3: остановить telemetry (по умолчанию otel-collector; SERVICES=tempo,loki) +SERVICES ?= otel-collector +incident-telemetry: + $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" telemetry-down --services "$(SERVICES)" + +# Журнал переходов alert'ов (запускать в отдельном терминале параллельно с инцидентом) +alerts-watch: + $(PYTHON) scripts/alerts_watch.py --duration 900 diff --git a/REPORT.md b/REPORT.md index b97ae42..d0ee4c7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,136 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: OpenTelemetry Python SDK 1.29.0 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`), инструментации 0.50b0 (`fastapi`, `psycopg`, `botocore`); OpenTelemetry Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.6.1, Grafana 11.4.0. Документация: OTel Collector configuration (memory_limiter, batch, sending_queue/retry_on_failure), W3C Trace Context, OTel messaging semantic conventions (producer/consumer, `messaging.*`), HTTP semconv (`http.server.request.duration`), Prometheus histograms / `histogram_quantile`, Loki OTLP ingestion (structured metadata), Grafana provisioning. Остальной стек — как в lab5 (см. `docs/lab5/REPORT.md`). Помощь ИИ (Claude) — при проектировании схемы сигналов, написании `src/telemetry.py`, конфигураций, dashboard, alert rules, сценариев инцидентов и отчёта. + +Минимум lab5 перенесён целиком (HAProxy, 2 реплики API, PostgreSQL, Redis-кэш, SeaweedFS S3, Redis Streams broker, worker, outbox, retry/DLQ). **Бизнес-API не менялся**; новые поля только служебные: `traceparent`/`tracestate` в сообщении задания (уже разрешены `contracts/jobs.schema.json`). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Версии.** API и worker — один образ из `Dockerfile` (`python:3.12.7-slim-bookworm`), `postgres:16.4-alpine`, `redis:7.4.1-alpine` (кэш и broker), `haproxy:2.8.30-alpine`, `chrislusf/seaweedfs:4.50`, **`otel/opentelemetry-collector-contrib:0.115.1`, `prom/prometheus:v2.55.1`, `grafana/loki:3.3.2`, `grafana/tempo:2.6.1`, `grafana/grafana:11.4.0`**. Нужны Docker с Compose v2 и Python 3.9+ на хосте. Оркестрация — Compose. + +```sh +make up # = python scripts/init_secrets.py; cp .env.example .env; + # docker compose --env-file .env up -d --build; ожидание готовности API/S3/worker +make test # публичный smoke через LB http://localhost:8080 +make obs-urls # Grafana http://127.0.0.1:3000 (admin / secrets/grafana_admin_password), Prometheus :9090 +make down # безопасная остановка, все volumes сохраняются +make destroy # НАМЕРЕННОЕ удаление всех данных (down -v): БД, S3, очередь, история телеметрии +``` + +Bootstrap: секреты (`secrets/`, вне Git) создаёт `scripts/init_secrets.py`, в т.ч. новый `grafana_admin_password`. Миграции — one-shot `migrate`, бакет — `s3-init` (как в lab5). Provisioning Grafana (datasources + dashboard) и rules Prometheus подключаются из Git read-only при старте — ручной набор не нужен. Повторный `make up` идемпотентен. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент -> HAProxy -> API x2 ──SQL──> PostgreSQL (источник истины, job_outbox) + │ └─S3──> SeaweedFS + └─outbox relay─XADD(traceparent)─> broker (Redis Streams) ─> worker ─S3/SQL─> ... +API/worker ──OTLP/HTTP :4318──> otel-collector ─┬─ traces ─OTLP gRPC─> Tempo (volume tempodata) + ├─ metrics ─/metrics :8889 <─scrape─ Prometheus (promdata, rules) + └─ logs ─OTLP /otlp─> Loki (lokidata) + Grafana (grafanadata) читает все три +``` + +| Роль | Сервис | Образ | Назначение | Volume | Порт наружу | +| --- | --- | --- | --- | --- | --- | +| Collector | `otel-collector` | contrib 0.115.1 | приём OTLP, memory_limiter, batch, ограниченные очереди, экспорт | — (не хранилище) | нет | +| Метрики | `prometheus` | v2.55.1 | TSDB метрик 7 д, вычисление recording/alert rules, состояние firing/resolved | `promdata` | 127.0.0.1:9090 | +| Логи | `loki` | 3.3.2 | хранение логов 7 д; индекс только по `service_name`/`service_instance_id`, остальное — structured metadata | `lokidata` | нет | +| Трассы | `tempo` | 2.6.1 | хранение spans 72 ч, поиск TraceQL | `tempodata` | нет | +| UI | `grafana` | 11.4.0 | запросы и dashboard; сама сигналы не хранит | `grafanadata` | 127.0.0.1:3000 | + +**Почему три хранилища.** Метрика отвечает «сколько и как часто» (дёшево, агрегаты, алерты), но не «какой запрос»; лог отвечает «что именно произошло и почему» в одном процессе; trace отвечает «где ушло время» по цепочке процессов. Prometheus хранит числовые ряды с метками, Loki — строки с минимальным индексом, Tempo — деревья spans по trace_id. + +**Инструментация (`src/telemetry.py`).** Вызывается до создания пула БД/клиента S3: +- Resource: `service.name` = `messenger-api` / `messenger-worker`, `service.version` = `APP_VERSION`, `service.instance.id` = `X-Instance-Id` (hostname контейнера), `deployment.environment.name=lab`. +- Traces: FastAPI (SERVER span, только шаблон маршрута; health исключены; внутренние send/receive spans отключены), psycopg (SQL **без параметров**), botocore (S3). Redis **не** инструментируется: команды `AUTH` и ключи кэша сессий не должны попасть в spans. Sampler `DropOrphanClientSpans`: CLIENT-span без родителя не сохраняется — это фоновые SQL/S3 (sweeper outbox раз в секунду, GC, readiness, callbacks метрик), иначе Tempo заполнялся бы тысячами пустых трасс. Всё внутри SERVER/PRODUCER/CONSUMER сохраняется (ParentBased AlwaysOn). +- Logs: JSON в stdout (для `docker logs`) и OTLP. Поля: `timestamp`, `severity`, `service.name`, `event`, `trace_id`, `span_id` (если есть активный span), плюс технические поля (`request_id`, `http_route`, `outcome`...). Каждый бизнес-запрос даёт строку `event=http.request` с шаблоном маршрута, статусом, длительностью и `request_id`; каждое сообщение worker'а — `event=job.processed`. +- `X-Request-Id` записывается атрибутом `messenger.request_id` в SERVER span: ответ с ошибкой → trace → лог. + +**Перенос контекста через очередь (W3C).** `job_payload()` записывает `traceparent` текущего span в JSON задания, который хранится в `job_outbox` (PostgreSQL) — контекст переживает задержку публикации и рестарты. Relay (быстрый путь API или sweeper любого процесса) создаёт **PRODUCER** span `attachments.jobs publish` — ребёнок span'а из строки outbox — и кладёт в брокер уже его `traceparent`. Worker создаёт **CONSUMER** span `attachments.jobs process` — ребёнок producer. Внутри него — SQL claim/complete и S3 GetObject/PutObject. При временной ошибке `retry_or_fail()` пишет новую строку outbox с `traceparent` consumer span → следующая попытка в **той же** трассе (связь parent-child, а не link: одна логическая операция «upload → ready»). + +**Сигналы и их реальные имена в Prometheus** (Collector `prometheus` exporter: точки → `_`, единица → суффикс, counter → `_total`; `job` = service.name, `instance` = service.instance.id): + +| Логический сигнал (contracts/observability.md) | Имя OTel | Имя в Prometheus | Атрибуты | Ед. | +| --- | --- | --- | --- | --- | +| HTTP requests (counter) | `http.server.request.duration` (count) | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code` → класс через `label_replace` | шт | +| HTTP duration | `http.server.request.duration` | `http_server_request_duration_seconds_bucket` | те же; buckets 5 мс…10 с (View) | с | +| Jobs processed/failed | `messenger.jobs.processed` | `messenger_jobs_processed_total` | `job_type`, `outcome` = succeeded/retry/failed/noop/lease_lost/poison/error | шт | +| Job duration | `messenger.job.duration` | `messenger_job_duration_seconds_bucket` | `job_type`, `outcome`; buckets 50 мс…60 с | с | +| Queue depth | `messenger.queue.depth` | `messenger_queue_depth` | `queue`, `state`=queued/processing | шт | +| Oldest job age | `messenger.queue.oldest_age` | `messenger_queue_oldest_age_seconds` | `queue` | с | +| Outbox backlog | `messenger.outbox.pending` | `messenger_outbox_pending` | `queue` | шт | +| DB pool | `messenger.db.pool.connections` / `.waiting` / `.max` | `messenger_db_pool_connections{state=used\|idle}`, `messenger_db_pool_waiting`, `messenger_db_pool_max` | — | шт | +| Worker saturation | `messenger.worker.busy` / `.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | — | слоты | +| CPU / RAM процесса | `process.cpu.time`, `process.memory.rss` | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | — | с, байты | +| End-to-end time to ready | `messenger.attachment.time_to_ready` | `messenger_attachment_time_to_ready_seconds_bucket` | `job_type`; от COMMIT задания (ответ 202) до COMMIT ready | с | +| Публикации relay | `messenger.jobs.published` | `messenger_jobs_published_total` | `job_type` | шт | + +Глубина/возраст очереди считаются из PostgreSQL (источник истины) и публикуются **API-репликами**: при остановленном worker метрика продолжает жить. Реплики дают одинаковые значения — в запросах `max()`. В метках **нет** user_id/chat_id/message_id/attachment_id/request_id/URL/filename. Технические ID (job_id, attachment_id, request_id) есть только в атрибутах spans и логов — они нужны для поиска конкретного upload, данные синтетические. + +**Что не экспортируется:** тела сообщений, байты изображений, filename (в логах/spans нет), заголовки Authorization/Cookie (инструментация их не пишет), параметры SQL (`db.statement` — шаблон с `%s`), подписанные URL (их нет в API), пароли/токены. Проверено: `grep` по всем выгруженным spans/логам локального прогона на `Authorization|Bearer |password` — 0 совпадений. + +**Перцентили** считаются только из histogram: `histogram_quantile(0.95, sum by (le) (rate(..._bucket[5m])))` — buckets всех реплик суммируются, а не усредняются p95 реплик. Важная найденная ошибка: инструментация 0.50b0 при метрике в секундах ставит buckets по умолчанию 0…10000 (миллисекундные) — все запросы попадали бы в первый bucket; заданы явные buckets через View. + +**SLI/SLO** (`infra/prometheus/rules/messenger.yml`, recording rules): +- доступность: `1 − 5xx/все` бизнес-запросы, health исключены, 4xx не ошибка сервиса; **SLO 99% за скользящий 1 ч**; +- задержка: доля запросов быстрее 0.5 с (bucket `le="0.5"`); **SLO 95% за 1 ч**; +- очередь: возраст старейшего незавершённого задания; цель < 60 с. + +**Alert rules** (состояние — Prometheus `/alerts`, Grafana; доставка наружу не нужна), runbook — [docs/runbooks.md](docs/runbooks.md): + +| Alert | Условие | Окно / for | +| --- | --- | --- | +| `ApiHighErrorRate` | доля 5xx > 5% и трафик ≥ 0.2 RPS | rate 1m, for 1m | +| `QueueLagGrowing` | `max(messenger_queue_oldest_age_seconds) > 60` | for 30s | +| `TelemetryExportFailing` | растут `otelcol_exporter_send_failed_*` / `enqueue_failed_*` | rate 1m, for 1m | + +**Collector и ограниченная буферизация** (`infra/otel/collector.yaml`): `memory_limiter` (180 MiB, spike 40) → `batch` (512/1024, 2 с) → exporters с `sending_queue` **20 батчей** и `retry_on_failure` не дольше 60 с; `GOMEMLIMIT=200MiB`, лимит контейнера 384M. Переполнение очереди → отбрасывание с подсчётом (`otelcol_exporter_enqueue_failed_*`). На стороне SDK: `BatchSpanProcessor`/`BatchLogRecordProcessor` с очередью 2048, таймаут экспорта 5 с; при недоступном Collector батч отбрасывается, ошибки SDK — одна JSON-строка в stdout без трейсбека и **не** в OTLP (иначе петля «лог о неудачной отправке логов»). Ни API, ни worker не имеют `depends_on` на телеметрию. + +**Переменные `.env.example` → конфигурация:** `OTEL_SDK_DISABLED` (выключатель), `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, `OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab`; `OTEL_SERVICE_NAME` задаётся в compose отдельно для API и worker. `GRAFANA_PORT`, `PROMETHEUS_PORT` — порты UI на 127.0.0.1. Остальное — как в lab5. + +**Что изменено относительно lab5:** `src/telemetry.py` (новый); `src/main.py` (setup, FastAPI-инструментация, access-лог, метрики пула/очереди), `src/worker.py` (consumer span, метрики заданий/пула/занятости, time-to-ready), `src/outbox.py` (producer span, повторная инъекция контекста), `src/jobs.py` (`traceparent` в payload, `complete()` возвращает время до ready); `requirements.txt` (+OTel); `compose.yaml` (+5 сервисов, сеть `telemetry`, 4 volume, OTEL_* env); `infra/{otel,prometheus,loki,tempo,grafana}`; `scripts/incident.py`, `scripts/alerts_watch.py`; Makefile (`obs-*`, `incident-*`, `alerts-watch`, `destroy`); `docs/runbooks.md`. Отчёт lab5 и его доказательства перенесены в `docs/lab5/`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +**Важно о среде.** Доказательства в `evidence/local-sandbox/` получены в песочнице **без доступа к Docker Hub**: те же исходники запускались напрямую (Python 3.13, PostgreSQL 16.2, Redis 6.2, S3-эмулятор moto вместо SeaweedFS, 1 реплика API без HAProxy) с **настоящими** бинарниками Collector 0.115, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.6.1 и **конфигурациями из этого репозитория**. Grafana там не запускалась (дистрибутив недоступен) — все PromQL-запросы dashboard выполнены через Prometheus API (0 ошибок), LogQL/TraceQL — через API Loki/Tempo. **Docker-стенд**: Windows 10, Ryzen 5 7500f, 32 GB; доказательства в корне `evidence/`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke при включённой телеметрии | `make test` | `PASS: lab6 public API scenario` | `evidence/local-sandbox/smoke.txt` | +| `make check` | `make check` | `PASS: offline skeleton structure` | — | +| API и worker отдают OTLP в Collector; три хранилища | `make obs-probe` → Loki/Tempo API | лог `event=http.request` с `trace_id` в Loki; trace с тем же id в Tempo | `trace_retry_chain.txt` | +| RPS/errors/duration API; jobs success/error и duration worker | dashboard, Prometheus API | все 31 PromQL dashboard возвращают данные | `metric_names.txt` | +| Структурные логи с trace_id/span_id | Loki `{service_name=~"messenger-.+"} \| request_id=""` | API `http.request` и worker `job.processed` одного upload — один `trace_id` | `trace_retry_chain.txt` | +| Импортируемый dashboard | `infra/grafana/dashboards/messenger.json` + provisioning | загружается при старте Grafana, папка Messenger | — | +| Нет секретов/тел сообщений в telemetry | grep выгрузки spans/logs | 0 совпадений `Authorization`, `Bearer `, `password`; SQL без параметров | — | +| Уровень 4: W3C через broker, producer/consumer, retry, S3/DB | upload `fault-transient-until-3.png` при `WORKER_FAULTS=true` | SERVER → PRODUCER → CONSUMER(retry) → PRODUCER → CONSUMER(retry) → PRODUCER → CONSUMER(succeeded) → S3.PutObject×4, SQL — одна трасса из 76 spans | `trace_retry_chain.txt`, `trace_upload_chain.txt` | +| p50/p95/p99, ошибки, queue depth/age, DB pool, saturation worker | dashboard «Messenger lab6» | панели с единицами (s, percentunit, reqps, bytes) | `messenger.json` | +| SLI/SLO и окно; alert error rate | `make incident-db` + `make alerts-watch` | lock 180 с: pending +60 с, **firing +120 с**, **resolved** через 21 с после снятия lock | `alerts_db_lock_180s.txt` | +| Alert queue lag | `make incident-worker` + `make alerts-watch` | queued растёт, `QueueLagGrowing` pending → **firing** → **resolved**; 22 upload'а готовы через 8.7 с после старта worker | `alerts_queue_lag_and_db_short.txt`, `incident_worker_stop.jsonl` | +| Медленная БД отличима от медленного handler | `make incident-db` | 503 только на `/messages`; CLIENT span `INSERT` 5.01 с `QueryCanceled` | `db_lock_span.txt` | +| Collector memory limiter/batch, недоступный backend | `make incident-telemetry` (`SERVICES=otel-collector` и `SERVICES=tempo,loki`) | API 0 ошибок, ~300 RPS; RSS API 135.0→135.4 MiB; Collector пик 234 MiB, очередь 20/20, потери посчитаны | `telemetry_outage.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Локальная песочница: 1 реплика API (без LB), 1 worker (2 потока), нагрузка `tests/load.py` (closed loop, POST сообщений, 2–4 потока), ~300 успешных RPS. **Docker-стенд**: Windows 10, Ryzen 5 7500f, 32 GB; доказательства в корне `evidence/`. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен 110 с, upload каждые 5 с | 202 продолжаются; queue depth и oldest age растут; QueueLagGrowing firing | queued до 22, age > 90 с, firing; API без ошибок | старт worker → все 22 ready за 8.7 с, alert resolved; потерь нет | +| `LOCK TABLE messages` 180 с под нагрузкой | 503 на `/messages` после `statement_timeout` 5 с; ApiHighErrorRate firing; trace показывает SQL | доля 5xx = 100% на маршруте; firing через 120 с; SQL span 5.01 с `QueryCanceled` | resolved через 21 с после снятия lock; запросы с 503 не записаны (клиент повторяет с тем же Idempotency-Key) | +| Collector остановлен 45 с | API работает, RAM не растёт, телеметрия за период теряется | 0 ошибок, 303 RPS; RSS 135.0 → 135.4 MiB | после старта экспорт возобновился; spans/логи/метрики за 45 с потеряны (SDK отбрасывает) | +| Tempo + Loki остановлены 120 с (Collector жив) | очередь Collector заполняется и ограничена; потери считаются | queue 20/20; `enqueue_failed_spans` 131083, `..._log_records` 22016; пик RSS 234 MiB; API 0 ошибок | после старта backend очередь → 0, новые trace/логи видны; потеря — посчитанное число | + +Детекция ошибок не мгновенная: `rate(...[1m])` первую минуту «разбавлен» трафиком до инцидента (при closed-loop нагрузке RPS падает в сотни раз), поэтому pending наступает примерно через 60 с, firing — ещё через `for: 1m`. Инцидент короче ~2 мин до firing не доходит (наблюдалось на lock 100 с — только pending). Это осознанный компромисс против ложных срабатываний; для быстрой детекции нужны burn-rate alerts с короткими окнами (трек 5, не заявлен). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Весь стенд на одном Docker-host; Collector, Prometheus, Loki, Tempo — по одному экземпляру без репликации. Потеря host = потеря телеметрии и приложения. +- При недоступном Collector телеметрия за этот период теряется (SDK держит только ограниченную очередь в памяти) — так сделано намеренно: приложение важнее телеметрии. +- Метрики очереди публикуют API-реплики: если упадут все API, метрика пропадёт (alert на отсутствие данных не настроен). +- Sampling 100% (кроме фоновых сиротских SQL) — на учебной нагрузке приемлемо; цену наблюдаемости и tail sampling не исследовал (трек 5 не заявлен). +- Метрики `process_*` — RSS/CPU процесса, а не cgroup контейнера; для контейнеров можно добавить cAdvisor. diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 0000000..91e8095 --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,17 @@ +# lab5: стенд внедрения отказов worker'а (только учебный, не для обычной работы). +# docker compose -f compose.yaml -f compose.faults.yaml up -d --no-deps worker +# Включает маркеры в filename (fault-crash-after-effects, fault-crash-before-ack, +# fault-transient-until-3, fault-transient-until-replay, fault-permanent, fault-slow) +# и укорачивает аренду/backoff, чтобы сценарии проходили за секунды. +# Возврат к обычному worker: docker compose up -d --no-deps --force-recreate worker +# Используется автоматически: python scripts/ops.py pipeline-faults +services: + worker: + environment: + WORKER_FAULTS: "true" + JOB_TIMEOUT_SECONDS: "25" + JOB_LEASE_GRACE_SECONDS: "2" + JOB_CLAIM_IDLE_SECONDS: "30" + JOB_RETRY_BASE_SECONDS: "1" + JOB_RETRY_MAX_SECONDS: "4" + WORKER_IMAGE_TIMEOUT_SECONDS: "5" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..4123a8c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,741 @@ +# Лабораторная 6: стенд lab5 + наблюдаемость. +# API x N / worker --OTLP/HTTP--> otel-collector --> Prometheus (метрики, alert rules) +# --> Loki (логи) +# --> Tempo (трассы) +# Grafana (127.0.0.1:3000) читает все три; provisioning и dashboard — infra/grafana. +# Сеть telemetry: api, worker, collector, хранилища, grafana. Недоступность любого +# telemetry-сервиса не блокирует API (нет depends_on, экспорт асинхронный и ограничен). +# +# База lab5: HAProxy -> API x N (по умолчанию 2) -> PostgreSQL (метаданные, задания, outbox) +# + Redis (кэш сессий) +# + SeaweedFS S3 (staging/ и final/ в приватном бакете) +# API/worker outbox relay -> broker (Redis Streams, AOF fsync always) -> worker x M +# Сети: backend — lb, api, worker, postgres, redis; storage (internal) — api, worker, s3, s3-init; +# queue (internal) — api, worker, broker. Наружу (127.0.0.1) опубликован только LB. +name: messenger-lab6 + +# lab5: общие настройки очереди для API (relay) и worker. +x-queue-env: &queue-env + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_STREAM: ${BROKER_STREAM:-attachments.jobs} + BROKER_GROUP: ${BROKER_GROUP:-attachment-workers} + BROKER_TIMEOUT_SECONDS: ${BROKER_TIMEOUT_SECONDS:-2} + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_GRACE_SECONDS: ${JOB_LEASE_GRACE_SECONDS:-10} + JOB_CLAIM_IDLE_SECONDS: ${JOB_CLAIM_IDLE_SECONDS:-75} + JOB_MAX_DELIVERIES: ${JOB_MAX_DELIVERIES:-10} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + RELAY_POLL_INTERVAL_SECONDS: ${RELAY_POLL_INTERVAL_SECONDS:-1} + RELAY_FAST_PATH_GRACE_SECONDS: ${RELAY_FAST_PATH_GRACE_SECONDS:-2} + +# lab6: OpenTelemetry SDK (src/telemetry.py). Очереди SDK ограничены: при недоступном +# Collector батчи отбрасываются, RAM процесса не растёт. +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + # Python SDK: секунды + OTEL_EXPORTER_OTLP_TIMEOUT: "5" + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + OTEL_SEMCONV_STABILITY_OPT_IN: http + OTEL_BSP_MAX_QUEUE_SIZE: "2048" + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: "512" + OTEL_BSP_EXPORT_TIMEOUT: "5000" + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" + OTEL_BLRP_EXPORT_TIMEOUT: "5000" + OTEL_METRIC_EXPORT_INTERVAL: "10000" + +x-api-env: &api-env + APP_ENV: ${APP_ENV:-lab} + APP_VERSION: ${APP_VERSION:-lab6} + OTEL_SERVICE_NAME: messenger-api + <<: [*queue-env, *otel-env] + SESSION_STORE: postgres + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.25} + CACHE_RETRY_AFTER_SECONDS: ${CACHE_RETRY_AFTER_SECONDS:-5} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + READY_DB_TIMEOUT_SECONDS: ${READY_DB_TIMEOUT_SECONDS:-1} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + GRACEFUL_TIMEOUT_SECONDS: ${GRACEFUL_TIMEOUT_SECONDS:-10} + KEEP_ALIVE_SECONDS: ${KEEP_ALIVE_SECONDS:-30} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + # Реплики НЕ применяют миграции — это делает сервис migrate. + RUN_MIGRATIONS_ON_START: "false" + # lab4: S3 (credentials API — отдельные, только Read/Write/List одного бакета) + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10} + S3_MAX_ATTEMPTS: ${S3_MAX_ATTEMPTS:-2} + # Общий бюджет вызова S3 (вкл. DNS) и circuit breaker — см. src/main.py s3_call + S3_DOWNLOAD_DEADLINE_SECONDS: ${S3_DOWNLOAD_DEADLINE_SECONDS:-5} + S3_UPLOAD_DEADLINE_SECONDS: ${S3_UPLOAD_DEADLINE_SECONDS:-10} + S3_CIRCUIT_OPEN_SECONDS: ${S3_CIRCUIT_OPEN_SECONDS:-5} + # Лимиты загрузки (не выше контракта: 10 MiB, 40 Мп) + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-4} + UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} + UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-60} + IMAGE_DECODE_CONCURRENCY: ${IMAGE_DECODE_CONCURRENCY:-2} + # Сборщик незавершённых загрузок и объектов-сирот (src/reconcile.py) + ATTACHMENT_GC_INTERVAL_SECONDS: ${ATTACHMENT_GC_INTERVAL_SECONDS:-300} + ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900} + ORPHAN_SWEEP_INTERVAL_SECONDS: ${ORPHAN_SWEEP_INTERVAL_SECONDS:-3600} + +x-api-healthcheck: &api-healthcheck + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 3s + timeout: 2s + retries: 2 + start_period: 20s + +x-api-limits: &api-limits + limits: + cpus: "0.50" + # lab4: 4 одновременные загрузки по 10 MiB; замер пика — evidence/upload_memory*.json + memory: 320M + +x-api-secrets: &api-secrets + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + +services: + postgres: + image: postgres:16.4-alpine + restart: unless-stopped + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + # (API_REPLICAS + 1 при rolling update) x DB_POOL_MAX + migrate + запас < 100 + command: ["postgres", "-c", "max_connections=100"] + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [backend] + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + redis: + image: redis:7.4.1-alpine + restart: unless-stopped + user: redis + # Только кэш проверок сессии: без RDB/AOF, ограничение памяти и LRU. + command: + - sh + - -c + - exec redis-server --requirepass "$$(tr -d '\r\n' < /run/secrets/redis_password)" --save "" --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru + secrets: + - redis_password + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(tr -d '\r\n' < /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: + cpus: "0.25" + memory: 96M + + # lab5: брокер заданий — ОТДЕЛЬНЫЙ от кэша сессий Redis (тот же закреплённый образ). + # Кэш: без persistence и с вытеснением LRU; брокер: AOF с fsync на каждую запись + # (ответ на XADD = сообщение на диске, аналог publisher confirm) и noeviction + # (поток никогда не вытесняется). Volume brokerdata переживает рестарт/пересоздание. + broker: + image: redis:7.4.1-alpine + restart: unless-stopped + user: redis + command: + - sh + - -c + - exec redis-server --requirepass "$$(tr -d '\r\n' < /run/secrets/broker_password)" --dir /data --appendonly yes --appendfsync always --save "" --maxmemory 256mb --maxmemory-policy noeviction + secrets: + - broker_password + volumes: + - brokerdata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(tr -d '\r\n' < /run/secrets/broker_password)\" redis-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [queue] + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 320M + + # lab4: S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 gateway в + # одном процессе `weed server`). Наружу НЕ публикуется. Административные HTTP-порты + # (master 9333, filer 8888, volume 8080) слушают только 127.0.0.1 ВНУТРИ контейнера + # (-ip.bind), в сеть storage выставлен только S3 API (8333). Анонимного доступа нет: + # identities из secrets/s3_config (bootstrap-admin и messenger-api с правами + # Read/Write/List только на свой бакет). Bucket создаёт s3-init. + s3: + image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.50} + restart: unless-stopped + entrypoint: ["/usr/bin/weed"] + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + - -s3.port=8333 + - -s3.ip.bind=0.0.0.0 + - -s3.config=/run/secrets/s3_config + - -s3.iam=false + - -s3.port.iceberg=0 + - -s3.port.lance=0 + - -s3.autoCreateBucket=false + # lab5: без ограничения 8 одновременных PUT по 9.5 MiB (2 реплики x + # MAX_CONCURRENT_UPLOADS=4) доводили weed до лимита 512M -> OOM-kill и + # рестарт S3 (docker events: oom/die/start). Лишние загрузки ждут в S3. + - -s3.concurrentUploadLimitMB=128 + - -master.volumeSizeLimitMB=256 + - -master.electionTimeout=2s + - -master.telemetry=false + - -volume.max=0 + # Go runtime не знает о лимите cgroup и при GOGC=100 копит мусор до ~2x живой + # кучи. Мягкий лимит заставляет GC работать раньше, чем придёт OOM killer. + environment: + GOMEMLIMIT: 400MiB + secrets: + - s3_config + volumes: + - s3data:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [storage] + # Штатная остановка SeaweedFS длится ~20 с; Docker ждёт 10 с и шлёт SIGKILL. + # Подтверждённые записи уже в page cache ОС, том консистентен (см. REPORT.md). + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + # Идемпотентный bootstrap бакета + проверка приватности и прав API-ключей. + # Повторный запуск: "already exists (idempotent no-op)". Admin-ключи есть только здесь. + s3-init: + build: . + command: ["python", "-m", "src.s3_bootstrap"] + restart: "no" + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: + condition: service_healthy + networks: [storage] + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.50" + memory: 192M + + # Однократные миграции до старта реплик. Повтор — no-op (schema_migrations), + # параллельный запуск сериализуется pg_advisory_lock. + migrate: + build: . + command: ["python", "-m", "src.migrate"] + restart: "no" + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + # DDL может идти дольше обычного запроса API + DB_STATEMENT_TIMEOUT_MS: "60000" + MIGRATE_WAIT_SECONDS: "60" + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + networks: [backend] + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + + api: + build: . + image: messenger-api:${APP_VERSION:-lab6} + restart: unless-stopped + environment: *api-env + secrets: *api-secrets + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + # Порт только внутри сети backend — в обход LB API недоступен. + expose: + - "8080" + healthcheck: *api-healthcheck + # SIGTERM -> осушение DRAIN_SECONDS (3 с) -> graceful shutdown (<= 10 с). + stop_signal: SIGTERM + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=16m + networks: [backend, storage, queue, telemetry] + # Остановленный s3 исчезает из Docker DNS; короткий таймаут резолвера. + dns_opt: ["timeout:1", "attempts:1"] + deploy: + replicas: ${API_REPLICAS:-2} + resources: *api-limits + + # lab5: worker — отдельный процесс без HTTP. Берёт задания из broker, пишет + # final/ в S3 и результат в PostgreSQL. Также крутит outbox relay (sweeper + + # сборщик просроченных аренд) и сборщик staging/сирот: публикация не зависит от + # того, живы ли реплики API. Недоступность worker НЕ делает API синхронным. + worker: + # Тот же образ, что у API (один Dockerfile); build здесь, чтобы `up --build` + # не пытался скачать локальный тег из registry. + build: . + image: messenger-api:${APP_VERSION:-lab6} + command: ["python", "-m", "src.worker"] + restart: unless-stopped + environment: + <<: [*queue-env, *otel-env] + OTEL_SERVICE_NAME: messenger-worker + APP_VERSION: ${APP_VERSION:-lab6} + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL_FILE: /run/secrets/database_url + DB_POOL_MIN: "1" + DB_POOL_MAX: ${WORKER_DB_POOL_MAX:-6} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10} + S3_MAX_ATTEMPTS: ${S3_MAX_ATTEMPTS:-2} + S3_CIRCUIT_OPEN_SECONDS: ${S3_CIRCUIT_OPEN_SECONDS:-5} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_IMAGE_TIMEOUT_SECONDS: ${WORKER_IMAGE_TIMEOUT_SECONDS:-30} + WORKER_IMAGE_MEMORY_MB: ${WORKER_IMAGE_MEMORY_MB:-1024} + ATTACHMENT_GC_INTERVAL_SECONDS: ${ATTACHMENT_GC_INTERVAL_SECONDS:-300} + ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900} + ORPHAN_SWEEP_INTERVAL_SECONDS: ${ORPHAN_SWEEP_INTERVAL_SECONDS:-3600} + WORKER_FAULTS: "false" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + # Heartbeat обновляется, только пока все consumer-потоки проходят цикл. + test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 90 else 1)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + # SIGTERM: не брать новые сообщения, дождаться текущих (< JOB_TIMEOUT_SECONDS). + stop_signal: SIGTERM + stop_grace_period: 75s + read_only: true + tmpfs: + # forkserver-сокет и heartbeat; изображения в памяти, не на диске + - /tmp:size=16m + networks: [backend, storage, queue, telemetry] + dns_opt: ["timeout:1", "attempts:1"] + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "1.00" + # 2 подпроцесса обработки (RLIMIT_AS 1024 MiB виртуальной памяти, реальный + # пик 361 MiB на 40 Мп RGBA PNG, замер в REPORT.md) + процесс worker + memory: 1280M + + # Экземпляр с коротким TTL — только для автотеста истечения сессии (профиль ttl). + # Не входит в LB (другое DNS-имя), портов наружу не публикует: тест запускается + # внутри сети командой `make test-expiry` + api-short-ttl: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [ttl] + restart: "no" + environment: + <<: *api-env + SESSION_TTL_SECONDS: ${SHORT_SESSION_TTL_SECONDS:-2} + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: + - "8080" + healthcheck: *api-healthcheck + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=16m + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + # lab4: экземпляры с ВНЕДРЁННЫМИ отказами пути загрузки (профиль faults), только для + # student-tests/test_attachment_consistency.py. Не входят в LB (другие DNS-имена), + # порты не публикуют. Запуск: python scripts/ops.py in-network consistency + api-fault-s3: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: s3_error_on_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + api-fault-db: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: db_error_after_s3_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + # lab5: разрыв COMMIT -> publish. Реплика отвечает 202 (задание и outbox уже + # в PostgreSQL), затем в background task до XADD завершает процесс (os._exit). + # Восстановление: sweeper любой другой реплики/worker публикует строку outbox. + api-fault-publish: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: crash_after_commit + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + # lab5: разрыв publish -> подтверждение в outbox: XADD выполнен, процесс умирает до + # DELETE строки outbox -> повторная публикация (дубль в брокере) -> worker no-op. + api-fault-dup: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: crash_after_publish + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + # Процесс завершается (os._exit) между S3 PUT (staging) и COMMIT — «падение» реплики. + api-fault-crash: + image: messenger-api:${APP_VERSION:-lab6} + profiles: [faults] + restart: "no" + environment: + <<: *api-env + FAULT_INJECTION: crash_after_s3_put + secrets: *api-secrets + depends_on: + api: + condition: service_healthy + expose: ["8080"] + read_only: true + tmpfs: ["/tmp:size=16m"] + networks: [backend, storage, queue, telemetry] + deploy: + resources: *api-limits + + lb: + image: haproxy:2.8.30-alpine + restart: unless-stopped + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + stop_grace_period: 5s + networks: [backend] + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + + # ------------------------------------------------------------------ lab6: наблюдаемость + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1 + restart: unless-stopped + command: ["--config=/etc/otelcol/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + environment: + # Go runtime: мягкий лимит ниже лимита контейнера (memory_limiter — 180 MiB) + GOMEMLIMIT: 200MiB + networks: [telemetry] + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + # Замер: пик RSS 234 MiB при недоступных Tempo+Loki и ~300 RPS (очереди 20 батчей полны). + memory: 384M + + prometheus: + image: prom/prometheus:v2.55.1 + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-lifecycle + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + ports: + # Только чтение состояния alerts/запросов с хоста: http://127.0.0.1:9090/alerts + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 20 + networks: [telemetry] + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + + loki: + image: grafana/loki:3.3.2 + restart: unless-stopped + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + networks: [telemetry] + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + + # Образ Tempo работает от uid 10001, а новый named volume принадлежит root: + # однократно отдаём каталог пользователю tempo (как в примерах Grafana). + tempo-init: + image: grafana/tempo:2.6.1 + user: root + entrypoint: ["chown", "10001:10001", "/var/tempo"] + restart: "no" + volumes: + - tempodata:/var/tempo + network_mode: none + + tempo: + image: grafana/tempo:2.6.1 + restart: unless-stopped + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + depends_on: + tempo-init: + condition: service_completed_successfully + networks: [telemetry] + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + + grafana: + image: grafana/grafana:11.4.0 + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + secrets: + - grafana_admin_password + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + depends_on: + prometheus: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 30 + networks: [telemetry] + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + +volumes: + pgdata: + s3data: + brokerdata: + # lab6: история телеметрии переживает пересоздание контейнеров + promdata: + lokidata: + tempodata: + grafanadata: + +networks: + backend: + # Внутренняя сеть хранилища объектов: без выхода наружу и без LB. + storage: + internal: true + # lab5: сеть брокера — только API (relay), worker и broker. + queue: + internal: true + # lab6: OTLP и хранилища телеметрии. Не internal: Grafana/Prometheus публикуются + # на 127.0.0.1; OTLP-порты Collector и порты Loki/Tempo наружу не публикуются. + telemetry: + +secrets: + postgres_password: + file: ./secrets/postgres_password + redis_password: + file: ./secrets/redis_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url + grafana_admin_password: + file: ./secrets/grafana_admin_password diff --git a/docs/lab3/REPORT.md b/docs/lab3/REPORT.md new file mode 100644 index 0000000..641d022 --- /dev/null +++ b/docs/lab3/REPORT.md @@ -0,0 +1,231 @@ +# Отчёт — лабораторная 3 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0; PostgreSQL 16, Redis 7, HAProxy 2.8 LTS. Документация: Docker Compose (services, deploy, depends_on, healthcheck), HAProxy 2.8 Configuration Manual (server-template, resolvers, http-check, retries/redispatch, http-error), PostgreSQL (INSERT … ON CONFLICT, deferrable constraints, advisory locks), Uvicorn (graceful shutdown). Помощь ИИ (Claude) использовалась при проектировании схемы идемпотентности и drain-фазы, написании скриптов стенда, тестов и формулировке отчёта; решения проверены тестами, приведёнными ниже. + +## Запуск из чистого clone + +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`. Зависимости Python закреплены в `requirements.txt`. Нужны Docker Desktop / Docker Engine с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Вариант оркестрации — **Compose + HAProxy** (Swarm не используется). + +```powershell +$env:PYTHONUTF8 = "1" +# 1. Локальные секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py +copy .env.example .env # Linux/macOS: cp .env.example .env + +# 2. Проверка, сборка и запуск: postgres+redis -> migrate (однократно) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Ожидание готовности: 2 реплики healthy, обе UP в HAProxy, /health/ready через LB = 200 +python scripts/ops.py wait +python scripts/ops.py ps # X-Instance-Id <-> контейнер <-> статус в LB +``` + +Время прогрева: первая сборка образа 1–2 мин; дальше стенд готов примерно через 20–30 с (healthcheck PostgreSQL, миграции, `start_period` API, `rise 2` в HAProxy). + +```powershell +# Публичные проверки (Linux/macOS: make check / make test / make test-persistence ... ) +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd # Linux/macOS: scripts/restart.sh +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd # Linux/macOS: scripts/stop-one.sh +python scripts/ops.py restore # вернуть остановленную реплику; затем повторить failover для другой + +# Собственные проверки уровня 4 +python student-tests/run_all.py --base-url http://localhost:8080 # API, auth, идемпотентность через реплики, race +python scripts/ops.py in-network storage # тесты с прямым доступом к БД (внутри compose-сети) +python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl, TTL 2 с) +python student-tests/failover_series.py --output evidence/failover_series.json # каждая реплика по очереди, SIGTERM +python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json # то же, SIGKILL +python student-tests/rolling_update_load.py --output evidence/rolling_update.json # rolling update под нагрузкой +python tests/load.py --duration 30 --concurrency 8 --output evidence/load.json + +# Всё перечисленное одной командой с записью результатов в evidence/ (~8–10 мин): +python scripts/collect_evidence.py +``` + +**Rolling update.** Задать новый тег `APP_VERSION` (например, `lab3.1`) в `.env` или в окружении и выполнить `python scripts/ops.py rolling-update`: сборка образа, однократные миграции (`docker compose run --rm migrate`), затем для каждой старой реплики: +1 новая → healthy и UP в HAProxy → graceful stop старой → удаление. + +**Остановка.** Безопасная, данные сохраняются: `docker compose --env-file .env --profile ttl down` (volume `pgdata` остаётся). Намеренное удаление всех учебных данных — только вручную: `docker compose --env-file .env --profile ttl down -v`. Hooks и тесты никогда не удаляют volumes. + +**Действия оператора:** создать секреты до первого запуска; пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` не перезаписывает существующие файлы. После `stop-one`/тестов отказа вернуть реплику `python scripts/ops.py restore`. + +## Архитектура и решения + +``` + 127.0.0.1:8080 (единственная публичная точка входа API) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки GET /health/ready + │ 127.0.0.1:8404 — страница состояния (UP/DOWN реплик), только чтение + │ обнаружение реплик: Docker DNS имени `api` -> server-template api1..api6 + ▼ + ┌───────── api (deploy.replicas = 2, без ports, без container_name) ─────────┐ + │ api-1: X-Instance-Id = hostname = короткий ID контейнера 0.5 CPU / 256M │ + │ api-2: ... 0.5 CPU / 256M │ + └──────┬──────────────────────────────────────────────────────────┬──────────┘ + ▼ ▼ + postgres 16 (1 CPU / 512M, volume pgdata) redis 7 (0.25 CPU / 96M) + источник истины: users, chats, messages, общий кэш проверок токена, + sessions, idempotency_keys, schema_migrations без persistence + ▲ + migrate (one-shot): python -m src.migrate — до старта реплик, один раз +``` + +Порядок старта задаётся `depends_on`: `postgres`, `redis` (healthy) → `migrate` (`service_completed_successfully`) → `api` (healthy) → `lb`. API-реплики не публикуют порты: `expose: 8080` только внутри сети `backend`. Наружу (на 127.0.0.1) опубликован только `lb`. + +### Почему HAProxy, а не Nginx + +В open-source Nginx нет активных health checks: он узнаёт об отказе только по ошибкам живых запросов (`max_fails`). HAProxy сам опрашивает `/health/ready` с заданным периодом. Поэтому время обнаружения отказа можно указать числом, а фаза осушения при SIGTERM действительно выводит реплику из ротации до остановки. `server-template` + `resolvers` синхронизируют список реплик с DNS-записями сервиса `api` каждые 2 с. Масштабирование и rolling update поэтому не требуют перезапуска или перечитывания конфига LB. Sticky sessions не используются (`balance roundrobin`, без cookie). + +### Что перенесено из lab2 и что изменено + +Перенесены без изменения семантики: модели и валидация, формат ошибок, keyset-пагинация, Argon2id, Basic → Bearer, сессии в PostgreSQL + кэш в Redis с надёжной инвалидацией (`session_cache_invalidations`), собственные тесты lab2. Схема lab2 уже рассчитана на несколько реплик: сессии лежат в общей БД, кэш — в общем Redis, а долг инвалидации перечитывается всеми репликами не реже раза в секунду. + +Изменено в lab3: + +1. **Переход 2 → 3 по контракту:** `X-Instance-Id` во всех ответах API, включая ошибки и health (middleware + `error_response`). Обязательный `Idempotency-Key` для `POST /chats/{id}/messages`. +2. **Общая идемпотентность** — таблица `idempotency_keys` (миграция `003`), подробности ниже. +3. **Миграции вынесены из реплик** в one-shot сервис `migrate` (`RUN_MIGRATIONS_ON_START=false`). Реплика готова только когда в `schema_migrations` есть последняя версия, которую знает её образ. +4. **Graceful shutdown с осушением** (`src/server.py`): по SIGTERM реплика сразу отвечает 503 на readiness, но ещё 3 с обслуживает запросы, затем штатно останавливается. +5. **Лимит одновременных хешей Argon2id** (`PASSWORD_HASH_CONCURRENCY=4`). У реплики лимит 256M, а threadpool FastAPI — до 40 потоков; без лимита всплеск входов мог бы занять 40 × 19 MiB и привести к OOM-kill. +6. **Readiness** использует отдельный короткий таймаут БД (`READY_DB_TIMEOUT_SECONDS=1`) меньше `timeout check` HAProxy. +7. Тесты lab2 адаптированы к переходу: при отправке сообщения добавляется `Idempotency-Key`, имя проекта сменилось на `messenger-lab3`. Тест истечения сессии и тесты хранилища теперь идут внутри compose-сети, без публикации портов в обход LB. + +### Идемпотентность отправки сообщения (`src/storage.py`, `migrations/003_idempotency_keys.sql`) + +- Ключ: `PRIMARY KEY (user_id, method, chat_id, idem_key)` — ровно область из контракта «пользователь + метод + путь чата + ключ». В строке хранятся SHA-256 канонического JSON тела (`sort_keys`, без пробелов: порядок ключей и форматирование не важны), `message_id`, `expires_at = now + IDEMPOTENCY_TTL_SECONDS (86400)`. +- **Одна транзакция READ COMMITTED:** + 1. `INSERT INTO idempotency_keys … ON CONFLICT DO UPDATE … WHERE expires_at <= now() RETURNING`. Конкурентная транзакция с тем же ключом **ждёт на уникальном индексе**, пока первая не сделает COMMIT/ROLLBACK. Это и даёт «ровно одно сообщение» при одновременных повторах через разные реплики: арбитр — PostgreSQL, а не память процесса. + 2. Ключ наш → `INSERT messages … WHERE EXISTS (членство)` с заранее выбранным id. FK `message_id → messages` объявлен `DEFERRABLE INITIALLY DEFERRED` и проверяется при COMMIT, поэтому `message_id NOT NULL` с первого шага. + 3. Ключ уже подтверждён → снова проверяется членство (и сессия — dependency `current_user`). Hash совпал → 201 и то же сообщение (+ информационный заголовок `Idempotency-Replayed: true`); не совпал → 409. +- Любая ошибка или обрыв до COMMIT откатывает и ключ, и сообщение: транзиентный 5xx не фиксируется. 404 и 400 ключ не «сжигают». Истёкший, но ещё не удалённый ключ можно занять заново. Фоновая очистка раз в 5 мин удаляет истёкшие ключи пачками под `pg_try_advisory_xact_lock`, то есть одной репликой за раз. +- Порядок проверок: аутентификация → ключ (нет/неверный → 400) → тело → доступ к чату (404). + +### Readiness, таймауты и остановка — согласованные значения + +| Параметр | Значение | Почему | +| --- | --- | --- | +| HAProxy `inter` / `fastinter` / `fall` / `rise` | 1 с / 0.5 с / 2 / 2 | DOWN после 2 неудачных проверок подряд, вторая — через 0.5 с | +| HAProxy `timeout check` | 1.5 с | больше `READY_DB_TIMEOUT_SECONDS` = 1 с | +| HAProxy `timeout connect`, `retries`, `option redispatch 1` | 1 с, 2, да | несостоявшееся соединение повторяется на другой реплике; запрос ещё не отправлен — безопасно и для POST | +| HAProxy `timeout server` | 10 с | > максимум запроса API: пул 3 с + `statement_timeout` 5 с | +| `resolvers` hold valid/obsolete | 2 с / 2 с | новые/исчезнувшие контейнеры попадают в LB за ≤ ~2–3 с | +| API `DRAIN_SECONDS` | 3 с | > худшее обнаружение DOWN при ответах 503 (≈ 1 + 0.5 с + задержка проверок) | +| API `GRACEFUL_TIMEOUT_SECONDS` | 10 с | дождаться запросов в обработке (≤ 8 с) | +| compose `stop_grace_period` (api) | 20 с | > 3 + 10 — SIGKILL не прилетает во время graceful stop | +| Docker healthcheck api | 3 с / timeout 2 с / retries 2 / start_period 20 с | для `depends_on` и `ops.py` (rolling update ждёт healthy) | +| Uvicorn keep-alive | 30 с | дольше, чем LB держит простаивающие соединения к серверу | + +**Максимальное время обнаружения отказа балансировщиком:** graceful stop (сразу 503) и падение процесса (соединение отклонено) — до ≈ 1.5–2 с; зависшая реплика без ответа — до 1 + 1.5 + 0.5 + 1.5 = **4.5 с**. Пока реплика не помечена DOWN, запрос к ней, не сумевший соединиться за 1 с, уходит на другую. **Измерено** (`evidence/failover_series*.json`, `evidence/shutdown_timeline.txt`): DOWN через 0.96–1.41 с после SIGTERM и через 1.10–1.28 с после SIGKILL. + +**Временная шкала graceful stop** (`evidence/shutdown_timeline.txt`): SIGTERM → +0.01 с readiness 503 → +1.30 с HAProxy DOWN (новые запросы больше не приходят) → +3.02 с закрыт listening-сокет → +3.12 с процесс завершён. LB выводит реплику из ротации на ~1.7 с раньше, чем она перестаёт принимать соединения, поэтому graceful stop не даёт ошибок клиенту. + +**Критичные зависимости readiness:** PostgreSQL (и наличие мигрированной схемы). Redis не критичен — это кэш с обходом (circuit breaker 5 с), как в lab2. Если ни одна реплика не готова, LB сам отвечает `503` в общем JSON-формате ошибок с тем же `X-Request-Id` (`http-error`, `evidence/lb_errors.txt`). `X-Instance-Id` в таком ответе нет — контракт это допускает. + +### Миграции и совместимость с работающей версией + +- `migrate` выполняется **один раз** до старта реплик. Повтор — no-op (`schema_migrations`); две одновременные копии сериализуются `pg_advisory_lock` (`evidence/migrations.txt`: копия B применила 3 миграции, копия A — `nothing applied`). Реплики миграции не запускают и не готовы, пока схема не мигрирована: на пустой БД readiness = 503 `Database schema is not migrated`, после `migrate` — 200 без рестарта. +- Rolling update применяет миграции **до** замены реплик, поэтому допускаются только **расширяющие** (expand) миграции, совместимые со старой версией. `003` только добавляет таблицу. Проверено: код lab2 (старая версия) проходит smoke lab2 на схеме lab3 одновременно с репликами lab3 (`evidence/migrations.txt`). +- Ограничение: смешанный парк «lab2 + lab3» во время обновления формально совместим по схеме, но реплики lab2 не требуют и не записывают `Idempotency-Key`. Поэтому гарантия идемпотентности действует только когда все реплики уже lab3. Удаляющие/переименовывающие миграции нужно делить на expand → переключение кода → contract отдельным релизом. + +### Соответствие `.env.example` и реальной конфигурации + +| Переменная | Где используется | +| --- | --- | +| `APP_HOST`, `APP_PORT` | внутри контейнера Uvicorn слушает `0.0.0.0:8080`; `APP_PORT` — публикуемый порт LB `127.0.0.1:${APP_PORT}` | +| `LB_STATS_PORT` | страница состояния HAProxy `127.0.0.1:8404` (`ops.py`, тесты отказов) | +| `APP_ENV`, `APP_VERSION` | `APP_VERSION` — тег образа `messenger-api:${APP_VERSION}` (rolling update) и строка в логе старта | +| `SESSION_TTL_SECONDS`, `SHORT_SESSION_TTL_SECONDS` | TTL сессий; второй — для `api-short-ttl` (профиль `ttl`) | +| `DATABASE_URL_FILE`, `CACHE_URL_FILE` | Compose secrets `/run/secrets/database_url`, `/run/secrets/cache_url` | +| `SESSION_STORE` | `postgres` — источник истины сессий; Redis — общий кэш | +| `SESSION_CACHE_TTL_SECONDS` | TTL записи кэша и «надгробия» logout | +| `API_REPLICAS` | `deploy.replicas` сервиса `api` (2); ожидаемое число реплик в `ops.py` | +| `IDEMPOTENCY_TTL_SECONDS` | срок хранения результата ключа (≥ 86400, иначе API не стартует) | +| `DB_POOL_*`, `DB_*_TIMEOUT*` | пул (1..10 на реплику), ожидание соединения 3 с, connect 3 с, statement 5 с | +| `CACHE_TIMEOUT_SECONDS`, `CACHE_RETRY_AFTER_SECONDS` | таймаут Redis 250 мс и обход кэша на 5 с после ошибки | +| `READY_DB_TIMEOUT_SECONDS`, `DRAIN_SECONDS`, `GRACEFUL_TIMEOUT_SECONDS`, `KEEP_ALIVE_SECONDS` | readiness и остановка — таблица выше | +| `PASSWORD_HASH_CONCURRENCY` | одновременных хешей Argon2id на реплику | + +`max_connections=100` у PostgreSQL ≥ (2 реплики + 1 во время rolling update) × 10 + migrate + api-short-ttl + запас. + +### Инфраструктура + +| Роль | Service | Версия | Health/readiness | Volume | Сеть/порт | Credentials | Лимиты | +| --- | --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | `haproxy:2.8.30-alpine` | `wget /lb-health` (monitor-uri) | — (конфиг ro bind) | `backend`; **127.0.0.1:8080**, 127.0.0.1:8404 | — | 0.5 CPU / 128M | +| api × 2 | `api` | сборка `Dockerfile`, `python:3.12.7-slim-bookworm` | `/health/ready` (БД + схема + не в drain) | нет, `read_only` + tmpfs `/tmp` | `backend`, `expose 8080` (наружу нет) | secrets `database_url`, `cache_url` | 0.5 CPU / 256M | +| миграции | `migrate` | тот же Dockerfile | one-shot, exit 0 | — | `backend` | secret `database_url` | 0.5 CPU / 256M | +| postgres | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `pgdata` | `backend`, наружу нет | secret `postgres_password` | 1 CPU / 512M | +| redis | `redis` | `redis:7.4.1-alpine`, user `redis` | `redis-cli ping` | нет (кэш) | `backend`, наружу нет | secret `redis_password` | 0.25 CPU / 96M | +| тест TTL | `api-short-ttl` (профиль `ttl`) | образ api | `/health/ready` | — | `backend`, не в LB, портов нет | как api | 0.5 CPU / 256M | + +`container_name` нигде не задан. Секреты общие только там, где нужны: пароль PostgreSQL есть только у `postgres`, пароль Redis — у `redis`, строки подключения — у `api`/`migrate`. Bootstrap: схема создаётся сервисом `migrate` на пустом volume; повтор безопасен. Hooks: `scripts/restart.sh|.cmd` → `ops.py restart` (stop lb+api → restart postgres+redis → start api → healthy → start lb); `scripts/stop-one.sh|.cmd` → `ops.py stop-one` (одна реплика по `TARGET_INSTANCE_ID`, остаётся остановленной). Каждое действие пишется в `evidence/ops.log`. + +## Доказательства критериев + +**Стенд, на котором сняты приложенные результаты.** Windows 10, Ryzen 5 7500f, 32 GB RAM. PostgreSQL 16, Redis 7 и HAProxy **2.8.30**, Docker Desktop. Код приложения, миграции и **тот же `infra/haproxy/haproxy.cfg`** использованы как есть. Только `server-template` + DNS заменены на 6 статических слотов, реплики запущены как `python -m src.server` (2 процесса), CPU/RAM-лимитов нет. Compose-файл проверен `docker compose config`, конфиг HAProxy — `haproxy -c` той же версии 2.8.30, что в образе. `scripts/ops.py` прогнан против заглушки `docker`. + +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab3 public API scenario` | `evidence/smoke.txt` | +| Контракт | `make check`, `make validate`, `docker compose config --quiet`, `haproxy -c` | все PASS / valid | `evidence/check.txt` | +| Стенд в `compose.yaml`, ≥ 2 реплики, одна точка входа, общая БД и хранилище сессий | `compose.yaml`; `python scripts/ops.py ps` | `api` × 2 без `ports`; публикуется только `lb`; одна `postgres`, один `redis` | `compose.yaml`, `evidence/stand.txt` (после `collect_evidence.py`) | +| Разные запросы одного пользователя — разными репликами без повторного входа | 20 × `GET /users/me` с одним токеном через LB | чередование двух `X-Instance-Id`, все 200 | `evidence/instances.txt` | +| Остановка любой одной реплики: ≤ 10 с, чтение/запись продолжаются, подтверждённые доступны | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` × 2 (вторая реплика после восстановления первой) + SIGKILL | PASS ×3; `failed_attempts: 0`; запись не прерывалась (первая запись после возврата hook — 3.3 с = длительность graceful hook; при SIGKILL — 0.11 с); остановлены разные реплики | `evidence/failover.txt` | +| Какой процесс реально остановлен | журнал hook | `SIGTERM nat-api-4` в прогоне 1, `SIGTERM nat-api-3` в прогоне 2 (в compose — `evidence/ops.log`: имя контейнера, IP, состояние после) | `evidence/failover.txt` | +| Повтор POST с тем же ключом не создаёт дубль, в т.ч. конкурентно и при смене реплики | smoke; `test_idempotency.py` (24 параллельных POST, Barrier); failover-тест повторяет неопределённые POST тем же ключом | 24 × 201, один `id`, ответы через **обе** реплики, в истории 1 сообщение; дублей нет ни в одном сценарии отказа | `evidence/student-tests.txt`, `evidence/failover*.txt` | +| Persistence: рестарт стенда с volumes и тем же токеном | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; PID до/после: postgres 1469→2901, redis 1479→2911, api 2846→2917 и 2866→2925, HAProxy перезапущен | `evidence/persistence.txt` | +| Миграции отдельно и согласованно | сервис `migrate`; readiness проверяет схему | реплики не мигрируют; повтор no-op; гонка двух `migrate` безопасна | `evidence/migrations.txt` | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Health/readiness, timeouts и SIGTERM согласованы; LB исключает недоступный backend; период проверок и макс. время обнаружения | `infra/haproxy/haproxy.cfg`, `src/server.py`; замер шкалы остановки | проверки каждые 1 с (0.5 с после первой неудачи), fall 2; обнаружение ≤ 1.5–2 с (503/RST), ≤ 4.5 с (зависание). Замер: 503 на readiness через 0.01 с, DOWN через 1.30 с, приём прекращён через 3.02 с | таблица выше, `evidence/shutdown_timeline.txt`, `evidence/lb_log_sample.txt` | +| Последовательно остановить каждую реплику (восстанавливая предыдущую); RPS, ошибки, время восстановления | `failover_series.py` (4 писателя + 2 читателя, timeout клиента 2 с) | **SIGTERM:** 0 ошибок; DOWN в LB через 1.41 / 0.96 с; RPS 407 → 324 (одна реплика) → 354 после restore; p95 30–31 мс; реплика снова в ротации через 3.0 / 2.6 с после restore. **SIGKILL:** 6 / 3 оборванных запроса (502) в окне ≤ 0.21 с после kill, все записи повторены тем же ключом без дублей; DOWN через 1.28 / 1.10 с | `evidence/failover_series.json`, `evidence/failover_series_kill.json`, `evidence/failover_series.txt` | +| Общая идемпотентность параллельными запросами с одним ключом | `test_idempotency.py`, `test_storage_idempotency.py` | 24 параллельных → 1 сообщение через 2 реплики; 12 разных тел с одним ключом → один победитель, остальные 409; 16 потоков на уровне хранилища → одна вставка; сбой после вставки ключа и сообщения → ни ключа, ни сообщения, повтор успешен | `evidence/student-tests.txt` | +| CPU/RAM-лимиты; API без портов в обход LB; секреты/хранилища общие только где нужно; нет `container_name` | `compose.yaml` | лимиты у всех сервисов (api 0.5 CPU / 256M); у `api` только `expose`; `container_name` отсутствует; RSS реплики под нагрузкой 132 MB < 256M | `compose.yaml`, `evidence/load.txt` | +| Rolling update под нагрузкой | `rolling_update_load.py` → `ops.py rolling-update` (на нативном стенде — тот же алгоритм) | замена 2 реплик за 11.8 с, **0 ошибок**, RPS 388 → 381 во время → 385 после, p95 31 → 34 мс; после — только новые реплики; 5709 подтверждённых сообщений на месте, дублей нет | `evidence/rolling_update.json`, `evidence/rolling_update.txt` | +| Миграции однократно и совместимы с работающей версией | сервис `migrate`; код lab2 на схеме lab3 | см. раздел «Миграции»; ограничение смешанного парка описано | `evidence/migrations.txt` | +| Возможности lab2 сохранены | `run_all.py` (47 тестов + race), `test_expiry` на экземпляре с TTL 2 с | все PASS (1 skip — expiry в общем прогоне, запускается отдельно и проходит) | `evidence/student-tests.txt` | + +## Эксперименты и отказы + +**Гипотеза.** Если всё состояние, от которого зависит корректность (данные, сессии, ключи идемпотентности), лежит в общем PostgreSQL, а LB активно проверяет готовность и реплика по SIGTERM сначала выходит из ротации, то: (1) graceful-остановка любой реплики и rolling update проходят без ошибок для клиента; (2) падение реплики даёт только кратковременные ошибки запросов, которые были в обработке, и клиент безопасно повторяет их с тем же ключом; (3) дублей и потерь подтверждённых сообщений нет. Подтверждено измерениями ниже. + +**Методика.** Генератор `student-tests/loadgen.py`: 4 писателя (POST с `Idempotency-Key`; при обрыве, timeout или 5xx — повтор с тем же ключом и телом до 201) и 2 читателя (история, `/users/me`); timeout клиента 2 с; опрос страницы HAProxy каждые 100 мс. Каждый раунд: 5 с прогрева → действие → 15 с без реплики → restore → 3 с. После раунда — полный обход истории и сверка со всеми подтверждёнными ответами. Стенд — см. «Доказательства»: клиент на той же машине, без лимитов. Повторов для статистики нет: разброс RPS между раундами ±5–8 %. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| SIGTERM реплики 1 под нагрузкой | 0 ошибок, LB выводит до остановки | 0 ошибок; DOWN через 1.41 с; реплика отвечала ещё 1.34 с (до DOWN) | hook 3.3 с; RPS 407 → 324 → 354; возврат в ротацию 3.0 с после restore; потерь нет | +| SIGTERM реплики 2 | то же | 0 ошибок; DOWN через 0.96 с | hook 3.4 с; RPS 393 → 304 → 348; возврат 2.6 с; потерь нет | +| SIGKILL реплики 1 | ошибки только у запросов в обработке, повтор без дублей | 6 × 502 в окне 0.21 с, 4 повтора записи тем же ключом; DOWN через 1.28 с | новые запросы успешны сразу (redispatch); возврат 2.8 с; дублей нет | +| SIGKILL реплики 2 | то же | 3 × 502 в окне 0.14 с, 1 повтор записи; DOWN через 1.10 с | возврат 2.7 с; дублей нет | +| Публичный failover (×2 graceful, ×1 kill) | PASS, ≤ 10 с | PASS; 0 failed attempts | первая запись после hook: 3.3 с / 3.4 с / 0.11 с | +| Rolling update 2 реплик под нагрузкой | 0 ошибок, после — только новые | 0 ошибок; `[api1,api2] → [api1,api2,api3] → [api2,api3] → … → [api3,api4]` | 11.8 с; RPS не просел (388 / 381 / 385) | +| Рестарт API + PostgreSQL + Redis + LB | данные и сессия сохраняются | PASS, тот же токен | потерь нет | +| Все реплики недоступны | LB отвечает 503 в формате контракта | JSON 503 с `request_id` = `X-Request-Id` | — | + +**Нагрузка** (`tests/load.py`, closed-loop POST, без retry): 30 с, concurrency 8 → 10 955 успешных, 0 ошибок, **364 RPS**, p50 19.9 мс, p95 42.8 мс, p99 53.4 мс (только успешные); набор данных — один чат, ~11 тыс. новых сообщений. RSS каждой реплики 132 MB, 14 потоков. Во время отказа одной реплики RPS падает примерно на 20 %: при 2 vCPU на машину одна реплика (один процесс Uvicorn) упирается в CPU. Это пропускная способность HTTP-подтверждений в данном профиле, а не максимум системы; с лимитами 0.5 CPU на реплику в Docker цифры будут ниже. + +## Ограничения и следующие шаги + +- **Отказы, которые выдерживаются:** остановка или падение любой одной API-реплики, rolling update реплик, рестарт всего стенда с volumes. **Не выдерживаются:** отказ `lb`, `postgres` (единственная копия, без репликации) и Docker-host — это SPOF. Отказ `redis` не влияет на корректность (обход кэша, как в lab2). +- При падении (SIGKILL) запросы, которые реплика обрабатывала в этот момент, завершаются ошибкой 502. LB не повторяет уже отправленные POST — по контракту это делает клиент с тем же `Idempotency-Key`. Для `POST /users` и `POST /chats` повтор после неопределённого результата может создать второй объект (контракт этого не запрещает и прозрачных повторов не требует). +- HAProxy держит до 6 слотов реплик (`server-template api 1-6`). Больше — поменять число в конфиге. +- Страница состояния HAProxy на 127.0.0.1:8404 без аутентификации, только чтение и только localhost. +- Приложенные результаты сняты на нативном эквиваленте стенда (см. «Доказательства»), без CPU/RAM-лимитов и без Docker DNS. Перед защитой их нужно повторить `python scripts/collect_evidence.py` на Docker; конкретные цифры на ноутбуке будут другими. +- Время обнаружения на Docker дополнительно включает DNS-синхронизацию (до ~2 с на появление новой реплики в LB). На отказ это не влияет: остановленную реплику исключают активные проверки раньше, чем DNS. +- Идемпотентность гарантирована, только когда все реплики — lab3 (см. «Миграции»). +- HTTP без TLS — допустимо только на локальном учебном стенде. Пароль Redis передаётся `redis-server` аргументом внутри контейнера (как в lab2). + +## Вопросы защиты — кратко + +- **Отказ процесса vs отказ узла.** Процесс (контейнер реплики) падает, остальные реплики на том же host живы, общие PostgreSQL/Redis/LB целы — это и проверяет лаба. При отказе узла пропадает всё, что на нём: здесь это весь стенд, включая LB и единственную БД. Защита от него — несколько host, репликация БД, LB с переносом адреса (трек 2 уровня 5, не заявлен). +- **Почему depends_on/healthcheck — не балансировщик.** `depends_on` управляет только порядком старта, а Docker healthcheck лишь меняет статус контейнера. Ни один из них не перенаправляет трафик и не выводит больную реплику из ротации. Это делает LB своими проверками (здесь — HAProxy `http-check` каждые 1 с). +- **Какие POST можно повторять после timeout.** Только те, что защищены ключом идемпотентности: здесь `POST /chats/{id}/messages` с тем же `Idempotency-Key` и тем же телом. Ключ защищает от дубля, если ответ потерялся после COMMIT: повтор вернёт исходное сообщение, а не создаст второе. `POST /users`, `POST /chats`, `POST /auth/sessions` после неопределённого результата повторять «вслепую» нельзя — создастся новый объект или новая сессия. +- **Сессии в локальной памяти при росте числа реплик.** Токен, выданный репликой A, неизвестен реплике B: каждый второй запрос получит 401, logout на одной реплике не отзовёт сессию на другой, рестарт реплики разлогинит её пользователей. «Чинить» это sticky sessions — значит терять сессии при отказе реплики. Поэтому сессии лежат в общем PostgreSQL (кэш — общий Redis), а ключи идемпотентности — в общей БД. diff --git a/docs/lab3/evidence/README.md b/docs/lab3/evidence/README.md new file mode 100644 index 0000000..d074e56 --- /dev/null +++ b/docs/lab3/evidence/README.md @@ -0,0 +1,25 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Файлы этой работы (lab3) + +`python scripts/collect_evidence.py` на compose-стенде перезаписывает их результатами Docker; `SUMMARY.txt` — итог по шагам. + +| Файл | Что доказывает | +| --- | --- | +| `check.txt` | make check / validate, `docker compose config`, `haproxy -c` | +| `smoke.txt` | публичный smoke lab3 через LB | +| `instances.txt` | одна сессия обслуживается разными репликами (X-Instance-Id) | +| `student-tests.txt` | собственные тесты: lab2 + идемпотентность через реплики + хранилище + expiry | +| `failover.txt` | публичный failover для обеих реплик и SIGKILL; какой процесс/контейнер остановлен | +| `failover_series*.json/.txt` | уровень 4: каждая реплика по очереди под нагрузкой — RPS, ошибки, обнаружение, восстановление | +| `rolling_update*.json/.txt` | уровень 4: rolling update под нагрузкой | +| `persistence.txt` | рестарт API + PostgreSQL + Redis + LB с тем же токеном | +| `migrations.txt` | миграции однократно, readiness ждёт схему, совместимость со старой версией | +| `shutdown_timeline.txt` | SIGTERM → readiness 503 → DOWN в LB → выход процесса | +| `lb_errors.txt`, `lb_log_sample.txt` | ответы и журнал балансировщика | +| `load.json`, `load.txt` | нагрузка 30 с, RSS реплик | +| `ops.log` | журнал scripts/ops.py (реальные контейнеры — после прогона на Docker) | diff --git a/docs/lab3/evidence/SUMMARY.txt b/docs/lab3/evidence/SUMMARY.txt new file mode 100644 index 0000000..0a23b4e --- /dev/null +++ b/docs/lab3/evidence/SUMMARY.txt @@ -0,0 +1,34 @@ +# 2026-10-26 19:35:57 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Rolling update lab3 -> lab3.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab3/evidence/check.txt b/docs/lab3/evidence/check.txt new file mode 100644 index 0000000..a01561a --- /dev/null +++ b/docs/lab3/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-10-26 19:29:57 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-10-26 19:29:57 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab3/evidence/failover.txt b/docs/lab3/evidence/failover.txt new file mode 100644 index 0000000..6375f76 --- /dev/null +++ b/docs/lab3/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-10-26 19:30:28 UTC | Failover #1: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP +# 2026-10-26 19:30:44 UTC | Failover #1: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 4.672} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:30:44 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:30:51 UTC | Failover #1: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:30:51.183Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 6.3s + +# 2026-10-26 19:30:51 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:07 UTC | Failover #2: make test-failover | exit 0 | 16.2s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.094} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:07 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:31:13 UTC | Failover #2: restore | exit 0 | 6.0s +$ python scripts/ops.py restore +2026-10-26T19:31:13.832Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 5.9s + +# 2026-10-26 19:31:14 UTC | Failover #3: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:30 UTC | Failover #3: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.766} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:30 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 exited unhealthy messenger-api:lab3 - - +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:37 UTC | Failover #3: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:31:37.160Z restore: started ['messenger-lab3-api-1']; 2 replicas healthy and UP in LB after 6.3s + +# остановленные публичным тестом реплики: ['a21409fffeec', 'a21409fffeec', 'c4c5ae5b0949'] diff --git a/docs/lab3/evidence/failover_series.json b/docs/lab3/evidence/failover_series.json new file mode 100644 index 0000000..571c5c3 --- /dev/null +++ b/docs/lab3/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.09, + "lb_detect_down_seconds": 1.77, + "target_last_served_after_stop_seconds": 1.67, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.7, + "lb_back_up_seconds": 1.39, + "target_back_in_rotation_seconds": 6.69, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1322, + "ok": 1322, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 264.4, + "write_ok_rps": 159.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1059, + "ok": 1059, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 105.9, + "write_ok_rps": 64.0, + "latency_ms_ok": { + "p50": 47.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 53.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.0, + "requests": 1618, + "ok": 1618, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 161.8, + "write_ok_rps": 96.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.77, + [ + "api1" + ] + ], + [ + 21.39, + [ + "api1", + "api2" + ] + ], + [ + 21.77, + [ + "api1" + ] + ], + [ + 24.28, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2674, + "stored_messages": 2674, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.03, + "lb_detect_down_seconds": 0.97, + "target_last_served_after_stop_seconds": 0.91, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.56, + "lb_back_up_seconds": 0.67, + "target_back_in_rotation_seconds": 6.56, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1353, + "ok": 1353, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 269.7, + "write_ok_rps": 158.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 990, + "ok": 990, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 99.0, + "write_ok_rps": 59.2, + "latency_ms_ok": { + "p50": 62.0, + "p95": 156.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.03, + "requests": 410, + "ok": 410, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 81.8, + "write_ok_rps": 49.1, + "latency_ms_ok": { + "p50": 93.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.03, + "to_s": 29.86, + "requests": 1602, + "ok": 1601, + "errors": 1, + "error_ratio": 0.0006, + "ok_rps": 162.9, + "write_ok_rps": 98.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 5.98, + [ + "api2" + ] + ], + [ + 20.7, + [ + "api1", + "api2" + ] + ], + [ + 20.94, + [ + "api2" + ] + ], + [ + 23.47, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2604, + "stored_messages": 2604, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/failover_series.txt b/docs/lab3/evidence/failover_series.txt new file mode 100644 index 0000000..e4f0e74 --- /dev/null +++ b/docs/lab3/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-10-26 19:32:39 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 61.9s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=stop) under load + hook 4.09s | LB DOWN after 1.77 s | errors 0 (-) | RPS before 264.4, outage 105.9, after restore 161.8 | p95 outage 110.0 ms | back in rotation 6.69 s | data ok=True +-- stopping c4c5ae5b0949 (mode=stop) under load + hook 4.03s | LB DOWN after 0.97 s | errors 0 (-) | RPS before 269.7, outage 99.0, after restore 162.9 | p95 outage 156.0 ms | back in rotation 6.56 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-10-26 19:33:42 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 63.1s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=kill) under load + hook 1.01s | LB DOWN after 2.53 s | errors 0 (-) | RPS before 235.6, outage 76.1, after restore 146.5 | p95 outage 187.0 ms | back in rotation 7.06 s | data ok=True +-- stopping c4c5ae5b0949 (mode=kill) under load + hook 0.92s | LB DOWN after 2.38 s | errors 0 (-) | RPS before 270.1, outage 78.3, after restore 149.7 | p95 outage 187.0 ms | back in rotation 7.19 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab3/evidence/failover_series_kill.json b/docs/lab3/evidence/failover_series_kill.json new file mode 100644 index 0000000..f54d5b8 --- /dev/null +++ b/docs/lab3/evidence/failover_series_kill.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.01, + "lb_detect_down_seconds": 2.53, + "target_last_served_after_stop_seconds": 0.5, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.09, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1178, + "ok": 1178, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 235.6, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 761, + "ok": 761, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 76.1, + "write_ok_rps": 46.0, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.01, + "requests": 454, + "ok": 454, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 90.5, + "write_ok_rps": 53.8, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 30.44, + "requests": 1527, + "ok": 1527, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 146.5, + "write_ok_rps": 85.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.53, + [ + "api1" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.94, + [ + "api1" + ] + ], + [ + 24.48, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2318, + "stored_messages": 2318, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 0.92, + "lb_detect_down_seconds": 2.38, + "target_last_served_after_stop_seconds": 0.44, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.25, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.19, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1355, + "ok": 1355, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 270.1, + "write_ok_rps": 156.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 783, + "ok": 783, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 78.3, + "write_ok_rps": 47.5, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1016.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 52.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 125.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 30.56, + "requests": 1579, + "ok": 1579, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 149.7, + "write_ok_rps": 89.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.39, + [ + "api2" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.58, + [ + "api2" + ] + ], + [ + 24.11, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2474, + "stored_messages": 2474, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/instances.txt b/docs/lab3/evidence/instances.txt new file mode 100644 index 0000000..788d114 --- /dev/null +++ b/docs/lab3/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-10-26 19:29:59 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C08C-1F90-6ADFAA37167-088A-0008 + 2 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C09C-1F90-6ADFAA37189-088B-0008 + 3 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A0-1F90-6ADFAA371A4-088C-0008 + 4 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0A4-1F90-6ADFAA371B1-088D-0008 + 5 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A6-1F90-6ADFAA371B5-088E-0008 + 6 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0AA-1F90-6ADFAA371B8-088F-0008 + 7 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0B2-1F90-6ADFAA371D0-0890-0008 + 8 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0BE-1F90-6ADFAA371E0-0891-0008 + 9 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0C6-1F90-6ADFAA371E3-0892-0008 +10 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0CE-1F90-6ADFAA371FE-0893-0008 +11 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0D6-1F90-6ADFAA37202-0894-0008 +12 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0E4-1F90-6ADFAA3721D-0895-0008 +13 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0EC-1F90-6ADFAA37220-0896-0008 +14 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0FC-1F90-6ADFAA3723C-0897-0008 +15 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C106-1F90-6ADFAA3724B-0898-0008 +16 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C10E-1F90-6ADFAA3724D-0899-0008 +17 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C11A-1F90-6ADFAA37268-089A-0008 +18 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C126-1F90-6ADFAA3726B-089C-0008 +19 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C130-1F90-6ADFAA37286-089D-0008 +20 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C140-1F90-6ADFAA3728A-089E-0008 +разных реплик: 2 diff --git a/docs/lab3/evidence/lb_errors.txt b/docs/lab3/evidence/lb_errors.txt new file mode 100644 index 0000000..4a1f868 --- /dev/null +++ b/docs/lab3/evidence/lb_errors.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker) +# все реплики остановлены -> ответ формирует сам HAProxy (http-error): общий JSON-формат ошибок, X-Request-Id совпадает с request_id; X-Instance-Id нет (контракт это допускает) +$ curl -si http://127.0.0.1:8080/api/v1/users/me +HTTP/1.1 503 Service Unavailable +x-request-id: 7F000001CD38-1F90-6ADCA7141E5-336F-0B75 +content-length: 128 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"7F000001CD38-1F90-6ADCA7141E5-336F-0B75"}} + +$ curl -si -H "X-Request-Id: demo-123" http://127.0.0.1:8080/health/ready +HTTP/1.1 503 Service Unavailable +x-request-id: demo-123 +content-length: 97 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"demo-123"}} diff --git a/docs/lab3/evidence/lb_log_sample.txt b/docs/lab3/evidence/lb_log_sample.txt new file mode 100644 index 0000000..a05fe96 --- /dev/null +++ b/docs/lab3/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-10-26 19:35:57 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.2s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:34156 [26/Oct/2026:19:35:54.564] public api/api1 0/0/0/7/7 201 401 ---- 8/8/7/1/0 0/0 rid=AC120001856C-1F90-6ADFAB9A234-1128-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34116 [26/Oct/2026:19:35:54.563] public api/api1 0/0/0/10/10 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018544-1F90-6ADFAB9A233-1126-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34168 [26/Oct/2026:19:35:54.578] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018578-1F90-6ADFAB9A242-1129-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34064 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/58/59 201 401 ---- 8/8/7/7/0 0/0 rid=AC1200018510-1F90-6ADFAB9A218-111F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34040 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/62/62 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000184F8-1F90-6ADFAB9A218-111D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34098 [26/Oct/2026:19:35:54.537] public api/api2 0/0/0/61/62 201 401 ---- 6/6/5/5/0 0/0 rid=AC1200018532-1F90-6ADFAB9A219-1121-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34100 [26/Oct/2026:19:35:54.538] public api/api2 0/0/0/63/63 201 401 ---- 6/6/5/4/0 0/0 rid=AC1200018534-1F90-6ADFAB9A21A-1123-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34162 [26/Oct/2026:19:35:54.578] public api/api2 0/0/0/27/28 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018572-1F90-6ADFAB9A242-112A-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34186 [26/Oct/2026:19:35:54.600] public api/api1 0/0/0/7/7 201 401 ---- 6/6/3/0/0 0/0 rid=AC120001858A-1F90-6ADFAB9A258-112C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34128 [26/Oct/2026:19:35:54.563] public api/api2 0/0/0/44/44 201 401 ---- 5/5/2/2/0 0/0 rid=AC1200018550-1F90-6ADFAB9A233-1125-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34182 [26/Oct/2026:19:35:54.587] public api/api2 0/0/0/21/21 201 401 ---- 4/4/1/1/0 0/0 rid=AC1200018586-1F90-6ADFAB9A24B-112B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34142 [26/Oct/2026:19:35:54.564] public api/api2 0/0/0/45/45 201 401 ---- 3/3/0/0/0 0/0 rid=AC120001855E-1F90-6ADFAB9A234-1127-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34198 [26/Oct/2026:19:35:54.614] public api/api1 0/0/0/14/14 201 401 ---- 8/8/6/2/0 0/0 rid=AC1200018596-1F90-6ADFAB9A25F-112E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34230 [26/Oct/2026:19:35:54.621] public api/api1 0/0/0/10/11 201 401 ---- 7/7/5/1/0 0/0 rid=AC12000185B6-1F90-6ADFAB9A26A-1130-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34242 [26/Oct/2026:19:35:54.622] public api/api1 0/0/0/9/9 201 401 ---- 6/6/4/0/0 0/0 rid=AC12000185C2-1F90-6ADFAB9A26E-1132-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34270 [26/Oct/2026:19:35:54.638] public api/api1 0/0/0/6/7 201 401 ---- 8/8/6/1/0 0/0 rid=AC12000185DE-1F90-6ADFAB9A27D-1134-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34266 [26/Oct/2026:19:35:54.640] public api/api1 0/0/0/6/6 201 401 ---- 7/7/5/0/0 0/0 rid=AC12000185DA-1F90-6ADFAB9A26E-1136-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34278 [26/Oct/2026:19:35:54.656] public api/api1 0/0/0/5/5 201 401 ---- 8/8/6/0/0 0/0 rid=AC12000185E6-1F90-6ADFAB9A27F-1138-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34302 [26/Oct/2026:19:35:54.686] public api/api1 0/0/0/6/6 201 401 ---- 8/8/7/0/0 0/0 rid=AC12000185FE-1F90-6ADFAB9A299-113A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34282 [26/Oct/2026:19:35:54.639] public api/api2 0/0/0/66/66 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185EA-1F90-6ADFAB9A27F-1135-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34254 [26/Oct/2026:19:35:54.622] public api/api2 0/0/0/83/84 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000185CE-1F90-6ADFAB9A26E-1133-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34294 [26/Oct/2026:19:35:54.671] public api/api2 0/0/0/36/36 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000185F6-1F90-6ADFAB9A28C-1139-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34338 [26/Oct/2026:19:35:54.715] public api/api1 0/0/0/5/5 201 401 ---- 8/8/5/0/0 0/0 rid=AC1200018622-1F90-6ADFAB9A2CB-113C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34346 [26/Oct/2026:19:35:54.734] public api/api1 0/0/1/6/7 201 401 ---- 8/8/7/0/0 0/0 rid=AC120001862A-1F90-6ADFAB9A2CC-113E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34360 [26/Oct/2026:19:35:54.745] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018638-1F90-6ADFAB9A2E9-1140-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34290 [26/Oct/2026:19:35:54.652] public api/api2 0/0/0/56/123 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185F2-1F90-6ADFAB9A28C-1137-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34192 [26/Oct/2026:19:35:54.609] public api/api2 0/0/0/171/172 201 401 ---- 8/8/7/6/0 0/0 rid=AC1200018590-1F90-6ADFAB9A25E-112D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34240 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/159/160 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000185C0-1F90-6ADFAB9A26C-1131-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34214 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/160/161 201 401 ---- 6/6/5/4/0 0/0 rid=AC12000185A6-1F90-6ADFAB9A269-112F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34312 [26/Oct/2026:19:35:54.698] public api/api2 0/0/0/87/87 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018608-1F90-6ADFAB9A2BA-113B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34374 [26/Oct/2026:19:35:54.779] public api/api1 0/0/0/8/8 201 401 ---- 4/4/3/0/0 0/0 rid=AC1200018646-1F90-6ADFAB9A30B-1142-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34364 [26/Oct/2026:19:35:54.755] public api/api2 0/0/0/35/35 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001863C-1F90-6ADFAB9A2F3-1141-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34326 [26/Oct/2026:19:35:54.734] public api/api2 0/0/0/59/59 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018616-1F90-6ADFAB9A2CB-113F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34354 [26/Oct/2026:19:35:54.725] public api/api2 0/0/0/68/68 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018632-1F90-6ADFAB9A2D5-113D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34384 [26/Oct/2026:19:35:54.801] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC1200018650-1F90-6ADFAB9A31B-1144-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34426 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/15/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC120001867A-1F90-6ADFAB9A322-1148-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34392 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018658-1F90-6ADFAB9A31D-1146-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34440 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 5/5/4/0/0 0/0 rid=AC1200018688-1F90-6ADFAB9A322-114A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34390 [26/Oct/2026:19:35:54.799] public api/api2 0/0/1/18/20 201 401 ---- 4/4/3/3/0 0/0 rid=AC1200018656-1F90-6ADFAB9A31C-1143-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34382 [26/Oct/2026:19:35:54.801] public api/api2 0/0/0/18/18 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001864E-1F90-6ADFAB9A31B-1145-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34408 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/17/17 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018668-1F90-6ADFAB9A322-1147-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34422 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/18/18 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018676-1F90-6ADFAB9A322-1149-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34462 [26/Oct/2026:19:35:54.837] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC120001869E-1F90-6ADFAB9A344-114E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34508 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC12000186CC-1F90-6ADFAB9A346-1152-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34450 [26/Oct/2026:19:35:54.834] public api/api1 0/0/0/20/20 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018692-1F90-6ADFAB9A33F-114C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34490 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/17 201 401 ---- 5/5/4/0/0 0/0 rid=AC12000186BA-1F90-6ADFAB9A346-1150-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34534 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/10/10 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186E6-1F90-6ADFAB9A361-1156-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34530 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/12/12 201 401 ---- 7/7/6/0/0 0/0 rid=AC12000186E2-1F90-6ADFAB9A361-1155-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34446 [26/Oct/2026:19:35:54.834] public api/api2 0/0/0/59/59 201 401 ---- 8/8/6/6/0 0/0 rid=AC120001868E-1F90-6ADFAB9A337-114B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34458 [26/Oct/2026:19:35:54.836] public api/api2 0/0/0/62/62 201 401 ---- 8/8/6/5/0 0/0 rid=AC120001869A-1F90-6ADFAB9A343-114D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34538 [26/Oct/2026:19:35:54.897] public api/api1 0/0/1/8/9 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186EA-1F90-6ADFAB9A372-1158-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34478 [26/Oct/2026:19:35:54.837] public api/api2 0/0/0/71/71 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000186AE-1F90-6ADFAB9A345-114F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34570 [26/Oct/2026:19:35:54.904] public api/api1 0/0/0/9/9 201 401 ---- 8/8/5/0/0 0/0 rid=AC120001870A-1F90-6ADFAB9A388-115A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34594 [26/Oct/2026:19:35:54.934] public api/api1 0/0/1/5/6 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018722-1F90-6ADFAB9A391-115D-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34498 [26/Oct/2026:19:35:54.838] public api/api2 0/0/0/71/136 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000186C2-1F90-6ADFAB9A346-1151-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34522 [26/Oct/2026:19:35:54.856] public api/api2 0/0/0/121/121 201 401 ---- 5/5/4/4/0 0/0 rid=AC12000186DA-1F90-6ADFAB9A358-1153-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34536 [26/Oct/2026:19:35:54.882] public api/api2 0/0/0/96/96 201 401 ---- 4/4/3/3/0 0/0 rid=AC12000186E8-1F90-6ADFAB9A372-1157-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34528 [26/Oct/2026:19:35:54.865] public api/api2 0/0/0/114/115 201 401 ---- 3/3/2/2/0 0/0 rid=AC12000186E0-1F90-6ADFAB9A361-1154-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34556 [26/Oct/2026:19:35:54.899] public api/api2 0/0/0/86/86 201 401 ---- 2/2/1/1/0 0/0 rid=AC12000186FC-1F90-6ADFAB9A383-1159-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34586 [26/Oct/2026:19:35:54.934] public api/api2 0/0/1/51/53 201 401 ---- 1/1/0/0/0 0/0 rid=AC120001871A-1F90-6ADFAB9A391-115C-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" diff --git a/docs/lab3/evidence/load.json b/docs/lab3/evidence/load.json new file mode 100644 index 0000000..87063ba --- /dev/null +++ b/docs/lab3/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab3/evidence/load.txt b/docs/lab3/evidence/load.txt new file mode 100644 index 0000000..c7c17fb --- /dev/null +++ b/docs/lab3/evidence/load.txt @@ -0,0 +1,28 @@ +# 2026-10-26 19:35:55 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.4s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-10-26 19:35:57 UTC | CPU/RAM после нагрузки | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-api-6 19.92% 89.26MiB / 256MiB 34.87% +messenger-lab3-api-5 0.67% 89.33MiB / 256MiB 34.89% +messenger-lab3-lb-1 0.14% 14.34MiB / 128MiB 11.20% +messenger-lab3-postgres-1 11.42% 68.96MiB / 512MiB 13.47% +messenger-lab3-redis-1 2.36% 4.148MiB / 96MiB 4.32% + diff --git a/docs/lab3/evidence/migrations.txt b/docs/lab3/evidence/migrations.txt new file mode 100644 index 0000000..b07edae --- /dev/null +++ b/docs/lab3/evidence/migrations.txt @@ -0,0 +1,26 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +## 1. Реплика не готова, пока схема не мигрирована; migrate однократный, повтор — no-op; две копии одновременно — advisory lock +$ # реплика против ПУСТОЙ БД, миграции ещё не выполнялись (RUN_MIGRATIONS_ON_START=false) +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready +{"error":{"code":"unavailable","message":"Database schema is not migrated","request_id":"1f3109e3-a934-4c1b-bfbd-22350e35e05d"}} 503 +$ psql -c "select to_regclass('schema_migrations')" +NULL +$ python -m src.migrate # = сервис migrate (one-shot) +applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ python -m src.migrate # повтор: no-op +schema is up to date; nothing applied +$ # две копии migrate одновременно (advisory lock) на ещё одной пустой БД +NOTICE: database "messenger_race" does not exist, skipping +copy A: schema is up to date; nothing applied +copy B: applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +schema_migrations rows: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready # та же реплика, без рестарта +{"status":"ok"} 200 + +## 2. Совместимость миграции 003 со старой версией (rolling update): код lab2 работает на схеме lab3 одновременно с репликами lab3 +$ # старая версия: код lab2 (без Idempotency-Key/X-Instance-Id) против БД, мигрированной до 003 +migrations applied at startup: none +$ cd lab2 && python tests/smoke.py --base-url http://127.0.0.1:18098 # контракт lab2 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +$ cd lab3 && python tests/smoke.py --base-url http://127.0.0.1:8080 # одновременно реплики lab3 за LB +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/persistence.txt b/docs/lab3/evidence/persistence.txt new file mode 100644 index 0000000..6113558 --- /dev/null +++ b/docs/lab3/evidence/persistence.txt @@ -0,0 +1,24 @@ +# 2026-10-26 19:35:10 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 31 seconds (healthy) 32 seconds ago +messenger-lab3-api-6 Up 21 seconds (healthy) 22 seconds ago +messenger-lab3-lb-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 4 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 18 minutes (healthy) 44 minutes ago +# 2026-10-26 19:35:24 UTC | Persistence: make test-persistence | exit 0 | 14.1s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:35:24 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 7 seconds (healthy) 46 seconds ago +messenger-lab3-api-6 Up 7 seconds (healthy) 36 seconds ago +messenger-lab3-lb-1 Up Less than a second (health: starting) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 5 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 7 seconds (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 7 seconds (healthy) 44 minutes ago + diff --git a/docs/lab3/evidence/rolling_back.json b/docs/lab3/evidence/rolling_back.json new file mode 100644 index 0000000..e79934d --- /dev/null +++ b/docs/lab3/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 27.69, + "replicas_before": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "replicas_after": [ + "c38982836a1d", + "ecaa2acc2a0a" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1342, + "ok": 1342, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 267.5, + "write_ok_rps": 159.5, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 94.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + }, + "during_update": { + "from_s": 5.02, + "to_s": 32.7, + "requests": 6993, + "ok": 6993, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 252.6, + "write_ok_rps": 149.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 140.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3", + "c38982836a1d", + "ecaa2acc2a0a" + ] + }, + "after": { + "from_s": 32.7, + "to_s": 40.77, + "requests": 2045, + "ok": 2045, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 253.6, + "write_ok_rps": 152.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 94.0 + }, + "instances": [ + "c38982836a1d", + "ecaa2acc2a0a" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api1", + "api3" + ] + ], + [ + 11.42, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.23, + [ + "api1", + "api3" + ] + ], + [ + 14.77, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.16, + [ + "api1", + "api2" + ] + ], + [ + 25.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.66, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6166, + "stored_messages": 6166, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.json b/docs/lab3/evidence/rolling_update.json new file mode 100644 index 0000000..e4d09f8 --- /dev/null +++ b/docs/lab3/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.38, + "replicas_before": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "replicas_after": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1161, + "ok": 1161, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 232.2, + "write_ok_rps": 136.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.38, + "requests": 6950, + "ok": 6950, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.9, + "write_ok_rps": 145.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 156.0 + }, + "instances": [ + "6e2f0c643449", + "a21409fffeec", + "b6e350f043f3", + "c4c5ae5b0949" + ] + }, + "after": { + "from_s": 33.38, + "to_s": 41.41, + "requests": 2008, + "ok": 2008, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 250.0, + "write_ok_rps": 147.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 12.5, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.73, + [ + "api1", + "api2" + ] + ], + [ + 15.25, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.28, + [ + "api2", + "api3" + ] + ], + [ + 25.73, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.34, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5981, + "stored_messages": 5981, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.txt b/docs/lab3/evidence/rolling_update.txt new file mode 100644 index 0000000..1fef464 --- /dev/null +++ b/docs/lab3/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-10-26 19:34:26 UTC | Rolling update lab3 -> lab3.1 под нагрузкой | exit 0 | 44.1s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['a21409fffeec', 'c4c5ae5b0949'] +2026-10-26T19:33:48.176Z rolling-update: start; replicas=2 old=['c4c5ae5b0949', 'a21409fffeec'] +2026-10-26T19:33:51.770Z rolling-update: images built in 3.6s (tag lab3.1) +2026-10-26T19:33:53.990Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-10-26T19:34:04.788Z rolling-update: step 1/2 new=6e2f0c643449 (messenger-api:lab3.1, ready+UP in 6.8s) replaced old=c4c5ae5b0949 (graceful stop 3.8s) +2026-10-26T19:34:15.884Z rolling-update: step 2/2 new=b6e350f043f3 (messenger-api:lab3.1, ready+UP in 6.7s) replaced old=a21409fffeec (graceful stop 4.3s) +2026-10-26T19:34:16.330Z rolling-update: done in 28.1s; replicas now ['6e2f0c643449', 'b6e350f043f3'] +update 28.4s rc=0 | replicas ['a21409fffeec', 'c4c5ae5b0949'] -> ['6e2f0c643449', 'b6e350f043f3'] | errors during update 0 (-) | RPS before 232.2, during 244.9, after 250.0 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-10-26 19:35:09 UTC | Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой | exit 0 | 43.4s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:32.642Z rolling-update: start; replicas=2 old=['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:35.497Z rolling-update: images built in 2.9s (tag lab3) +2026-10-26T19:34:37.818Z rolling-update: migrations once in 2.3s: schema is up to date; nothing applied +2026-10-26T19:34:48.605Z rolling-update: step 1/2 new=c38982836a1d (messenger-api:lab3, ready+UP in 7.0s) replaced old=6e2f0c643449 (graceful stop 3.7s) +2026-10-26T19:34:59.414Z rolling-update: step 2/2 new=ecaa2acc2a0a (messenger-api:lab3, ready+UP in 6.8s) replaced old=b6e350f043f3 (graceful stop 3.9s) +2026-10-26T19:34:59.817Z rolling-update: done in 27.4s; replicas now ['c38982836a1d', 'ecaa2acc2a0a'] +update 27.7s rc=0 | replicas ['6e2f0c643449', 'b6e350f043f3'] -> ['c38982836a1d', 'ecaa2acc2a0a'] | errors during update 0 (-) | RPS before 267.5, during 252.6, after 253.6 | p95 during 78.0 ms | data ok=True +PASS: rolling update under load + +# 2026-10-26 19:35:09 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c38982836a1d messenger-lab3-api-5 running healthy messenger-api:lab3 api2 UP +ecaa2acc2a0a messenger-lab3-api-6 running healthy messenger-api:lab3 api3 UP + diff --git a/docs/lab3/evidence/shutdown_timeline.txt b/docs/lab3/evidence/shutdown_timeline.txt new file mode 100644 index 0000000..7c3bddb --- /dev/null +++ b/docs/lab3/evidence/shutdown_timeline.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +# Настройки: HAProxy inter 1s / fastinter 500ms / fall 2 / rise 2 / timeout check 1.5s; API DRAIN_SECONDS=3, GRACEFUL_TIMEOUT_SECONDS=10 +# graceful stop timeline, t=0 — начало наблюдения; интервал опроса 50 мс + +0.02s (от SIGTERM -0.48s) LB api3: UP + +0.02s (от SIGTERM -0.48s) nat-api-3 /health/ready -> 200 + +0.50s (от SIGTERM -0.00s) SIGTERM -> nat-api-3 (pid 3196) + +0.51s (от SIGTERM +0.01s) nat-api-3 /health/ready -> 503 + +1.26s (от SIGTERM +0.76s) LB api3: UP 1/2 + +1.80s (от SIGTERM +1.30s) LB api3: DOWN + +3.52s (от SIGTERM +3.02s) nat-api-3 /health/ready -> connection refused + +3.62s (от SIGTERM +3.12s) nat-api-3 process exited + +# лог реплики: +2026-10-24 15:39:19,576 INFO messenger.server signal 15: instance nat-api-3 draining for 3.0s (readiness=503), then graceful stop +2026-10-24 15:39:22,580 INFO uvicorn.error Shutting down +2026-10-24 15:39:22,681 INFO messenger.api shutdown: instance=nat-api-3 closing pool +2026-10-24 15:39:22,682 INFO uvicorn.error Finished server process [3196] diff --git a/docs/lab3/evidence/smoke.txt b/docs/lab3/evidence/smoke.txt new file mode 100644 index 0000000..574b73d --- /dev/null +++ b/docs/lab3/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-10-26 19:30:00 UTC | Публичный smoke | exit 0 | 0.9s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/stand.txt b/docs/lab3/evidence/stand.txt new file mode 100644 index 0000000..20ae6e5 --- /dev/null +++ b/docs/lab3/evidence/stand.txt @@ -0,0 +1,33 @@ +# 2026-10-26 19:29:57 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab3-api-1 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 12 minutes (healthy) 8080/tcp +messenger-lab3-api-2 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 13 minutes (healthy) 8080/tcp +messenger-lab3-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 39 minutes ago Up 13 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab3-migrate-1 messenger-lab3-migrate "python -m src.migra…" migrate 39 minutes ago Exited (0) 13 minutes ago +messenger-lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 39 minutes ago Up 13 minutes (healthy) 5432/tcp +messenger-lab3-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 39 minutes ago Up 13 minutes (healthy) 6379/tcp +# 2026-10-26 19:29:57 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:29:59 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-lb-1 1.87% 12.87MiB / 128MiB 10.06% +messenger-lab3-api-2 16.79% 204.4MiB / 256MiB 79.86% +messenger-lab3-api-1 0.31% 222.9MiB / 256MiB 87.07% +messenger-lab3-postgres-1 2.23% 58.11MiB / 512MiB 11.35% +messenger-lab3-redis-1 0.22% 4.559MiB / 96MiB 4.75% + +# 2026-10-26 19:29:59 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.3s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp + diff --git a/docs/lab3/evidence/student-tests.txt b/docs/lab3/evidence/student-tests.txt new file mode 100644 index 0000000..2489d48 --- /dev/null +++ b/docs/lab3/evidence/student-tests.txt @@ -0,0 +1,134 @@ +# 2026-10-26 19:30:06 UTC | Собственные тесты (HTTP через LB) | exit 0 | 6.2s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests) ... ok +test_password_never_returned (test_api.ApiBehaviorTests) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests) ... ok +test_login_with_body_rejected (test_auth.AuthTests) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests) ... ok +test_malformed_basic_headers (test_auth.AuthTests) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 47 tests in 4.640s + +OK (skipped=13) + + 24 parallel POSTs, one key -> 1 message; replicas: ['a21409fffeec', 'c4c5ae5b0949'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-10-26 19:30:14 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.5s +$ python scripts/ops.py in-network storage + Container messenger-lab3-api-run-bdc116a8fd34 Creating + Container messenger-lab3-api-run-bdc116a8fd34 Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.223s + +OK + Container messenger-lab3-api-run-630b208e9588 Creating + Container messenger-lab3-api-run-630b208e9588 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.864s + +OK + +# 2026-10-26 19:30:27 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 13.7s +$ python scripts/ops.py in-network expiry + Container messenger-lab3-redis-1 Running + Container messenger-lab3-postgres-1 Running + Container messenger-lab3-api-1 Running + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-short-ttl-1 Creating + Container messenger-lab3-api-short-ttl-1 Created + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Starting + Container messenger-lab3-migrate-1 Started + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-redis-1 Waiting + Container messenger-lab3-redis-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-short-ttl-1 Starting + Container messenger-lab3-api-short-ttl-1 Started + Container messenger-lab3-api-run-a36baf72714e Creating + Container messenger-lab3-api-run-a36baf72714e Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab3-api-run-c445a7ef00cd Creating + Container messenger-lab3-api-run-c445a7ef00cd Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.620s + +OK + diff --git a/docs/lab4/REPORT.md b/docs/lab4/REPORT.md new file mode 100644 index 0000000..0e67451 --- /dev/null +++ b/docs/lab4/REPORT.md @@ -0,0 +1,273 @@ +# Отчёт — лабораторная 4 + +> Перенесено из корня репозитория lab4 при переходе на lab5 без изменений по существу. Пути `evidence/...` в этом файле указывают на `docs/lab4/evidence/`. Отчёт lab5 — [REPORT.md](../../REPORT.md). + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0, **boto3 1.35.99**, **Pillow 11.1.0**; PostgreSQL 16.4, Redis 7.4.1, HAProxy 2.8.30, **SeaweedFS 4.50**. Документация: Amazon S3 (PutObject, Content-MD5, ListObjectsV2), SeaweedFS wiki (Amazon S3 API, S3 credentials/identities, `weed server`), спецификация PNG (ISO/IEC 15948: чанки, CRC, Adam7), JPEG (ITU T.81: маркеры SOF/SOS/EOI), Pillow (`Image.open`, `draft`), glibc `mallopt` (MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_). Помощь ИИ (Claude) — при проектировании схемы согласованности БД/S3, проверки изображений, тестов, экспериментов и при написании отчёта. + +Решения lab3 (балансировщик, реплики, идемпотентность, drain, rolling update) перенесены без изменений семантики; их отчёт и доказательства сохранены в [docs/lab3/REPORT.md](../lab3/REPORT.md). + +## Запуск из чистого clone + +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`; **`chrislusf/seaweedfs:4.50`** (тег задаётся `SEAWEEDFS_VERSION` в `.env`, не `latest`). Нужны Docker Engine / Docker Desktop с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Оркестрация — **Compose + HAProxy** (Swarm не используется). + +```sh +# 1. Секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py # + 4 файла S3-ключей и secrets/s3_config.json +cp .env.example .env # Windows: copy .env.example .env + +# 2. Сборка и запуск: postgres+redis -> migrate; s3 -> s3-init (bucket) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Готовность: 2 реплики healthy и UP в HAProxy; S3 реально пишет/читает +python scripts/ops.py wait +python scripts/ops.py wait-s3 +python scripts/ops.py ps +``` + +Прогрев: первая сборка 1–3 мин, далее стенд готов за ~30–40 с (healthcheck PostgreSQL и SeaweedFS, миграции, bootstrap бакета, `start_period` API, `rise 2` в HAProxy). + +```sh +# Публичные проверки +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh # Windows: scripts\restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh # Windows: scripts\stop-one.cmd +python scripts/ops.py restore # вернуть реплику, затем failover повторить для второй + +# Собственные проверки уровня 4 (lab4) +python student-tests/run_all.py --base-url http://localhost:8080 # lab2-3 + test_attachments.py (17 тестов) +python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы +python scripts/ops.py s3-check # приватный бакет вне API, права ключей, закрытые админ-порты +python scripts/ops.py reconcile # сверка БД<->S3, dry-run (--apply — исправить) +python student-tests/s3_outage.py --output evidence/s3_outage.json +python student-tests/upload_memory.py --concurrency 10 --docker --output evidence/upload_memory_c10.json + +# Всё перечисленное (и проверки lab3) одной командой с записью в evidence/: +python scripts/collect_evidence.py # ~12–15 мин; --quick ~7–8 мин +``` + +**Остановка.** Безопасная (данные PostgreSQL и S3 сохраняются): `docker compose --env-file .env --profile ttl --profile faults down`. Намеренное удаление всех учебных данных (volumes `pgdata`, `s3data`) — только вручную: `docker compose --env-file .env --profile ttl --profile faults down -v`. Hooks и тесты volumes не удаляют. + +**Действия оператора.** Создать секреты до первого запуска (пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` существующие файлы не перезаписывает; S3-ключи не связаны с данными и могут быть заменены — см. docstring скрипта). После `stop-one` вернуть реплику `python scripts/ops.py restore`. Если тег `chrislusf/seaweedfs:4.50` недоступен в вашем registry-зеркале, укажите соседний проверенный тег в `SEAWEEDFS_VERSION`. + +## Архитектура и решения + +``` + 127.0.0.1:8080 (единственная публичная точка входа) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки /health/ready + │ POST .../attachments: timeout server 90 s, остальное 10 s + ▼ сеть backend + ┌──────── api x2 (read-only rootfs, tmpfs /tmp 16m, 0.5 CPU / 320M) ────────┐ + │ загрузка: поток -> лимит 10 MiB -> проверка PNG/JPEG -> [1] pending в БД │ + │ -> [2] S3 PUT (Content-MD5) -> [3] ready в БД -> 201 │ + │ скачивание: проверка членства -> поток из S3 (64 KiB) -> клиент │ + │ фон: сборщик pending/сирот (advisory lock, одна реплика за раз) │ + └──────┬───────────────────────┬────────────────────────────┬───────────────┘ + ▼ backend ▼ backend ▼ storage (internal: true) + postgres 16 (pgdata) redis 7 (кэш сессий) s3: SeaweedFS 4.50 (s3data) + users, chats, messages, только S3 API :8333 в сети storage; + sessions, idempotency_keys, master/filer/volume UI — 127.0.0.1 + attachments, message_attachments внутри контейнера; без анонимов + ▲ ▲ + migrate (one-shot) s3-init (one-shot, admin-ключи): + бакет + проверки прав и приватности +``` + +Порядок старта: `postgres`, `redis` (healthy) → `migrate`; `s3` (healthy) → `s3-init` (оба `service_completed_successfully`) → `api` (healthy) → `lb`. + +**Источник истины.** PostgreSQL — для всех метаданных, включая вложения (владелец, чат, имя, тип, размер, SHA-256, размеры в пикселях, ключ объекта, состояние). Байты оригинала — только в приватном бакете `messenger-attachments`. На API нет ни volume, ни каталога для файлов: rootfs read-only, `/tmp` — tmpfs 16 MiB (тело загрузки держится в памяти процесса до записи в S3, не на диске; `python scripts/ops.py api-fs`). + +### Выбор S3-сервера: SeaweedFS + +MinIO с осени 2025 г. не публикует community-образы и бинарники для свежих релизов, а консоль администратора у него встроена в тот же процесс. SeaweedFS (Apache-2.0): версия закрепляется тегом, S3-identities с правами на конкретный бакет задаются статическим JSON-конфигом (`-s3.config`), а все административные HTTP-интерфейсы (master 9333, filer 8888, volume 8080) привязываются к 127.0.0.1 внутри контейнера (`-ip.bind=127.0.0.1`), наружу в сеть `storage` выставлен только S3 (`-s3.ip.bind=0.0.0.0`). Встроенный IAM API отключён (`-s3.iam=false`), автосоздание бакетов — тоже (`-s3.autoCreateBucket=false`). + +| Роль | Сервис | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | haproxy:2.8.30-alpine | `/lb-health`; бэкенды по `/health/ready` | — | backend; 127.0.0.1:8080, :8404 | — | +| API ×2 | `api` | сборка `Dockerfile` | `/health/ready` (PostgreSQL + схема) | нет (read-only) | backend + storage; без ports | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key | +| БД | `postgres` | postgres:16.4-alpine | `pg_isready` | `pgdata` | backend | postgres_password | +| Кэш сессий | `redis` | redis:7.4.1-alpine | `redis-cli ping` | нет (только кэш) | backend | redis_password | +| S3 | `s3` | chrislusf/seaweedfs:4.50 | `wget /healthz` (+ `ops.py wait-s3`) | `s3data` | storage (internal); только :8333 | secret s3_config (identities) | +| Миграции | `migrate` | сборка | one-shot | — | backend | database_url | +| Bucket bootstrap | `s3-init` | сборка | one-shot | — | storage | admin + api S3-ключи | +| Тестовые отказы | `api-fault-*` (профиль `faults`) | образ API | `/health/ready` | — | backend + storage | как у api | + +### Модель данных (`migrations/004_attachments.sql`) + +- `attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key UNIQUE, state pending|ready, created_at, updated_at)`. CHECK-ограничения повторяют контракт: имя 1–255, тип png/jpeg, размер 1..10 485 760, `width*height ≤ 40 000 000`, sha256 в нижнем регистре. +- `object_key = "originals/" + случайный UUID`. Он **не** совпадает с `id` вложения, не содержит `filename` и не раскрывается в API. +- `message_attachments(message_id, attachment_id, chat_id, position)`: порядок `attachment_ids` сохраняется. **Составные внешние ключи** `(message_id, chat_id) → messages(id, chat_id)` и `(attachment_id, chat_id) → attachments(id, chat_id)` на уровне БД запрещают привязать файл чужого чата. `ON DELETE RESTRICT` не даёт удалить вложение, на которое ссылается сообщение. +- `messages_text_len` расширен до 0..4000: `text=""` допустим только вместе с вложениями (проверяет API). Миграция только расширяющая — реплики lab3 работают на новой схеме (rolling update). + +### Загрузка, `201` и границы транзакций + +`POST /api/v1/chats/{id}/attachments?filename=…` (сырые bytes, `src/main.py`): + +1. Дешёвые проверки до чтения тела: аутентификация → `filename` (400) → `Content-Type` только `image/png|image/jpeg` (415) → `Content-Length > 10 MiB` → 413 без чтения → членство в чате (404) → слот загрузки (`MAX_CONCURRENT_UPLOADS=4` на реплику; ожидание ≤ 5 с, иначе 503 + `Retry-After`). +2. Тело читается потоком; как только счётчик превысил 10 MiB — 413 (и для chunked без `Content-Length`). Остаток тела (до 10 MiB) дочитывается в никуда, чтобы клиент получил 413, а не обрыв. Общий дедлайн приёма — `UPLOAD_READ_TIMEOUT_SECONDS=60` (медленный клиент → 400). Обрыв клиентом → ничего не записано. +3. Проверка содержимого (`src/images.py`, ≤ 2 одновременных на реплику): + - сигнатура PNG/JPEG; **сигнатура должна совпасть с заявленным Content-Type** (подмена MIME → 415); + - размеры из IHDR / SOFn **до** распаковки: > 40 Мп → 413 (pixel bomb 8000×6000 размером ~50 КБ отклоняется за миллисекунды); + - PNG: все CRC, порядок IHDR…IDAT…IEND, допустимые bit depth/color type, **потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»**: объём должен в точности совпасть с ожидаемым (с учётом Adam7). Так обнаруживаются обрезанные и испорченные данные, а zip-бомба обрывается, как только вывод превысил ожидание; память O(64 KiB) при любом размере картинки; + - JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI → обрезанный файл = 415). Baseline дополнительно декодируется Pillow в режиме `draft` (1/8 масштаба). Progressive Pillow не декодирует: libjpeg держит коэффициенты всего кадра, замер — **119 MiB** на 39.7 Мп против **4.9 MiB** у baseline; + - независимый парсер Pillow должен увидеть тот же формат и размеры. +4. `[1]` `INSERT attachments … state='pending'` (с проверкой членства в том же операторе) — **COMMIT до S3**. +5. `[2]` S3 `PutObject` с `Content-MD5` (S3 сам отклоняет повреждённую передачу — BadDigest; проверено) и `x-amz-meta-sha256`. +6. `[3]` `UPDATE … SET state='ready' WHERE id=… AND state='pending'` — **COMMIT** → `201 Attachment {status: ready, variants: [], error_code: null}`. + +**Когда честно вернуть 201:** только после шага `[3]`. Байты к этому моменту записаны в S3 (PUT подтверждён с проверкой MD5), а метаданные зафиксированы в PostgreSQL. Ответ 201 до любого из этих событий означал бы, что «готовое» вложение может не скачаться. + +**S3 PUT прошёл, а COMMIT `[3]` — нет.** Остаются строка `pending` (не видна в API, прикрепить нельзя) и объект. При ошибке БД API сразу компенсирует: удаляет объект и пытается удалить строку, клиент получает 503. Если процесс умер (компенсации нет), строку и объект уберёт сборщик после grace period. Обе ветки проверены внедрением отказа (`api-fault-db`, `api-fault-crash`). + +### Согласованность БД и S3: компенсация + сборщик (`src/reconcile.py`) + +Главный инвариант: **ключ объекта записан в БД раньше, чем объект появится в бакете.** Поэтому объект без строки может появиться только после того, как строку удалили. Ключ одноразовый, удалённая строка не возрождается. + +| Ситуация | Кто исправляет | Как | +| --- | --- | --- | +| Ошибка S3 в `[2]` | запрос | DELETE своей pending-строки, 503 | +| Ошибка БД в `[3]` | запрос | DELETE объекта (+ попытка DELETE строки), 503 | +| Процесс умер между `[1]`/`[2]`/`[3]`, обрыв, сеть | сборщик, проход A | pending старше grace: **сначала** `DELETE … WHERE state='pending' RETURNING object_key` (атомарный захват; если запрос успел `[3]`, строка уже ready и не трогается), **потом** DELETE объекта | +| Объект без строки (сбой удаления в A, ручная запись) | сборщик, проход B | листинг `originals/` страницами; нет строки **и** `LastModified` старше grace → повторная проверка → DELETE | +| ready-строка без объекта | только отчёт (C) | байты не восстановить; авто-удаление метаданных опасно — `missing_objects` в отчёте | + +- **Grace period** `ATTACHMENT_GC_GRACE_SECONDS=900` (минимум 300, конфиг меньше не примет). Это намного больше самой долгой загрузки: чтение ≤ 60 с + проверка + S3 PUT (connect 2 с + read 10 с) × 2 попытки ≈ 90 с. Если загрузка всё же переживёт grace, `[3]` вернёт 0 строк → API удалит свой объект и ответит 503. +- **Свежие и используемые вложения не удаляются:** сборщик трогает только `pending` старше grace. ready-строки не удаляет никогда, а на используемые ещё и стоит FK RESTRICT. Тесты: `test_fresh_pending_is_kept`, `test_ready_and_referenced_are_never_collected` (прямой `DELETE` получает ForeignKeyViolation), `test_pending_cannot_be_attached`. +- Фоновый проход A — каждые 300 с, полный обход бакета B — раз в час. Выполняет одна реплика за раз (`pg_try_advisory_lock`). Ручной запуск: `python scripts/ops.py reconcile` (**dry-run** по умолчанию, ничего не меняет) и `… reconcile --apply`. Повторный `--apply` — no-op (проверено). Стоимость обхода на стенде: 238 объектов — 0.036 с (листинг + один `SELECT … = ANY` на страницу из 1000 ключей). + +### Ограничение памяти и потоковая обработка + +- Лимит байт применяется **во время** чтения (на 10 MiB + 1 байт). В памяти одной загрузки — тело (≤ 10 MiB) + кратковременно куски до склейки. Проверка PNG — O(64 KiB), JPEG baseline — ≤ ~5 MiB. Тело уходит в S3 как `BytesIO` поверх тех же bytes, без копии. +- Одновременно на реплику ≤ 4 загрузки (`MAX_CONCURRENT_UPLOADS`). Остальные ждут слот ≤ 5 с **до чтения тела**, поэтому их байты не накапливаются в памяти. Скачивание идёт потоком из S3 кусками по 64 KiB (`StreamingResponse`), без буферизации файла. +- Найдено и исправлено замерами (см. «Эксперименты»): `zlib.unconsumed_tail` копировал остаток IDAT на каждом шаге (пик 2× файла и квадратичное копирование), а botocore склеивал заголовки с телом `bytes` в один буфер. В образе также настроен glibc malloc (`MALLOC_ARENA_MAX=2`, фиксированный порог mmap 1 MiB), чтобы большие буферы возвращались ОС сразу. + +### Безопасность бакета и ключей + +- **Bootstrap идемпотентен** (`src/s3_bootstrap.py`, сервис `s3-init`): `CreateBucket` → `BucketAlreadyOwnedByYou` считается успехом (`evidence/sandbox/s3_bootstrap.txt`: повторный запуск — `already exists (idempotent no-op)`). После создания бакета s3-init проверяет права ключей и приватность; при нарушении он завершается с ошибкой, и API не стартует. +- **Отдельные ключи API** (`messenger_api`) имеют только `Read/Write/List` на `messenger-attachments`. `CreateBucket`/`DeleteBucket` → AccessDenied. Admin-ключи (`bootstrap_admin`) смонтированы только в `s3-init`. Identity `anonymous` не объявлена. +- **Проверка приватности вне API** (`python scripts/ops.py s3-check`): из сети `storage`, без подписи, к S3 напрямую. Анонимный GET реального объекта последнего вложения → 403, анонимный листинг бакета и списка бакетов → 403. Тот же объект с ключами API → 200 и совпадающий SHA-256. +- **Админ-консоль не открыта в общей сети:** порты 9333/8888/8080/19333/18888 с адреса `s3` закрыты (проверка в том же скрипте). Сеть `storage` — `internal: true`. LB в неё не входит, порты S3 на хост не публикуются. +- **Случайный ключ и приватный бакет не заменяют ACL приложения:** доступ к `GET /attachments/{id}` и `/content` проверяется по членству в чате на каждый запрос (посторонний → 404 без раскрытия существования). Download идёт через API без redirect и без подписанных URL. Ответ содержит `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `Content-Disposition` с безопасно закодированным именем. + +### Недоступность S3 и readiness + +S3 **не входит** в readiness: если бы реплики при падении S3 становились not-ready, LB вывел бы обе, и вместе с файлами пропали бы чат и вход. Вместо этого файловые операции быстро отвечают `503 unavailable` + `Retry-After` (таймауты клиента S3: connect 2 с, read 10 с, ≤ 2 попытки), а всё остальное работает. Критичные зависимости readiness — по-прежнему PostgreSQL и мигрированная схема. + +Таймауты boto3 не покрывают DNS: в Docker остановленный контейнер `s3` пропадает из DNS, и разрешение имени висит секундами. В первом прогоне на Docker Desktop загрузка ждала 12–13 с, а скачивание превысило `timeout server` HAProxy (10 с) и получило **504 от LB** вместо 503. Исправление (`src/main.py`, `s3_call`; `src/objectstore.py`, `CircuitBreaker`): +- у каждого вызова S3 из запроса есть общий дедлайн: скачивание 5 с, загрузка 10 с; +- после отказа circuit breaker 5 с сразу отвечает 503, затем делает одну пробную попытку; +- у API в compose `dns_opt: timeout:1, attempts:1`. + +Проверено на худшем случае — S3, который принимает соединение и молчит: upload 503 за 10.0 с, download 503 за 5.0 с (`evidence/sandbox/s3_outage_hang.json`). При S3, который сразу отказывает: 1.8–2.3 с и 0.01 с (breaker открыт). + +**Особенность SeaweedFS после рестарта** (замер): `/healthz` отвечает сразу, но первые операции блокируются. Первое чтение после рестарта шло ~15 с, первая запись ~29 с: выборы Raft-лидера (`electionTimeout` 10 с), затем heartbeat volume-сервера. С `-master.electionTimeout=2s` первая запись — ~19 с. Поэтому restart-hook и `ops.py wait-s3` ждут **реальной** готовности: проба put/get/delete и чтение существующего объекта, каждая операция < 2 с (`s3_bootstrap.py --wait-ready`); на стенде готовность наступала через 14.5–17.1 с. Штатная остановка SeaweedFS длится ~22 с, поэтому `stop_grace_period: 10s` (как у Docker по умолчанию) и затем SIGKILL. Подтверждённые записи к этому моменту уже в page cache ОС; тест сохранности после такого рестарта проходит. + +### Что перенесено из lab3 и что изменено + +Перенесены без изменения семантики: всё API lab2–3, HAProxy-конфигурация (обнаружение реплик через DNS, drain, `retries`/`redispatch`), идемпотентность, миграции one-shot, rolling update, hooks и собственные тесты lab2–3 (47 HTTP-тестов + 12 тестов хранилища проходят на lab4). + +Изменено в lab4: + +1. Переход контракта 3 → 4: поля `attachment_ids` в запросе и ответе сообщения (отсутствие = `[]` и для отпечатка идемпотентности; порядок значим), три новых endpoint. +2. Сервисы `s3`, `s3-init`, volume `s3data`, сеть `storage`; секреты S3; профиль `faults`. +3. HAProxy: для `POST …/attachments` — `timeout server 90s` (`http-request set-timeout`), остальным запросам — прежние 10 с. +4. Лимит памяти API 256M → 320M (4 загрузки по 10 MiB, замер ниже); glibc malloc tuning в `Dockerfile`. +5. `scripts/ops.py`: restart включает S3 и ждёт его прогрева; новые `wait-s3`, `s3-check`, `reconcile`, `api-fs`, `in-network consistency`. `collect_evidence.py` дополнен проверками lab4. + +### Соответствие `.env.example` и реальной конфигурации + +Переменные lab3 — без изменений (см. [docs/lab3/REPORT.md](../lab3/REPORT.md)). Новые: + +| Переменная | Значение | Где используется | +| --- | --- | --- | +| `SEAWEEDFS_VERSION` | 4.50 | тег образа `s3` | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | http://s3:8333, us-east-1, messenger-attachments | api, s3-init, `src/config.py` | +| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | /run/secrets/… | ключи API (compose secrets) | +| `S3_CONNECT_TIMEOUT_SECONDS`, `S3_READ_TIMEOUT_SECONDS`, `S3_MAX_ATTEMPTS` | 2, 10, 2 | `src/objectstore.py` | +| `S3_DOWNLOAD_DEADLINE_SECONDS`, `S3_UPLOAD_DEADLINE_SECONDS`, `S3_CIRCUIT_OPEN_SECONDS` | 5, 10, 5 | общий бюджет вызова S3 (вкл. DNS), circuit breaker | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | 10485760, 40000000 | не выше контракта (иначе API не стартует) | +| `MAX_CONCURRENT_UPLOADS`, `UPLOAD_QUEUE_TIMEOUT_SECONDS` | 4, 5 | слоты загрузки на реплику | +| `UPLOAD_READ_TIMEOUT_SECONDS` | 60 | дедлайн приёма тела | +| `IMAGE_DECODE_CONCURRENCY` | 2 | разборов изображения на реплику | +| `ATTACHMENT_GC_INTERVAL_SECONDS`, `ATTACHMENT_GC_GRACE_SECONDS`, `ORPHAN_SWEEP_INTERVAL_SECONDS` | 300, 900, 3600 | `src/reconcile.py` | +| `FAULT_INJECTION` | пусто | только сервисы `api-fault-*` | + +## Доказательства критериев + +**Где получены результаты.** Код проверен на стенде из тех же компонентов, запущенных нативными процессами: PostgreSQL 16, Redis 7, SeaweedFS 4.50 (тот же бинарник, те же флаги), HAProxy 2.8.30 с этим `haproxy.cfg` (статические серверы вместо DNS-шаблона) и 2 реплики API. Среда: Windows 10, Ryzen 5 7500f, 32 GB, Python 3.12, без cgroup-лимитов. Результаты — в `evidence/`. + +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | PASS: файл → сообщение с attachment_ids → download Бобом, SHA-256 совпал, Ева 404 | evidence/sandbox/smoke.txt; evidence/smoke.txt | +| Возможности lab3 сохранены | `run_all.py`, `in-network storage`, failover, persistence | 47 HTTP + 12 DB-тестов lab2–3 OK; failover/persistence PASS | evidence/sandbox/student-tests.txt | +| S3 в общем стенде; метаданные в PostgreSQL, байты в приватном бакете | `compose.yaml`, `ops.py api-fs`, `s3-check` | API без volumes, read-only rootfs; объекты только в `s3data` | evidence/s3_private.txt | +| 201 ready после сохранения; оригинал побайтно | `test_png_roundtrip…`, `test_exactly_max_size_is_accepted` | 201, `status=ready`, `variants=[]`; SHA-256 и Content-Length совпадают (вкл. ровно 10 485 760 байт) | evidence/sandbox/student-tests.txt | +| Message с attachment_ids, посторонний без доступа | smoke, `test_message_with_attachments…` | порядок id сохранён; Ева: 404 на метаданные и bytes | там же | +| PNG/JPEG ≤10 MiB, ≤40 Мп; MIME по содержимому; опасные имена; лимиты до распаковки | `test_too_large_*`, `test_too_many_pixels`, `test_mime_spoofing_is_415`, `test_dangerous_filenames_are_400` | 413/413/415/400; 48 Мп файл 50 КБ → 413 без распаковки | там же | +| Upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2 | PASS, 138–139 записей, 0 ошибок; файл скачан после остановки | evidence/sandbox/failover.txt | +| Restart стенда сохраняет метаданные и оригинал | `make test-persistence` | PASS; сменились PID PostgreSQL, SeaweedFS и обеих реплик; файл скачан со старым токеном | evidence/sandbox/persistence.txt | +| Недоступность S3, чужой чат, restart | `s3_outage.py`; `test_attachment_from_other_chat…`; persistence | 503 за 1.0–1.7 с, чат работает; 404; PASS | evidence/sandbox/s3_outage.json | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Тесты: большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `student-tests/test_attachments.py` (17 тестов) | 413 (Content-Length и chunked), 415 (обрезанный PNG/JPEG, неверный CRC, неверный объём IDAT, мусор, пустое тело, GIF), 415 (6 подмен MIME), 404 (чужое/неизвестное id, файл другого чата, Ева в своём чате), 400 (12 опасных имён); каждый негативный ответ < 500 в общем формате | evidence/sandbox/student-tests.txt | +| Обрыв загрузки и отказ между S3/БД не оставляют вечных объектов; компенсация/сверка с grace | `test_attachment_consistency.py` (10 тестов) в сети стенда | обрыв → 0 строк, 0 объектов; ошибка S3 → 503 + нет строки; ошибка БД после PUT → 503 + объект удалён; **падение процесса после PUT** → pending + объект → сборщик удаляет оба; dry-run ничего не меняет; повтор no-op; свежие/ready/используемые не удаляются | evidence/sandbox/consistency.txt; evidence/consistency.txt | +| Лимиты/таймауты/потоковая обработка; ≥10 одновременных загрузок с замером пика | `upload_memory.py --concurrency 10/20` | 10×9.5 MiB ×3 волны: 30/30 201, пик RSS 156–166 MiB на реплику (база 88–95); при 20 одновременных пик тот же (159–162); без лимитера 262–291 MiB | evidence/sandbox/upload_memory_*.json | +| Bootstrap идемпотентен | повторный `s3-init` | `already exists (idempotent no-op)`, все проверки ok | evidence/sandbox/s3_bootstrap.txt | +| Отдельные S3 credentials с ограниченными правами | `s3_bootstrap` проверки | API-ключи: put/get своего бакета — ok; CreateBucket/DeleteBucket — AccessDenied | evidence/sandbox/s3_private.txt | +| Админ-консоль не в общей сети | `ops.py s3-check` | 9333/8888/8080/19333/18888 закрыты, S3 :8333 — единственный | там же | +| Private bucket вне API | `ops.py s3-check` | анонимный GET реального объекта → 403, листинг → 403; с ключами API тот же объект → 200, SHA-256 совпал | там же | + +## Эксперименты и отказы + +**Стенд замеров:** Windows 10, Ryzen 5 7500f, 32 GB; все процессы (2 реплики API, HAProxy, PostgreSQL, Redis, SeaweedFS, генератор нагрузки) на одном хосте. Реплика: 1 процесс Uvicorn, threadpool по умолчанию. Файлы — синтетический шум, PNG без сжатия, 9.5 MiB (несжимаемые, проверка проходит по полной программе). Память — VmRSS процесса реплики (опрос каждые 20 мс) и пик RSS, который считает ядро (VmHWM, обнуляется перед прогоном). Загрузки через LB (roundrobin по двум репликам), повторы — волнами. + +**Гипотеза:** пик памяти реплики ≈ база + `MAX_CONCURRENT_UPLOADS` × (~2 × размер файла) и не зависит от числа одновременных клиентов; без лимитера память растёт линейно с конкуренцией. + +| Прогон | Одновременно | Загрузок | 201 / SHA ok | p50 / p95, с | База → пик RSS, MiB (реплика 1 / 2) | +| --- | --- | --- | --- | --- | --- | +| до исправлений (zlib tail, bytes в botocore) | 10 | 30 | 30 / 30 | 2.01 / 2.29 | 124 → 237 / 124 → 244 | +| исправления кода, malloc по умолчанию | 10 | 30 | 30 / 30 | 1.20 / 1.34 | 124 → 187 / 124 → 187 | +| + MALLOC_ARENA_MAX=2, mmap ≥ 1 MiB (**итог**) | 10 | 30 | 30 / 30 | 1.32 / 2.07 | 95 → 163 / 95 → 156 (пик ядра VmHWM 166 / 165) | +| итог, вдвое больше клиентов | 20 | 40 | 40 / 40 | 2.16 / 3.08 | 115 → 162 / 97 → 159 | +| лимитер выключен (`MAX_CONCURRENT_UPLOADS=64`) | 20 | 40 | 40 / 40 | 2.90 / 3.03 | 88 → 262 / 89 → 291 | + +Вывод: гипотеза подтверждена. Прирост ~61–72 MiB ≈ 4 слота × 2 × 9.5 MiB. При удвоении клиентов пик не растёт (лишние ждут слот, их тело ещё не прочитано), растёт только задержка. Без лимитера пик +170–200 MiB при той же пропускной способности: при лимите 256 MiB контейнер был бы убит OOM. Поэтому лимит API — 320M, запас ~155 MiB. Микрозамеры (tracemalloc, 9.5 MiB PNG): проверка 19 MiB → 1.5 MiB и 0.17 → 0.06 с; `put_object(bytes)` 25 MiB → `BytesIO` 0.1 MiB. Ограничения: один хост и общий CPU для клиента и сервера; RSS не включает page cache; в Docker тот же скрипт (`--docker`) читает VmHWM процесса API внутри контейнера, `docker stats` и cgroup `memory.peak`. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| SIGTERM обслужившей реплики (public failover ×2) | записи идут через другую, файл скачивается | 138/139 записей, 0 ошибок, файл совпал | первая запись после hook ≤ 4.5 с (hook ждёт drain 3 с); потерь нет | +| Каждая реплика по очереди под нагрузкой (`failover_series.py`) | 0 ошибок, LB DOWN < 2 с | DOWN 1.30 / 1.53 с, 0 ошибок, RPS 318 → 239 / 283 → 243 | возврат в ротацию 2.9 / 3.1 с; данные целы | +| Рестарт API + PostgreSQL + Redis + S3 + LB | данные, сессия и файл сохранены | PASS, PID всех процессов сменились | ~38 с hook (из них прогрев S3 16 с); потерь нет | +| Остановка S3 | файлы → 503 быстро, остальное работает | upload 503 за 1.0–1.7 с, download 503 за 1.0 с, readiness 200, сообщения 201 | после старта S3 прогрев 17 с; старый файл побайтно цел | +| S3 «висит» (принимает соединение, не отвечает) | 503 в пределах дедлайна, не 504 | upload 503 за 10.0 с, download 503 за 5.0 с | прогрев 16 с; старый файл цел | +| Ошибка S3 PUT (`api-fault-s3`) | 503, строки нет | 503, 0 строк | сразу | +| Ошибка БД после PUT (`api-fault-db`) | 503, объект удалён | 503, 0 строк, число объектов не изменилось | сразу | +| Падение процесса после PUT (`api-fault-crash`) | pending + объект, убирает сборщик | 1 pending + объект → после grace (сдвинутого в тесте) 0 / 0 | ≤ grace + интервал (900 + 300 с по умолчанию) | +| Обрыв клиентом на 0.2 из 5 MiB | ничего не записано | 0 строк, объектов столько же | сразу | +| Нагрузка lab3 (`tests/load.py`, 20 с, c=4) | без регрессии | 276 RPS, 0 ошибок, p50/p95/p99 13/24/33 мс | — | + +## Ограничения и следующие шаги + +- Один Docker-host, один PostgreSQL, один процесс SeaweedFS без репликации: отказ host/БД/S3 не переживается. S3 недоступен → файловые операции 503 (остальное работает). Восстановление S3 занимает 15–30 с прогрева. +- Тело загрузки держится в памяти процесса (≤ 10 MiB × 4 слота). Это ограничено и измерено, но не «настоящий» стриминг в S3. Следующий шаг — multipart upload кусками по 5 MiB с проверкой по частям или presigned upload (трек 1). +- Progressive JPEG проверяется структурно (маркеры, EOI), без декодирования пикселей: экономия ~115 MiB на 40 Мп. Испорченные энтропийные данные внутри структурно целого progressive-файла не обнаруживаются (их увидит lab5 при создании thumbnail → `failed`). +- `missing_objects` (ready без байтов) сборщик только сообщает; решение о судьбе метаданных — за оператором. +- Ключи идемпотентности, созданные репликой lab3 до обновления, имели отпечаток без `attachment_ids`; повтор такого ключа на lab4 даст 409 (окно 24 ч после rolling update). +- SIGKILL SeaweedFS через 10 с при остановке: безопасно для подтверждённых записей при живом ядре ОС, не при потере питания. +- Docker-прогон (`collect_evidence.py`) выполняется на машине студента. Нативный прогон подтверждает код, HAProxy-конфиг и флаги SeaweedFS, но не compose-обвязку. + +## Вопросы защиты — кратко + +- **Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?** Volume привязан к одному контейнеру и host. Другая реплика его не видит, поэтому после остановки загрузившей реплики файл недоступен (тест failover это и проверяет). У volume нет общего API доступа, ключей с правами и проверки целостности (Content-MD5), он не масштабируется независимо от API, а с read-only rootfs его пришлось бы ещё и открывать на запись. S3 — общий сервис с собственными ACL, его читают все реплики. +- **Когда можно честно вернуть 201?** Только после того, как S3 подтвердил PUT (с проверкой MD5) **и** COMMIT перевёл строку в `ready`. Раньше — нельзя: «готовое» вложение могло бы не скачаться. +- **S3 PUT прошёл, COMMIT — нет?** Остаются невидимая pending-строка и объект. Ошибку БД API компенсирует сам (удаляет объект, 503). Если процесс умер, сборщик после grace сначала атомарно удаляет pending-строку, потом объект. Объект без строки убирает проход по бакету. Строка всегда создаётся раньше объекта, поэтому «вечных» объектов нет. +- **Почему случайный ключ и приватный бакет не заменяют ACL приложения?** Приватный бакет защищает от анонимного доступа к S3, но клиенты в него и не ходят: все запросы идут через API с ключами API, которые читают **любой** объект бакета. Кто из пользователей что может скачать, знает только приложение (членство в чате). Случайный ключ — защита «неугадываемостью»: он может утечь (лог, ссылка), а доступ после выхода из чата или для чужого пользователя он не ограничивает. Поэтому каждый запрос проверяет членство и отвечает 404 постороннему. diff --git a/docs/lab4/evidence/README.md b/docs/lab4/evidence/README.md new file mode 100644 index 0000000..59eb259 --- /dev/null +++ b/docs/lab4/evidence/README.md @@ -0,0 +1,11 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Что лежит здесь (lab4) + +- `sandbox/` — результаты, полученные при разработке на стенде из тех же компонентов, запущенных нативными процессами (PostgreSQL 16, Redis 7, SeaweedFS 4.50, HAProxy 2.8.30, 2 реплики API; без Docker). Как получены — REPORT.md, раздел «Доказательства критериев». +- Остальные файлы создаёт `python scripts/collect_evidence.py` на compose-стенде (сводка — `SUMMARY.txt`). Токены и пароли в файлы не пишутся. +- Доказательства lab3 сохранены в `docs/lab3/evidence/`. diff --git a/docs/lab4/evidence/SUMMARY.txt b/docs/lab4/evidence/SUMMARY.txt new file mode 100644 index 0000000..c28d9f4 --- /dev/null +++ b/docs/lab4/evidence/SUMMARY.txt @@ -0,0 +1,46 @@ +# 2026-11-04 14:12:25 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Образ SeaweedFS (закреплённая версия) evidence/stand.txt +OK S3 bootstrap: лог первого запуска s3-init evidence/s3_bootstrap.txt +OK S3 bootstrap: повторный запуск (идемпотентность) evidence/s3_bootstrap.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB, включая test_attachments.py) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) evidence/consistency.txt +OK Сверка БД <-> S3 (dry-run) evidence/reconcile.txt +OK Приватный бакет вне API, права ключей, закрытые админ-порты evidence/s3_private.txt +OK На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes evidence/s3_private.txt +OK API: файлы в /tmp (tmpfs) после загрузок evidence/s3_private.txt +OK Пик памяти: 10 одновременных загрузок по 9.5 MiB evidence/upload_memory.txt +OK Пик памяти: 20 одновременных загрузок evidence/upload_memory.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Недоступность S3 и восстановление evidence/s3_outage.txt +OK Rolling update lab4 -> lab4.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Persistence: сверка БД <-> S3 после рестарта evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab4/evidence/check.txt b/docs/lab4/evidence/check.txt new file mode 100644 index 0000000..6abe771 --- /dev/null +++ b/docs/lab4/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-11-04 14:03:55 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-04 14:03:55 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab4/evidence/consistency.txt b/docs/lab4/evidence/consistency.txt new file mode 100644 index 0000000..fd9c005 --- /dev/null +++ b/docs/lab4/evidence/consistency.txt @@ -0,0 +1,75 @@ +# 2026-11-04 14:04:49 UTC | Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) | exit 0 | 15.8s +$ python scripts/ops.py in-network consistency + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-fault-s3-1 Creating + Container messenger-lab4-api-fault-db-1 Creating + Container messenger-lab4-api-fault-crash-1 Creating + Container messenger-lab4-api-fault-db-1 Created + Container messenger-lab4-api-fault-s3-1 Created + Container messenger-lab4-api-fault-crash-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-crash-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-s3-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-db-1 Starting + Container messenger-lab4-api-fault-crash-1 Started + Container messenger-lab4-api-fault-s3-1 Started + Container messenger-lab4-api-fault-db-1 Started + Container messenger-lab4-api-run-46322fbc844a Creating + Container messenger-lab4-api-run-46322fbc844a Created +ready: http://api-fault-s3:8080/health/ready + Container messenger-lab4-api-run-ff8a88e550db Creating + Container messenger-lab4-api-run-ff8a88e550db Created +ready: http://api-fault-db:8080/health/ready + Container messenger-lab4-api-run-476fc515a8f0 Creating + Container messenger-lab4-api-run-476fc515a8f0 Created +ready: http://api-fault-crash:8080/health/ready + Container messenger-lab4-api-run-9b2dce8846fe Creating + Container messenger-lab4-api-run-9b2dce8846fe Created +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.224s + +OK diff --git a/docs/lab4/evidence/failover.txt b/docs/lab4/evidence/failover.txt new file mode 100644 index 0000000..65ce6a9 --- /dev/null +++ b/docs/lab4/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-11-04 14:05:30 UTC | Failover #1: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP +# 2026-11-04 14:05:46 UTC | Failover #1: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.187} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:05:46 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:05:52 UTC | Failover #1: restore | exit 0 | 6.3s +$ python scripts/ops.py restore +2026-11-04T14:05:52.981Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.2s + +# 2026-11-04 14:05:53 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:09 UTC | Failover #2: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:09 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:06:16 UTC | Failover #2: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-04T14:06:16.514Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-04 14:06:16 UTC | Failover #3: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:33 UTC | Failover #3: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:33 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 exited unhealthy messenger-api:lab4 - - +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:40 UTC | Failover #3: restore | exit 0 | 6.7s +$ python scripts/ops.py restore +2026-11-04T14:06:40.028Z restore: started ['messenger-lab4-api-1']; 2 replicas healthy and UP in LB after 6.6s + +# остановленные публичным тестом реплики: ['4fc3ad11e268', '4fc3ad11e268', 'c9fab9c9e2c1'] diff --git a/docs/lab4/evidence/failover_series.json b/docs/lab4/evidence/failover_series.json new file mode 100644 index 0000000..9aad2e2 --- /dev/null +++ b/docs/lab4/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.12, + "lb_detect_down_seconds": 1.56, + "target_last_served_after_stop_seconds": 1.56, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.62, + "lb_back_up_seconds": 1.28, + "target_back_in_rotation_seconds": 7.62, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1155, + "ok": 1155, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 231.0, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1010, + "ok": 1010, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 101.0, + "write_ok_rps": 61.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 588, + "ok": 588, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 117.6, + "write_ok_rps": 43.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 219.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.92, + "requests": 1531, + "ok": 1531, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 140.2, + "write_ok_rps": 82.4, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.56, + [ + "api1" + ] + ], + [ + 21.28, + [ + "api1", + "api2" + ] + ], + [ + 21.61, + [ + "api1" + ] + ], + [ + 24.17, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2426, + "stored_messages": 2426, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.83, + "lb_detect_down_seconds": 1.55, + "target_last_served_after_stop_seconds": 1.39, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.0, + "lb_back_up_seconds": 1.62, + "target_back_in_rotation_seconds": 6.99, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1015, + "ok": 1015, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 203.0, + "write_ok_rps": 121.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 833, + "ok": 833, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 83.3, + "write_ok_rps": 49.3, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 371, + "ok": 371, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 74.2, + "write_ok_rps": 44.2, + "latency_ms_ok": { + "p50": 93.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.27, + "requests": 1496, + "ok": 1496, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 145.7, + "write_ok_rps": 86.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.55, + [ + "api2" + ] + ], + [ + 21.62, + [ + "api1", + "api2" + ] + ], + [ + 22.58, + [ + "api2" + ] + ], + [ + 25.03, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2215, + "stored_messages": 2215, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/failover_series.txt b/docs/lab4/evidence/failover_series.txt new file mode 100644 index 0000000..1a84b38 --- /dev/null +++ b/docs/lab4/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-11-04 14:07:43 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 63.4s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=stop) under load + hook 4.12s | LB DOWN after 1.56 s | errors 0 (-) | RPS before 231.0, outage 101.0, after restore 140.2 | p95 outage 110.0 ms | back in rotation 7.62 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=stop) under load + hook 4.83s | LB DOWN after 1.55 s | errors 0 (-) | RPS before 203.0, outage 83.3, after restore 145.7 | p95 outage 187.0 ms | back in rotation 6.99 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-11-04 14:08:46 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 62.7s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=kill) under load + hook 1.09s | LB DOWN after 1.97 s | errors 6 (502) | RPS before 244.2, outage 77.1, after restore 119.5 | p95 outage 187.0 ms | back in rotation 7.52 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=kill) under load + hook 1.14s | LB DOWN after 2.59 s | errors 6 (502) | RPS before 186.8, outage 82.9, after restore 134.8 | p95 outage 188.0 ms | back in rotation 6.83 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab4/evidence/failover_series_kill.json b/docs/lab4/evidence/failover_series_kill.json new file mode 100644 index 0000000..96f1688 --- /dev/null +++ b/docs/lab4/evidence/failover_series_kill.json @@ -0,0 +1,298 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.09, + "lb_detect_down_seconds": 1.97, + "target_last_served_after_stop_seconds": 0.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.56, + "restore_hook_seconds": 7.53, + "lb_back_up_seconds": 2.0, + "target_back_in_rotation_seconds": 7.52, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1221, + "ok": 1221, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.2, + "write_ok_rps": 145.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 777, + "ok": 771, + "errors": 6, + "error_ratio": 0.0077, + "ok_rps": 77.1, + "write_ok_rps": 40.7, + "latency_ms_ok": { + "p50": 63.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 565, + "ok": 565, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 113.0, + "write_ok_rps": 44.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 281.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.84, + "requests": 1296, + "ok": 1296, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 119.5, + "write_ok_rps": 68.1, + "latency_ms_ok": { + "p50": 31.0, + "p95": 172.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.97, + [ + "api1" + ] + ], + [ + 22.0, + [ + "api1", + "api2" + ] + ], + [ + 22.45, + [ + "api1" + ] + ], + [ + 24.97, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2100, + "stored_messages": 2100, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.14, + "lb_detect_down_seconds": 2.59, + "target_last_served_after_stop_seconds": 0.51, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.59, + "restore_hook_seconds": 6.83, + "lb_back_up_seconds": 1.61, + "target_back_in_rotation_seconds": 6.83, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 934, + "ok": 934, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 186.8, + "write_ok_rps": 111.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 835, + "ok": 829, + "errors": 6, + "error_ratio": 0.0072, + "ok_rps": 82.9, + "write_ok_rps": 40.8, + "latency_ms_ok": { + "p50": 32.0, + "p95": 188.0, + "p99": 1000.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 541, + "ok": 541, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 108.2, + "write_ok_rps": 37.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 203.0, + "p99": 282.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.16, + "requests": 1369, + "ok": 1369, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 134.8, + "write_ok_rps": 77.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.59, + [ + "api2" + ] + ], + [ + 21.61, + [ + "api1", + "api2" + ] + ], + [ + 21.95, + [ + "api2" + ] + ], + [ + 24.53, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 1941, + "stored_messages": 1941, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/instances.txt b/docs/lab4/evidence/instances.txt new file mode 100644 index 0000000..df79b9b --- /dev/null +++ b/docs/lab4/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-11-04 14:04:00 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001993A-1F90-6AEB3B50162-0571-0008 + 2 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019944-1F90-6AEB3B50169-0572-0008 + 3 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001994A-1F90-6AEB3B5017C-0573-0008 + 4 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019958-1F90-6AEB3B50199-0574-0008 + 5 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001995E-1F90-6AEB3B5019C-0575-0008 + 6 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019960-1F90-6AEB3B501B7-0576-0008 + 7 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019966-1F90-6AEB3B501C6-0577-0008 + 8 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019974-1F90-6AEB3B501C9-0578-0008 + 9 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019978-1F90-6AEB3B501E5-0579-0008 +10 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019980-1F90-6AEB3B501F4-057A-0008 +11 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019984-1F90-6AEB3B50205-057B-0008 +12 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019988-1F90-6AEB3B50209-057C-0008 +13 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019998-1F90-6AEB3B5020C-057D-0008 +14 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC120001999C-1F90-6AEB3B50223-057E-0008 +15 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001999E-1F90-6AEB3B50226-057F-0008 +16 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199AE-1F90-6AEB3B50241-0580-0008 +17 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199BA-1F90-6AEB3B50245-0581-0008 +18 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199CA-1F90-6AEB3B50260-0582-0008 +19 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199D2-1F90-6AEB3B50263-0584-0008 +20 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199DE-1F90-6AEB3B50266-0585-0008 +разных реплик: 2 diff --git a/docs/lab4/evidence/lb_log_sample.txt b/docs/lab4/evidence/lb_log_sample.txt new file mode 100644 index 0000000..234f9e0 --- /dev/null +++ b/docs/lab4/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-11-04 14:12:25 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:56696 [04/Nov/2026:14:12:22.325] public api/api2 0/0/0/11/11 201 421 ---- 8/8/7/2/0 0/0 rid=AC120001DD78-1F90-6AEB3D46145-0F07-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56712 [04/Nov/2026:14:12:22.328] public api/api2 0/0/0/10/11 201 421 ---- 8/8/6/1/0 0/0 rid=AC120001DD88-1F90-6AEB3D46148-0F09-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56732 [04/Nov/2026:14:12:22.330] public api/api2 0/0/0/9/9 201 421 ---- 7/7/5/0/0 0/0 rid=AC120001DD9C-1F90-6AEB3D4614A-0F0B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56746 [04/Nov/2026:14:12:22.340] public api/api2 0/0/0/6/6 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DDAA-1F90-6AEB3D46154-0F0D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56734 [04/Nov/2026:14:12:22.349] public api/api2 0/0/1/6/7 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DD9E-1F90-6AEB3D46152-0F0F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56764 [04/Nov/2026:14:12:22.365] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDBC-1F90-6AEB3D4615C-0F11-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56666 [04/Nov/2026:14:12:22.308] public api/api1 0/0/0/88/89 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD5A-1F90-6AEB3D46134-0F04-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56788 [04/Nov/2026:14:12:22.398] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDD4-1F90-6AEB3D4618E-0F13-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56688 [04/Nov/2026:14:12:22.324] public api/api1 0/0/0/84/85 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD70-1F90-6AEB3D46144-0F06-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56802 [04/Nov/2026:14:12:22.410] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE2-1F90-6AEB3D4619A-0F15-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56698 [04/Nov/2026:14:12:22.325] public api/api1 0/0/0/99/100 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD7A-1F90-6AEB3D46145-0F08-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56808 [04/Nov/2026:14:12:22.427] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE8-1F90-6AEB3D461AB-0F17-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56748 [04/Nov/2026:14:12:22.341] public api/api1 0/0/0/156/157 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DDAC-1F90-6AEB3D46155-0F0E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56768 [04/Nov/2026:14:12:22.358] public api/api1 0/0/0/140/140 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DDC0-1F90-6AEB3D46166-0F10-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56784 [04/Nov/2026:14:12:22.395] public api/api1 0/0/0/103/104 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DDD0-1F90-6AEB3D46174-0F12-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56678 [04/Nov/2026:14:12:22.334] public api/api1 0/0/1/164/166 201 421 ---- 8/8/4/4/0 0/0 rid=AC120001DD66-1F90-6AEB3D4613D-0F0C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56728 [04/Nov/2026:14:12:22.328] public api/api1 0/0/0/171/172 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DD98-1F90-6AEB3D46148-0F0A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56800 [04/Nov/2026:14:12:22.406] public api/api1 0/0/0/96/96 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE0-1F90-6AEB3D46196-0F14-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56806 [04/Nov/2026:14:12:22.418] public api/api1 0/0/0/88/88 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE6-1F90-6AEB3D461A2-0F16-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56846 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE0E-1F90-6AEB3D461F5-0F1A-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56856 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/12/12 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE18-1F90-6AEB3D461F5-0F1C-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56872 [04/Nov/2026:14:12:22.502] public api/api1 0/0/0/12/12 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DE28-1F90-6AEB3D461F6-0F1D-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56824 [04/Nov/2026:14:12:22.434] public api/api1 0/0/0/84/84 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DDF8-1F90-6AEB3D461B2-0F18-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56840 [04/Nov/2026:14:12:22.501] public api/api1 0/0/0/17/18 201 421 ---- 7/7/2/1/0 0/0 rid=AC120001DE08-1F90-6AEB3D461F5-0F1B-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56898 [04/Nov/2026:14:12:22.512] public api/api2 0/0/0/6/6 201 421 ---- 6/6/1/0/0 0/0 rid=AC120001DE42-1F90-6AEB3D46200-0F1E-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56860 [04/Nov/2026:14:12:22.518] public api/api2 0/0/1/11/12 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DE1C-1F90-6AEB3D461F6-0F20-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56926 [04/Nov/2026:14:12:22.521] public api/api2 0/0/0/12/12 201 421 ---- 8/8/5/1/0 0/0 rid=AC120001DE5E-1F90-6AEB3D46209-0F24-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56888 [04/Nov/2026:14:12:22.519] public api/api2 0/0/0/14/14 201 421 ---- 7/7/4/0/0 0/0 rid=AC120001DE38-1F90-6AEB3D461FD-0F22-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56874 [04/Nov/2026:14:12:22.519] public api/api1 0/0/0/15/16 201 421 ---- 6/6/5/3/0 0/0 rid=AC120001DE2A-1F90-6AEB3D461F8-0F21-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56916 [04/Nov/2026:14:12:22.520] public api/api1 0/0/0/14/14 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DE54-1F90-6AEB3D46208-0F23-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56904 [04/Nov/2026:14:12:22.516] public api/api1 0/0/0/19/20 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DE48-1F90-6AEB3D46204-0F1F-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56956 [04/Nov/2026:14:12:22.536] public api/api2 0/0/0/12/12 201 421 ---- 8/8/7/3/0 0/0 rid=AC120001DE7C-1F90-6AEB3D46218-0F2B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56942 [04/Nov/2026:14:12:22.535] public api/api2 0/0/0/15/16 201 421 ---- 8/8/6/2/0 0/0 rid=AC120001DE6E-1F90-6AEB3D46217-0F29-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56930 [04/Nov/2026:14:12:22.534] public api/api2 0/0/0/17/17 201 421 ---- 7/7/5/1/0 0/0 rid=AC120001DE62-1F90-6AEB3D46209-0F27-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56984 [04/Nov/2026:14:12:22.538] public api/api2 0/0/0/15/15 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DE98-1F90-6AEB3D4621A-0F2D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57024 [04/Nov/2026:14:12:22.564] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/1/0 0/0 rid=AC120001DEC0-1F90-6AEB3D4622A-0F30-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56998 [04/Nov/2026:14:12:22.565] public api/api2 0/0/0/8/8 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEA6-1F90-6AEB3D46227-0F2F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57044 [04/Nov/2026:14:12:22.596] public api/api2 0/0/0/6/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DED4-1F90-6AEB3D4623E-0F33-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56934 [04/Nov/2026:14:12:22.533] public api/api1 0/0/0/80/80 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DE66-1F90-6AEB3D46215-0F26-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57066 [04/Nov/2026:14:12:22.615] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEEA-1F90-6AEB3D46267-0F35-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56952 [04/Nov/2026:14:12:22.535] public api/api1 0/0/0/88/88 201 421 ---- 8/8/6/6/0 0/0 rid=AC120001DE78-1F90-6AEB3D46217-0F2A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56900 [04/Nov/2026:14:12:22.534] public api/api1 0/0/1/88/90 201 421 ---- 7/7/5/5/0 0/0 rid=AC120001DE44-1F90-6AEB3D46202-0F28-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56970 [04/Nov/2026:14:12:22.537] public api/api1 0/0/0/87/87 201 421 ---- 6/6/4/4/0 0/0 rid=AC120001DE8A-1F90-6AEB3D46219-0F2C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57018 [04/Nov/2026:14:12:22.565] public api/api1 0/0/0/62/63 201 421 ---- 8/8/6/5/0 0/0 rid=AC120001DEBA-1F90-6AEB3D46229-0F31-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57036 [04/Nov/2026:14:12:22.572] public api/api1 0/0/0/55/55 201 421 ---- 7/7/5/4/0 0/0 rid=AC120001DECC-1F90-6AEB3D4623C-0F32-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57014 [04/Nov/2026:14:12:22.553] public api/api1 0/0/0/74/75 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DEB6-1F90-6AEB3D46229-0F2E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57052 [04/Nov/2026:14:12:22.604] public api/api1 0/0/0/23/24 201 421 ---- 5/5/3/2/0 0/0 rid=AC120001DEDC-1F90-6AEB3D4625C-0F34-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57114 [04/Nov/2026:14:12:22.626] public api/api2 0/0/0/7/7 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DF1A-1F90-6AEB3D46272-0F37-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57138 [04/Nov/2026:14:12:22.629] public api/api2 0/0/0/9/9 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DF32-1F90-6AEB3D46275-0F39-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57142 [04/Nov/2026:14:12:22.630] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/0/0 0/0 rid=AC120001DF36-1F90-6AEB3D46276-0F3B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57156 [04/Nov/2026:14:12:22.641] public api/api2 0/0/0/5/5 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF44-1F90-6AEB3D46276-0F3D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57170 [04/Nov/2026:14:12:22.648] public api/api2 0/0/0/4/4 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF52-1F90-6AEB3D46288-0F3F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57168 [04/Nov/2026:14:12:22.657] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DF50-1F90-6AEB3D46282-0F41-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57096 [04/Nov/2026:14:12:22.625] public api/api1 0/0/0/86/86 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DF08-1F90-6AEB3D46271-0F36-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57108 [04/Nov/2026:14:12:22.626] public api/api1 0/0/0/89/89 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DF14-1F90-6AEB3D46272-0F38-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57160 [04/Nov/2026:14:12:22.656] public api/api1 0/0/0/61/61 201 421 ---- 5/5/4/4/0 0/0 rid=AC120001DF48-1F90-6AEB3D4627B-0F42-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57122 [04/Nov/2026:14:12:22.630] public api/api1 0/0/0/91/91 201 421 ---- 4/4/3/3/0 0/0 rid=AC120001DF22-1F90-6AEB3D46276-0F3A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57082 [04/Nov/2026:14:12:22.642] public api/api1 0/0/0/81/81 201 421 ---- 3/3/2/2/0 0/0 rid=AC120001DEFA-1F90-6AEB3D4626F-0F3E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57164 [04/Nov/2026:14:12:22.640] public api/api1 0/0/0/83/83 201 421 ---- 2/2/1/1/0 0/0 rid=AC120001DF4C-1F90-6AEB3D46280-0F3C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57180 [04/Nov/2026:14:12:22.654] public api/api1 0/0/0/69/70 201 421 ---- 1/1/0/0/0 0/0 rid=AC120001DF5C-1F90-6AEB3D4628E-0F40-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" diff --git a/docs/lab4/evidence/load.json b/docs/lab4/evidence/load.json new file mode 100644 index 0000000..281ec48 --- /dev/null +++ b/docs/lab4/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab4/evidence/load.txt b/docs/lab4/evidence/load.txt new file mode 100644 index 0000000..ef62856 --- /dev/null +++ b/docs/lab4/evidence/load.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:12:22 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.6s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-11-04 14:12:24 UTC | CPU/RAM после нагрузки | exit 0 | 2.1s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-api-6 17.48% 88.79MiB / 320MiB 27.75% +messenger-lab4-api-5 17.67% 89.4MiB / 320MiB 27.94% +messenger-lab4-lb-1 2.18% 14.14MiB / 128MiB 11.05% +messenger-lab4-postgres-1 0.14% 68.93MiB / 512MiB 13.46% +messenger-lab4-redis-1 2.43% 4.82MiB / 96MiB 5.02% +messenger-lab4-s3-1 0.31% 97.91MiB / 512MiB 19.12% + diff --git a/docs/lab4/evidence/persistence.txt b/docs/lab4/evidence/persistence.txt new file mode 100644 index 0000000..e453f74 --- /dev/null +++ b/docs/lab4/evidence/persistence.txt @@ -0,0 +1,48 @@ +# 2026-11-04 14:11:03 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 32 seconds (healthy) 32 seconds ago +messenger-lab4-api-6 Up 20 seconds (healthy) 21 seconds ago +messenger-lab4-lb-1 Up 11 minutes (healthy) 13 minutes ago +messenger-lab4-migrate-1 Exited (0) 6 minutes ago 13 minutes ago +messenger-lab4-postgres-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-redis-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-s3-1 Up About a minute (healthy) 13 minutes ago +messenger-lab4-s3-init-1 Exited (0) 6 minutes ago 13 minutes ago +# 2026-11-04 14:11:49 UTC | Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) | exit 0 | 45.6s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:11:49 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.2s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 7 seconds (healthy) About a minute ago +messenger-lab4-api-6 Up 7 seconds (healthy) About a minute ago +messenger-lab4-lb-1 Up 1 second (health: starting) 14 minutes ago +messenger-lab4-migrate-1 Exited (0) 7 minutes ago 14 minutes ago +messenger-lab4-postgres-1 Up 38 seconds (healthy) 14 minutes ago +messenger-lab4-redis-1 Up 39 seconds (healthy) 14 minutes ago +messenger-lab4-s3-1 Up 28 seconds (healthy) 14 minutes ago +messenger-lab4-s3-init-1 Exited (0) 7 minutes ago 14 minutes ago + +# 2026-11-04 14:11:52 UTC | Persistence: сверка БД <-> S3 после рестарта | exit 0 | 2.4s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-e029350f1930 Creating + Container messenger-lab4-api-run-e029350f1930 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 182, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.062, + "ready_rows": 182, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.073 +} + diff --git a/docs/lab4/evidence/reconcile.txt b/docs/lab4/evidence/reconcile.txt new file mode 100644 index 0000000..2147d08 --- /dev/null +++ b/docs/lab4/evidence/reconcile.txt @@ -0,0 +1,19 @@ +# 2026-11-04 14:04:52 UTC | Сверка БД <-> S3 (dry-run) | exit 0 | 2.2s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-fb929425df7f Creating + Container messenger-lab4-api-run-fb929425df7f Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 107, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.014, + "ready_rows": 107, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.025 +} diff --git a/docs/lab4/evidence/rolling_back.json b/docs/lab4/evidence/rolling_back.json new file mode 100644 index 0000000..0f4c8bb --- /dev/null +++ b/docs/lab4/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.28, + "replicas_before": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "replicas_after": [ + "4749f3b4b256", + "7d965c2245c9" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1026, + "ok": 1026, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.2, + "write_ok_rps": 123.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.28, + "requests": 5808, + "ok": 5808, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.4, + "write_ok_rps": 122.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "1c20a55eebeb", + "4749f3b4b256", + "7d965c2245c9", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 33.28, + "to_s": 41.34, + "requests": 1492, + "ok": 1492, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 185.1, + "write_ok_rps": 111.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4749f3b4b256", + "7d965c2245c9" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api3" + ] + ], + [ + 11.78, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.7, + [ + "api1", + "api3" + ] + ], + [ + 15.26, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 18.2, + [ + "api1", + "api2" + ] + ], + [ + 25.8, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.51, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 4970, + "stored_messages": 4970, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab4/evidence/rolling_update.json b/docs/lab4/evidence/rolling_update.json new file mode 100644 index 0000000..7c74c98 --- /dev/null +++ b/docs/lab4/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 29.38, + "replicas_before": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "replicas_after": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1180, + "ok": 1180, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 236.0, + "write_ok_rps": 140.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 34.38, + "requests": 6657, + "ok": 6657, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 226.6, + "write_ok_rps": 133.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "4fc3ad11e268", + "c9fab9c9e2c1", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 34.38, + "to_s": 42.45, + "requests": 1674, + "ok": 1674, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 207.2, + "write_ok_rps": 121.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 110.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 12.44, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 13.16, + [ + "api1", + "api2" + ] + ], + [ + 15.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.44, + [ + "api2", + "api3" + ] + ], + [ + 26.94, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 31.47, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5617, + "stored_messages": 5617, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab4/evidence/rolling_update.txt b/docs/lab4/evidence/rolling_update.txt new file mode 100644 index 0000000..03b1ccc --- /dev/null +++ b/docs/lab4/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:10:19 UTC | Rolling update lab4 -> lab4.1 под нагрузкой | exit 0 | 44.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['4fc3ad11e268', 'c9fab9c9e2c1'] +2026-11-04T14:09:41.527Z rolling-update: start; replicas=2 old=['c9fab9c9e2c1', '4fc3ad11e268'] +2026-11-04T14:09:45.307Z rolling-update: images built in 3.8s (tag lab4.1) +2026-11-04T14:09:47.489Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:09:59.096Z rolling-update: step 1/2 new=1c20a55eebeb (messenger-api:lab4.1, ready+UP in 7.0s) replaced old=c9fab9c9e2c1 (graceful stop 4.4s) +2026-11-04T14:10:09.967Z rolling-update: step 2/2 new=ee285aa5856f (messenger-api:lab4.1, ready+UP in 6.6s) replaced old=4fc3ad11e268 (graceful stop 4.0s) +2026-11-04T14:10:10.389Z rolling-update: done in 29.1s; replicas now ['1c20a55eebeb', 'ee285aa5856f'] +update 29.4s rc=0 | replicas ['4fc3ad11e268', 'c9fab9c9e2c1'] -> ['1c20a55eebeb', 'ee285aa5856f'] | errors during update 0 (-) | RPS before 236.0, during 226.6, after 207.2 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-11-04 14:11:03 UTC | Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой | exit 0 | 43.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:26.009Z rolling-update: start; replicas=2 old=['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:28.537Z rolling-update: images built in 2.5s (tag lab4) +2026-11-04T14:10:30.771Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:10:42.303Z rolling-update: step 1/2 new=4749f3b4b256 (messenger-api:lab4, ready+UP in 7.2s) replaced old=1c20a55eebeb (graceful stop 4.1s) +2026-11-04T14:10:53.630Z rolling-update: step 2/2 new=7d965c2245c9 (messenger-api:lab4, ready+UP in 7.0s) replaced old=ee285aa5856f (graceful stop 4.2s) +2026-11-04T14:10:54.052Z rolling-update: done in 28.0s; replicas now ['4749f3b4b256', '7d965c2245c9'] +update 28.3s rc=0 | replicas ['1c20a55eebeb', 'ee285aa5856f'] -> ['4749f3b4b256', '7d965c2245c9'] | errors during update 0 (-) | RPS before 205.2, during 205.4, after 185.1 | p95 during 94.0 ms | data ok=True +PASS: rolling update under load + +# 2026-11-04 14:11:03 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4749f3b4b256 messenger-lab4-api-5 running healthy messenger-api:lab4 api2 UP +7d965c2245c9 messenger-lab4-api-6 running healthy messenger-api:lab4 api3 UP + diff --git a/docs/lab4/evidence/s3_bootstrap.txt b/docs/lab4/evidence/s3_bootstrap.txt new file mode 100644 index 0000000..52e17d5 --- /dev/null +++ b/docs/lab4/evidence/s3_bootstrap.txt @@ -0,0 +1,225 @@ +# 2026-11-04 14:03:57 UTC | S3 bootstrap: лог первого запуска s3-init | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color s3-init +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +# 2026-11-04 14:04:00 UTC | S3 bootstrap: повторный запуск (идемпотентность) | exit 0 | 2.3s +$ docker compose --env-file .env run --rm -T s3-init + Container messenger-lab4-s3-1 Running + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-s3-init-run-8fa89d92841c Creating + Container messenger-lab4-s3-init-run-8fa89d92841c Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} + diff --git a/docs/lab4/evidence/s3_outage.json b/docs/lab4/evidence/s3_outage.json new file mode 100644 index 0000000..465a6c6 --- /dev/null +++ b/docs/lab4/evidence/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 20.34, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.53s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "6.53s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "6.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 3.66s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "3.66s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "20.34s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/s3_outage.txt b/docs/lab4/evidence/s3_outage.txt new file mode 100644 index 0000000..bcb31b0 --- /dev/null +++ b/docs/lab4/evidence/s3_outage.txt @@ -0,0 +1,24 @@ +# 2026-11-04 14:09:34 UTC | Недоступность S3 и восстановление | exit 0 | 48.7s +$ python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json +PASS baseline upload 201 — 201 +$ docker compose --env-file .env stop s3 +PASS readiness stays 200 while S3 is down (S3 is not a readiness dependency) — 200 +PASS text message 201 during S3 outage — 201 +PASS history 200 during S3 outage — 200 +PASS attachment metadata 200 (PostgreSQL only) — 200 +PASS message referencing existing attachment 201 — 201 +PASS upload #1 -> 503 unavailable — 503 unavailable in 6.53s +PASS upload #1 failed fast (<= 15s) — 6.53s +PASS upload #1 has Retry-After +PASS upload #2 -> 503 unavailable — 503 unavailable in 6.31s +PASS upload #2 failed fast (<= 15s) — 6.31s +PASS upload #2 has Retry-After +PASS download -> 503 during outage — 503 in 3.66s +PASS download failed fast — 3.66s +PASS session still valid — 200 +$ docker compose --env-file .env start s3 +$ C:\Users\mrend\AppData\Local\Programs\Python\Python312\python.exe scripts/ops.py wait-s3 +PASS S3 read/write warm-up after start — 20.34s +PASS old file downloads byte-for-byte after recovery — 200 0.02s +PASS new upload 201 after recovery (same token, no API restart) — 201 +PASS diff --git a/docs/lab4/evidence/s3_private.txt b/docs/lab4/evidence/s3_private.txt new file mode 100644 index 0000000..4a729d3 --- /dev/null +++ b/docs/lab4/evidence/s3_private.txt @@ -0,0 +1,67 @@ +# 2026-11-04 14:04:54 UTC | Приватный бакет вне API, права ключей, закрытые админ-порты | exit 0 | 2.3s +$ python scripts/ops.py s3-check + Container messenger-lab4-api-run-33922d9bc56f Creating + Container messenger-lab4-api-run-33922d9bc56f Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} +# 2026-11-04 14:04:54 UTC | На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes | exit 0 | 0.5s +$ python scripts/ops.py api-fs +messenger-lab4-api-1 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] +messenger-lab4-api-2 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] + +# 2026-11-04 14:04:55 UTC | API: файлы в /tmp (tmpfs) после загрузок | exit 0 | 0.3s +$ docker compose --env-file .env exec -T api sh -c df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l) +Filesystem Size Used Avail Use% Mounted on +tmpfs 16M 0 16M 0% /tmp +files_in_tmp=0 + diff --git a/docs/lab4/evidence/sandbox/check.txt b/docs/lab4/evidence/sandbox/check.txt new file mode 100644 index 0000000..395bd0f --- /dev/null +++ b/docs/lab4/evidence/sandbox/check.txt @@ -0,0 +1,6 @@ +# 2026-11-02T13:36:48Z | offline contract +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-02T13:36:48Z | full OpenAPI validation +$ python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/docs/lab4/evidence/sandbox/consistency.txt b/docs/lab4/evidence/sandbox/consistency.txt new file mode 100644 index 0000000..3ca0db1 --- /dev/null +++ b/docs/lab4/evidence/sandbox/consistency.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:02Z | DB<->S3 consistency, aborted upload, injected faults (api-11 s3_error_on_put, api-12 db_error_after_s3_put, api-13 crash_after_s3_put) +$ BASE_URL=... FAULT_*_URL=... python -m unittest discover -s student-tests -p test_attachment_consistency.py -v +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.391s + +OK diff --git a/docs/lab4/evidence/sandbox/failover.txt b/docs/lab4/evidence/sandbox/failover.txt new file mode 100644 index 0000000..d68450f --- /dev/null +++ b/docs/lab4/evidence/sandbox/failover.txt @@ -0,0 +1,12 @@ +# 2026-11-02T13:28:02Z failover run 1: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 4.468} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:27:47Z stop-one api-1 pid 7684 +restored; next run +# 2026-11-02T13:28:23Z failover run 2: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 0, "first_recovery_seconds": 4.475} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:28:08Z stop-one api-1 pid 7937 +restored; next run diff --git a/docs/lab4/evidence/sandbox/failover_series.json b/docs/lab4/evidence/sandbox/failover_series.json new file mode 100644 index 0000000..c69f3b3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/failover_series.json @@ -0,0 +1,268 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "api-1", + "api-2" + ], + "rounds": [ + { + "target_instance": "api-1", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.0, + "lb_detect_down_seconds": 1.3, + "target_last_served_after_stop_seconds": 1.31, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.86, + "lb_back_up_seconds": 2.91, + "target_back_in_rotation_seconds": 2.87, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1597, + "ok": 1597, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 318.1, + "write_ok_rps": 187.8, + "latency_ms_ok": { + "p50": 16.7, + "p95": 38.6, + "p99": 50.6 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 2392, + "ok": 2392, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 239.2, + "write_ok_rps": 140.9, + "latency_ms_ok": { + "p50": 24.4, + "p95": 39.5, + "p99": 48.8 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 1057, + "ok": 1057, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 211.4, + "write_ok_rps": 124.2, + "latency_ms_ok": { + "p50": 27.6, + "p95": 44.5, + "p99": 54.8 + }, + "instances": [ + "api-2" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 26.15, + "requests": 1671, + "ok": 1671, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 272.8, + "write_ok_rps": 160.7, + "latency_ms_ok": { + "p50": 21.2, + "p95": 39.0, + "p99": 49.4 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api-1", + "api-2" + ] + ], + [ + 6.32, + [ + "api-2" + ] + ], + [ + 22.93, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 3957, + "stored_messages": 3957, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "api-2", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.23, + "lb_detect_down_seconds": 1.53, + "target_last_served_after_stop_seconds": 1.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.45, + "lb_back_up_seconds": 3.05, + "target_back_in_rotation_seconds": 3.05, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.01, + "requests": 1416, + "ok": 1416, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 282.9, + "write_ok_rps": 164.8, + "latency_ms_ok": { + "p50": 19.4, + "p95": 41.3, + "p99": 53.4 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.01, + "to_s": 15.01, + "requests": 2431, + "ok": 2431, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 243.1, + "write_ok_rps": 142.8, + "latency_ms_ok": { + "p50": 24.8, + "p95": 39.6, + "p99": 46.3 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.01, + "to_s": 20.01, + "requests": 1154, + "ok": 1154, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 230.7, + "write_ok_rps": 136.0, + "latency_ms_ok": { + "p50": 25.4, + "p95": 39.9, + "p99": 46.5 + }, + "instances": [ + "api-1" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 26.3, + "requests": 1851, + "ok": 1851, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 293.9, + "write_ok_rps": 172.6, + "latency_ms_ok": { + "p50": 19.8, + "p95": 35.8, + "p99": 42.1 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.01, + [ + "api-1", + "api-2" + ] + ], + [ + 6.53, + [ + "api-1" + ] + ], + [ + 23.06, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 4020, + "stored_messages": 4020, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/load.json b/docs/lab4/evidence/sandbox/load.json new file mode 100644 index 0000000..f7d5a05 --- /dev/null +++ b/docs/lab4/evidence/sandbox/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.055, + "concurrency": 4, + "successful_requests": 5544, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 276.435, + "successful_latency_ms": { + "p50": 13.241, + "p95": 23.756, + "p99": 32.552 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab4/evidence/sandbox/persistence.txt b/docs/lab4/evidence/sandbox/persistence.txt new file mode 100644 index 0000000..707b5ef --- /dev/null +++ b/docs/lab4/evidence/sandbox/persistence.txt @@ -0,0 +1,7 @@ +# 2026-11-02T13:24:54Z native sandbox: python3 tests/resilience.py --mode persistence --action-script +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# hook log: PIDs before/after (postgres, seaweedfs, api x2) + S3 warm-up +2026-11-02T13:24:15Z restart begin pg=6347 s3=6937 api=6405,6525 +{"s3_ready_after_s": 16.4, "last_cycle_s": 1.028, "existing_object_read": true} +2026-11-02T13:24:53Z restart end pg=7134 s3=7149 api=7195,7219 diff --git a/docs/lab4/evidence/sandbox/reconcile.txt b/docs/lab4/evidence/sandbox/reconcile.txt new file mode 100644 index 0000000..78029ae --- /dev/null +++ b/docs/lab4/evidence/sandbox/reconcile.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:06Z | reconcile dry-run +$ python -m src.reconcile +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 238, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.036, + "ready_rows": 238, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.045 +} diff --git a/docs/lab4/evidence/sandbox/s3_bootstrap.txt b/docs/lab4/evidence/sandbox/s3_bootstrap.txt new file mode 100644 index 0000000..2a46dc3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_bootstrap.txt @@ -0,0 +1,45 @@ +# 2026-11-02T13:36:49Z | S3 bootstrap, 2nd run on existing bucket (idempotent) +$ python -m src.s3_bootstrap +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} diff --git a/docs/lab4/evidence/sandbox/s3_outage.json b/docs/lab4/evidence/sandbox/s3_outage.json new file mode 100644 index 0000000..21afe8a --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.42, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 1.75s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "1.75s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 2.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "2.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 0.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "0.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.42s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.01s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/s3_outage_hang.json b/docs/lab4/evidence/sandbox/s3_outage_hang.json new file mode 100644 index 0000000..4f58582 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_outage_hang.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.09, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.02s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "10.02s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.01s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "10.01s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 5.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "5.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.09s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/s3_private.txt b/docs/lab4/evidence/sandbox/s3_private.txt new file mode 100644 index 0000000..7398970 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_private.txt @@ -0,0 +1,54 @@ +# 2026-11-02T13:37:07Z | private bucket outside API (anonymous S3 requests), key permissions, admin ports +$ python -m src.s3_bootstrap --check-only --sample-from-db +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} diff --git a/docs/lab4/evidence/sandbox/smoke.txt b/docs/lab4/evidence/sandbox/smoke.txt new file mode 100644 index 0000000..c0127ee --- /dev/null +++ b/docs/lab4/evidence/sandbox/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-02T13:36:50Z | public smoke +$ python3 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab4/evidence/sandbox/student-tests.txt b/docs/lab4/evidence/sandbox/student-tests.txt new file mode 100644 index 0000000..bd5e966 --- /dev/null +++ b/docs/lab4/evidence/sandbox/student-tests.txt @@ -0,0 +1,120 @@ +# 2026-11-02T13:37:16Z | student tests over HTTP via LB (lab2-4); DB-level tests skip here and run below with DB access +$ python3 student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... + 24 parallel POSTs, one key -> 1 message; replicas: ['api-1', 'api-2'] +ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 6.028s + +OK (skipped=23) + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# DB-level tests (lab2-3) with direct DB access +$ python -m unittest discover -s student-tests -p "test_storage_*.py"; ... -p test_session_cache.py +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.022s + +OK +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.564s + +OK diff --git a/docs/lab4/evidence/sandbox/upload_memory_c10.json b/docs/lab4/evidence/sandbox/upload_memory_c10.json new file mode 100644 index 0000000..15d8fa0 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c10.json @@ -0,0 +1,43 @@ +{ + "label": "final code; peak via VmHWM and 20ms sampling", + "date_utc": "2026-11-02T13:41:45Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 5.06, + "latency_201_s": { + "p50": 1.316, + "p95": 2.069, + "max": 2.088280546000078, + "mean": 1.407 + }, + "memory": { + "pid-9293": { + "baseline_mib": 95.5, + "peak_mib": 163.4, + "growth_mib": 67.9, + "samples": 356 + }, + "pid-9364": { + "baseline_mib": 95.4, + "peak_mib": 156.1, + "growth_mib": 60.6, + "samples": 356 + } + }, + "process_peak_rss_mib": { + "pid-9293": 165.8, + "pid-9364": 165.3 + } +} diff --git a/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json b/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json new file mode 100644 index 0000000..dc21474 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-02T13:11:09Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.14, + "latency_201_s": { + "p50": 2.159, + "p95": 3.077, + "max": 3.089563180000141, + "mean": 2.079 + }, + "memory": { + "pid-3762": { + "baseline_mib": 114.5, + "peak_mib": 161.7, + "growth_mib": 47.2, + "samples": 387 + }, + "pid-3834": { + "baseline_mib": 96.5, + "peak_mib": 159.1, + "growth_mib": 62.6, + "samples": 387 + } + } +} diff --git a/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json b/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json new file mode 100644 index 0000000..84924f3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=64 (limiter effectively off)", + "date_utc": "2026-11-02T13:11:34Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.06, + "latency_201_s": { + "p50": 2.901, + "p95": 3.031, + "max": 3.03438846999984, + "mean": 2.702 + }, + "memory": { + "pid-4062": { + "baseline_mib": 88.2, + "peak_mib": 261.6, + "growth_mib": 173.4, + "samples": 400 + }, + "pid-4133": { + "baseline_mib": 88.7, + "peak_mib": 291.4, + "growth_mib": 202.8, + "samples": 400 + } + } +} diff --git a/docs/lab4/evidence/smoke.txt b/docs/lab4/evidence/smoke.txt new file mode 100644 index 0000000..8d23ad7 --- /dev/null +++ b/docs/lab4/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-04 14:04:01 UTC | Публичный smoke | exit 0 | 0.8s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab4/evidence/stand.txt b/docs/lab4/evidence/stand.txt new file mode 100644 index 0000000..d4cd20d --- /dev/null +++ b/docs/lab4/evidence/stand.txt @@ -0,0 +1,42 @@ +# 2026-11-04 14:03:55 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab4-api-1 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-api-2 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab4-migrate-1 messenger-lab4-migrate "python -m src.migra…" migrate 6 minutes ago Exited (0) 2 minutes ago +messenger-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 6 minutes ago Up 5 minutes (healthy) 5432/tcp +messenger-lab4-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 6 minutes ago Up 5 minutes (healthy) 6379/tcp +messenger-lab4-s3-1 chrislusf/seaweedfs:4.50 "/usr/bin/weed serve…" s3 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +messenger-lab4-s3-init-1 messenger-lab4-s3-init "python -m src.s3_bo…" s3-init 6 minutes ago Exited (0) 2 minutes ago +# 2026-11-04 14:03:55 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:03:57 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.4s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-lb-1 0.14% 13.6MiB / 128MiB 10.63% +messenger-lab4-api-2 0.26% 90.05MiB / 320MiB 28.14% +messenger-lab4-api-1 0.30% 106MiB / 320MiB 33.13% +messenger-lab4-postgres-1 0.13% 71.34MiB / 512MiB 13.93% +messenger-lab4-redis-1 2.31% 5.629MiB / 96MiB 5.86% +messenger-lab4-s3-1 0.21% 508.3MiB / 512MiB 99.28% + +# 2026-11-04 14:03:57 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.2s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp +s3 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp + +# 2026-11-04 14:03:57 UTC | Образ SeaweedFS (закреплённая версия) | exit 0 | 0.3s +$ docker compose --env-file .env images s3 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +messenger-lab4-s3-1 chrislusf/seaweedfs 4.50 linux/amd64 c2658820e763 197MB 10 days ago + diff --git a/docs/lab4/evidence/student-tests.txt b/docs/lab4/evidence/student-tests.txt new file mode 100644 index 0000000..edbc587 --- /dev/null +++ b/docs/lab4/evidence/student-tests.txt @@ -0,0 +1,168 @@ +# 2026-11-04 14:04:10 UTC | Собственные тесты (HTTP через LB, включая test_attachments.py) | exit 0 | 9.4s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 7.352s + +OK (skipped=23) + + 24 parallel POSTs, one key -> 1 message; replicas: ['4fc3ad11e268', 'c9fab9c9e2c1'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-11-04 14:04:18 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.9s +$ python scripts/ops.py in-network storage + Container messenger-lab4-api-run-7c01226dcaba Creating + Container messenger-lab4-api-run-7c01226dcaba Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.302s + +OK + Container messenger-lab4-api-run-b6f4e53090a5 Creating + Container messenger-lab4-api-run-b6f4e53090a5 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.860s + +OK + +# 2026-11-04 14:04:33 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 15.2s +$ python scripts/ops.py in-network expiry + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-short-ttl-1 Creating + Container messenger-lab4-api-short-ttl-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-short-ttl-1 Starting + Container messenger-lab4-api-short-ttl-1 Started + Container messenger-lab4-api-run-ae85ff205e89 Creating + Container messenger-lab4-api-run-ae85ff205e89 Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab4-api-run-87803f7efa19 Creating + Container messenger-lab4-api-run-87803f7efa19 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.644s + +OK + diff --git a/docs/lab4/evidence/upload_memory.txt b/docs/lab4/evidence/upload_memory.txt new file mode 100644 index 0000000..5eb2de4 --- /dev/null +++ b/docs/lab4/evidence/upload_memory.txt @@ -0,0 +1,108 @@ +# 2026-11-04 14:05:09 UTC | Пик памяти: 10 одновременных загрузок по 9.5 MiB | exit 0 | 14.2s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 10 --rounds 3 --docker --label c=10, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c10.json +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} +# 2026-11-04 14:05:29 UTC | Пик памяти: 20 одновременных загрузок | exit 0 | 20.4s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 20 --rounds 2 --docker --label c=20, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c20.json +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} + diff --git a/docs/lab4/evidence/upload_memory_c10.json b/docs/lab4/evidence/upload_memory_c10.json new file mode 100644 index 0000000..0505146 --- /dev/null +++ b/docs/lab4/evidence/upload_memory_c10.json @@ -0,0 +1,51 @@ +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/docs/lab4/evidence/upload_memory_c20.json b/docs/lab4/evidence/upload_memory_c20.json new file mode 100644 index 0000000..5093d11 --- /dev/null +++ b/docs/lab4/evidence/upload_memory_c20.json @@ -0,0 +1,52 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/docs/lab5/REPORT.md b/docs/lab5/REPORT.md new file mode 100644 index 0000000..ebbb567 --- /dev/null +++ b/docs/lab5/REPORT.md @@ -0,0 +1,257 @@ +# Отчёт — лабораторная 5 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, **redis-py 5.2.1 (Redis Streams)**, argon2-cffi 23.1.0, boto3 1.35.99, **Pillow 11.1.0 (JPEG/WebP, ImageOps.exif_transpose)**; PostgreSQL 16.4, Redis 7.4.1 (кэш сессий **и отдельный брокер**), HAProxy 2.8.30, SeaweedFS 4.50. Документация: Redis Streams (XADD, XREADGROUP, XACK, XAUTOCLAIM, XPENDING, XTRIM MINID), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (различие publisher confirm и consumer ack), шаблон transactional outbox, AWS Architecture Blog «Exponential Backoff And Jitter», Pillow (draft, reduce/reducing_gap, сохранение JPEG/WebP), Python `multiprocessing` (forkserver) и `resource` (RLIMIT_AS/RLIMIT_CPU). Помощь ИИ (Claude) — при проектировании outbox/аренд/fencing, написании worker, тестов отказов и отчёта. + +Минимум lab4 перенесён целиком (балансировщик, ≥2 реплики, сессии, идемпотентность, приватный S3, проверки изображений, сборщик БД↔S3). Единственное объявленное изменение API: `POST /chats/{id}/attachments` отвечает **202 queued** вместо 201 ready. + +## Запуск из чистого clone + +**Версии.** API и worker — один образ из `Dockerfile` (`python:3.12.7-slim-bookworm`); `postgres:16.4-alpine`; `redis:7.4.1-alpine` (два сервиса: `redis` — кэш сессий, `broker` — очередь); `haproxy:2.8.30-alpine`; `chrislusf/seaweedfs:4.50`. Нужны Docker Engine/Desktop с Compose v2 и Python 3.9+ на хосте (`scripts/ops.py` и тесты — только стандартная библиотека). Оркестрация — **Compose** (Swarm не используется). + +```sh +$env:PYTHONUTF8 = "1" +# 1. Секреты (один раз; secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py # lab5 добавляет secrets/broker_password и secrets/broker_url +cp .env.example .env # Windows: copy .env.example .env + +# 2. Сборка и запуск: +# postgres, redis, broker -> migrate (001..005); s3 -> s3-init (bucket) -> api x2 + worker -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Готовность: 2 реплики healthy и UP в HAProxy, S3 реально пишет/читает, worker healthy +python scripts/ops.py wait +python scripts/ops.py wait-s3 +python scripts/ops.py worker-start # идемпотентно: дождаться healthy worker +python scripts/ops.py ps +``` + +Всё то же одной командой: `make up` (= init_secrets + config + up --build + ожидания). Прогрев после первой сборки ~30–40 с. + +```sh +# Публичные проверки +make check # офлайн, без сервера +make test # smoke lab5: 202 -> ready -> thumbnail 256x160 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh # Windows: scripts\restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh # Windows: scripts\stop-one.cmd +python scripts/ops.py restore + +# Собственные проверки lab5 +make test-pipeline # варианты по фактическим байтам (7 тестов) +make test-worker-outage # сценарий защиты уровня 3 (останавливает/запускает worker, рестарт API и broker) +make test-pipeline-faults # 10 внедрённых отказов уровня 4 (~2–3 мин; worker временно пересоздаётся) +python student-tests/run_all.py --base-url http://localhost:8080 # lab2-4 (+ test_pipeline.py) + +# Всё сразу с записью в evidence/ (~20 мин; только на учебном стенде) +python scripts/collect_evidence.py # или --quick (~10 мин) +``` + +**Операторские команды очереди** (выполняются в one-off контейнере образа worker, секреты не печатаются): + +```sh +python scripts/ops.py jobs stats # задания по статусам, outbox, DLQ, длина/pending/lag потока +python scripts/ops.py jobs failed # реестр dead_letters +python scripts/ops.py jobs show # задание + журнал событий (job_events) +python scripts/ops.py jobs replay # явный replay failed (make replay ID=...) +python scripts/ops.py jobs replay --all-failed --error-code retry_exhausted --dry-run +python scripts/ops.py worker-stop | worker-start | worker-kill | api-restart | broker-restart +python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (staging/, final/, originals/) +``` + +**Остановка.** Безопасная (volumes `pgdata`, `s3data`, `brokerdata` сохраняются): `make down` или `docker compose --env-file .env --profile ttl --profile faults down`. **Намеренное удаление всех учебных данных:** `docker compose --env-file .env --profile ttl --profile faults down -v` (удаляет БД, бакет и очередь безвозвратно). + +## Архитектура и решения + +``` +клиент ──HTTP──> HAProxy ──> API x2 ──[1] INSERT pending ─────────────────────────> PostgreSQL + ├──[2] PUT staging/ (Content-MD5) ───────> S3 (SeaweedFS) + ├──[3] COMMIT: queued + attachment_jobs + job_outbox ──> PostgreSQL + │ └─> 202 queued + └──[4] после ответа: relay XADD (fsync) ─────────> broker (Redis Streams) + │ XREADGROUP (prefetch 1) + worker (WORKER_CONCURRENCY consumer'ов) <───────────────────────────────────────────────┘ + claim: аренда в PostgreSQL (lease_token, lease_until) + читает staging (ключ из БД) -> SHA-256 -> PUT final//original + подпроцесс (RLIMIT_AS, timeout): thumbnail/crop/optimized -> PUT final//p1/ + COMMIT (fenced по lease_token): attachment_variants + state=ready ──> PostgreSQL + XACK ──> broker; DELETE staging (иначе — сборщик) + sweeper outbox + сборщик просроченных аренд: в каждой реплике API и в worker +``` + +**Источник истины** — PostgreSQL: состояние вложения, задание, outbox, результаты, журнал. S3 хранит байты (staging/ и final/), брокер — только транспорт уведомлений (может быть пересоздан из outbox/таблицы заданий). Брокер **не** входит в синхронный путь API: при его недоступности upload всё равно отвечает 202, строка ждёт в `job_outbox`; `/health/ready` от брокера не зависит. + +### Что сохранено в момент 202 + +1. Все байты оригинала в приватном S3 `staging/` (PUT с Content-MD5, SHA-256 в метаданных объекта и в БД). Ключ генерирует сервер, filename в ключ не попадает. +2. Метаданные вложения (`state=queued`, size, sha256, размеры, будущий конечный ключ). +3. Намерение обработать: строка `attachment_jobs` (job_id, attempt=1, max_attempts) и строка `job_outbox` с JSON-сообщением **в той же транзакции**. + +Сообщение в брокере на момент 202 может ещё не существовать — это нормально: его гарантирует outbox, а не HTTP-запрос. Остановка API сразу после 202 работу не теряет (тест A). + +### Состояния и окна отказа + +Внутреннее `pending` (приём идёт) наружу не видно; публичные: `queued → processing → ready | failed`. + +| Окно отказа | Что произойдёт | Где доказано | +| --- | --- | --- | +| после staging PUT до COMMIT задания | строка pending + staging-объект; клиенту ошибка (не 202). Сборщик через grace удаляет и строку, и объект | lab4 `test_attachment_consistency.py` (crash_after_s3_put теперь между staging и COMMIT) | +| после COMMIT до publish | строка outbox осталась; sweeper любого процесса публикует через `RELAY_FAST_PATH_GRACE_SECONDS` | `test_a` (api-fault-publish, `FAULT_INJECTION=crash_after_commit`) | +| после publish до подтверждения в outbox | XADD выполнен, DELETE строки — нет -> повторная публикация, дубль в брокере -> worker no-op | `test_b` (api-fault-dup, `crash_after_publish`) | +| после записи S3 до COMMIT результата | сообщение в PEL; перезапущенный consumer читает свой PEL, перехватывает свою аренду, перезаписывает **те же** байты, COMMIT | `test_c` (`fault-crash-after-effects`) | +| после COMMIT результата до ack | повторная доставка -> claim видит succeeded -> `redelivery_noop already_succeeded`, ack | `test_d` (`fault-crash-before-ack`) | +| worker завис дольше аренды | сборщик аренд -> attempt+1 через outbox -> другой consumer -> ready; проснувшийся старый результат отсечён fencing'ом (`lease_lost`) | `test_h` (`fault-slow`) | + +### Брокер: Redis Streams как durable at-least-once очередь + +Выбран Redis Streams (разрешён README): тот же закреплённый образ, что и для кэша, но **отдельный сервис** с другими гарантиями. Кэш сессий — без persistence и с LRU-вытеснением; объединять их нельзя. + +- **Durability:** `--appendonly yes --appendfsync always` — Redis отвечает на XADD только после fsync AOF. Успешный XADD — наш **publisher confirm**: сообщение переживает рестарт/падение брокера с volume `brokerdata` (проверено в сценарии `worker_outage`: lag=1 до и после рестарта брокера). `--maxmemory-policy noeviction`: поток никогда не вытесняется; при переполнении XADD упадёт, строка останется в outbox. +- **Доставка at-least-once:** consumer group; XREADGROUP кладёт сообщение в PEL до XACK. **Consumer ack** worker делает только после COMMIT результата (или перехода в retry/failed) — это другая граница, чем confirm при публикации. +- **Упавший consumer:** имя consumer = `-<поток>`; после `restart` контейнера hostname тот же, и consumer первым делом читает свой PEL (`XREADGROUP ... 0`). Если контейнер пересоздан, сообщения забирает любой живой consumer через `XAUTOCLAIM` после `JOB_CLAIM_IDLE_SECONDS` (75 с > аренды 60 с). +- **Prefetch/конкуренция:** каждый consumer-поток берёт `COUNT 1`, потоков `WORKER_CONCURRENCY=2`, контейнеров `WORKER_REPLICAS=1` -> одновременно в обработке не больше 2 сообщений. +- **Очистка потока:** `XTRIM MINID` до минимума из (самый старый pending, last-delivered-id+1) — удаляются только доставленные и подтверждённые записи. +- **Содержимое сообщения** — ровно `contracts/jobs.schema.json` (`schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at`): без байтов, credentials, URL и filename. Worker проверяет схему строго; мусор подтверждается и логируется (`test_j`); сообщение, доставленное больше `JOB_MAX_DELIVERIES` раз, переводит задание в `failed poison_message`. + +### Transactional outbox + +`job_outbox` пишется в одной транзакции с `attachment_jobs` и `state=queued`. Публикация: + +- **быстрый путь** — после отправки 202 (FastAPI background task) API берёт свою строку `FOR UPDATE SKIP LOCKED`, делает XADD, удаляет строку, пишет событие `published`, COMMIT; +- **sweeper** — в каждой реплике API и в worker раз в `RELAY_POLL_INTERVAL_SECONDS` публикует строки с `available_at <= now()`. У свежей строки `available_at = now()+2 c`, чтобы не соревноваться с быстрым путём; у retry — время следующей попытки (так реализован отложенный повтор без delayed-очереди брокера). + +`SKIP LOCKED` разносит строки между relay; несколько relay безопасны. Сбой между XADD и COMMIT relay даёт дубль — это и есть at-least-once, дубль гасит идемпотентный worker. + +### Worker, идемпотентность и аренды + +- **Ключ идемпотентности эффекта** — `(attachment_id, pipeline_version, variant)`: PRIMARY KEY `attachment_variants` + **детерминированные ключи S3** `final//original`, `final//p1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор пишет те же байты по тем же ключам (Pillow-вывод детерминирован при одной версии библиотек — проверено в `test_a`/`test_c` сравнением SHA-256 с эталонной загрузкой). `job_id` не меняется при redelivery и retry. +- **Аренда (lease)**: claim атомарно ставит `status=processing, lease_token=, lease_owner=, lease_until=now()+JOB_TIMEOUT_SECONDS` (`SELECT ... FOR UPDATE`). Захват разрешён только для текущей попытки и если задание queued, или аренда истекла, или аренда принадлежит этому же consumer (его прошлая инкарнация мертва: consumer обрабатывает одно сообщение за раз). +- **Fencing**: публикация результата, retry и failed выполняются с условием `lease_token = <мой> AND status='processing'`. Worker, у которого аренду отобрали, ничего не может изменить (`test_h`, событие `lease_lost`). `ready` никогда не откатывается в processing старым дублем; `attachments.pipeline_version` обновляется только если новая версия ≥ текущей. +- **Зависший vs долгий worker:** аренда (60 с) > максимальной длительности попытки (обработка ≤ 30 с + S3-дедлайны), это проверяется при старте конфигурации. Пока аренда не истекла, второй worker задание не возьмёт (`leased_by_other`, ack дубля). Истекла + `JOB_LEASE_GRACE_SECONDS` -> сборщик аренд переводит задание в queued attempt+1 через outbox. Healthcheck worker'а — heartbeat, который пишется, только пока **все** consumer-потоки проходят цикл. +- **Staging** удаляется worker'ом после COMMIT и XACK; если не удалось — сборщик (проход D) удалит staging у ready-вложений. Staging failed-вложений хранится `STAGING_RETENTION_SECONDS` (24 ч) для replay, затем удаляется (replay после этого отклоняется: нужен повторный upload). Текущую работу (queued/processing) сборщик не трогает. + +### Варианты изображений (pipeline_version = 1) + +| Вариант | Формат | Правило | Параметры | +| --- | --- | --- | --- | +| thumbnail | JPEG | вписан в 256×256, пропорции, без увеличения, округление вниз (мин. 1 px): 320×200 → 256×160 | quality 85, optimize, baseline, 4:2:0 | +| crop | JPEG | центрированный квадрат min(w,h) → 128×128 (маленькие увеличиваются) | quality 85 | +| optimized | WebP | вписан в 1280×1280 без увеличения | lossy, quality 80, method 4; уменьшение каждого маленького файла не обещается | + +Общее: EXIF orientation применяется **до** расчёта размеров (400×100 c orientation=6 → thumbnail 64×256); прозрачность компонуется на белый (масштабирование RGBA в Pillow — с premultiplied alpha); **EXIF и XMP не копируются** (Pillow пишет их только при явной передаче), ICC-профиль сохраняется, чтобы не исказить цвет. Целочисленная арифметика размеров исключает ошибки округления float. + +**Ограничения ресурсов worker** (недоверенный файл, даже прошедший проверки API): + +| Ресурс | Ограничение | Поведение | +| --- | --- | --- | +| байты | staging-объект > `MAX_UPLOAD_BYTES` не читается; SHA-256 и размер сверяются с БД | failed `invalid_image` / `processing_failed` | +| пиксели | размеры из заголовка до декодирования, > `MAX_IMAGE_PIXELS` | failed `invalid_image` | +| память | обработка в отдельном процессе (forkserver) с `RLIMIT_AS = WORKER_IMAGE_MEMORY_MB` (1024) | MemoryError/смерть процесса → failed `resource_limit` | +| время | процесс убивается через `WORKER_IMAGE_TIMEOUT_SECONDS` (30), `RLIMIT_CPU` — страховка | временная ошибка `processing_timeout`, повтор ≤ JOB_MAX_ATTEMPTS | + +JPEG декодируется сразу в уменьшенном масштабе (`draft`), поэтому 40 Мп JPEG — пик 46 MiB; худший случай 40 Мп RGBA PNG — **361 MiB** (было 518 MiB, пока альфа компоновалась до масштабирования; порядок операций изменён). Отсюда лимит контейнера worker 1280 MiB при 2 подпроцессах. Замер: `evidence/local/imaging_memory.txt`; проверка самих лимитов: `python -m src.imaging FILE --memory-mb 200` → `resource_limit`, `--timeout 0.3` → `timeout`. + +### Retry, backoff, DLQ, replay + +- **Классификация ошибок.** Временные: S3 недоступен/таймаут, таймаут обработки, внедрённый `injected_transient` → retry. Постоянные: битое/неподдерживаемое изображение, превышение пикселей/байт, лимит памяти, нет источника, несовпадение SHA-256 staging → сразу `failed` без повторов (`test_f`). Если недоступна сама БД, сообщение **не** подтверждается и вернётся из PEL — решение о retry без БД принять нельзя. +- **Backoff с jitter** («equal jitter»): `d = min(JOB_RETRY_MAX_SECONDS, JOB_RETRY_BASE_SECONDS·2^(attempt−1))`, задержка равномерно в `[d/2, d]` (нижняя граница гарантирует паузу, случайная часть разносит повторы многих заданий). Реализован через `job_outbox.available_at`: попытка не начнётся раньше (проверено по журналу в `test_e`). +- **Предел:** `JOB_MAX_ATTEMPTS=5`; затем `failed retry_exhausted`, запись в `dead_letters` (реестр failed/DLQ), событие в `job_events`. +- **Replay** — только явная команда оператора (`ops.py jobs replay`). Разрешён только для `failed`; успешное задание **не исполняется повторно** (`noop`, событие `replay_refused`); без staging-источника — отказ. Новая попытка: attempt+1, свежий бюджет попыток, `replay_count+1`, `dead_letters.replayed_at`, публикация через outbox (`test_g`). + +### Роли и сервисы + +| Роль | Сервис | Версия | Health/readiness | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | haproxy:2.8.30-alpine | `/lb-health`; проверка `/health/ready` реплик | — | backend; 127.0.0.1:8080 (единственный публичный), 8404 stats | — | +| api ×2 | `api` | сборка из Dockerfile | `/health/ready` (PG + схема 005 + не drain) | нет (read-only rootfs, tmpfs /tmp) | backend, storage, queue; 8080 только внутри | secrets: database_url, cache_url, s3_*, broker_url | +| worker | `worker` | тот же образ | heartbeat-файл < 90 с | нет (read-only, tmpfs) | backend, storage, queue | secrets: database_url, s3_*, broker_url | +| broker | `broker` | redis:7.4.1-alpine, AOF always, noeviction | `redis-cli ping` | brokerdata | queue (internal) | secrets: broker_password | +| postgres | `postgres` | postgres:16.4-alpine | pg_isready | pgdata | backend | secrets: postgres_password | +| кэш сессий | `redis` | redis:7.4.1-alpine, без persistence | ping | — | backend | secrets: redis_password | +| s3 | `s3` | chrislusf/seaweedfs:4.50 | /healthz + прогрев `ops.py wait-s3` | s3data | storage (internal) | secrets: s3_config.json | +| bootstrap | `migrate`, `s3-init` | тот же образ | one-shot, идемпотентны | — | backend / storage | — | + +Топология брокера (поток + группа) создаётся кодом идемпотентно (`XGROUP CREATE ... MKSTREAM`, BUSYGROUP = no-op). + +### Переменные `.env.example` → реальная конфигурация + +| Логическое имя (skeleton) | Реализация | +| --- | --- | +| `BROKER_URL_FILE` | `/run/secrets/broker_url` (создаёт `init_secrets.py`) | +| `WORKER_CONCURRENCY=2` | число consumer-потоков в контейнере worker (prefetch 1 на поток) | +| `JOB_MAX_ATTEMPTS=5` | `attachment_jobs.max_attempts` при создании задания | +| `JOB_TIMEOUT_SECONDS=60` | срок аренды; ≥ `WORKER_IMAGE_TIMEOUT_SECONDS + 20` (проверка при старте) | +| `STAGING_RETENTION_SECONDS=86400` | хранение staging failed-вложений (окно replay) | +| дополнительно | `JOB_LEASE_GRACE_SECONDS`, `JOB_CLAIM_IDLE_SECONDS`, `JOB_MAX_DELIVERIES`, `JOB_RETRY_BASE/MAX_SECONDS`, `WORKER_IMAGE_TIMEOUT_SECONDS`, `WORKER_IMAGE_MEMORY_MB`, `RELAY_*`, `BROKER_STREAM/GROUP`, `PIPELINE_VERSION`, `WORKER_REPLICAS` — описаны в `.env.example` | + +### Что изменено относительно lab4 + +- `migrations/005_async_processing.sql`: состояния `queued/processing/failed`, `staging_key`, `error_code`, `pipeline_version`; таблицы `attachment_variants`, `attachment_jobs`, `job_outbox`, `dead_letters`, `job_events`. +- `src/main.py`: upload → staging + задание + outbox → **202**; content: 409 для queued/processing/failed, варианты; сообщение с failed-вложением → 409, с queued/processing — разрешено. +- Новые модули: `src/broker.py`, `src/outbox.py`, `src/jobs.py` (переходы состояний), `src/worker.py`, `src/imaging.py`, `src/jobs_admin.py`. +- `src/reconcile.py`: префиксы `staging/`, `final/`, `originals/` (lab4), проход D — очистка staging. +- `compose.yaml`: сервисы `broker`, `worker`, `api-fault-publish`, `api-fault-dup`, сеть `queue`; `compose.faults.yaml`. +- Тесты lab4, ожидавшие 201, переведены на 202 + ожидание ready (`test_attachments.py`, `test_attachment_consistency.py`, `s3_outage.py`, `upload_memory.py`). Публичные `tests/` и контракт не изменялись. + +## Доказательства критериев + + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab5 public API scenario` (202 → ready, оригинал, JPEG thumbnail 256×160) | evidence/local/smoke.txt | +| Минимум lab4 сохранён | `student-tests/run_all.py` через 2 реплики | 91 тест OK (33 skip — требуют сеть compose), race 100/100, persistence и failover PASS | evidence/local/student-tests.txt, persistence.txt, failover.txt | +| 3: 202 только после durable staging + задания | `test_pipeline.py`, `test_i`, `test_a` | 202 приходит до публикации, задача доходит до ready даже при падении API сразу после 202 | pipeline_and_attachments.txt, pipeline_faults.txt | +| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта | compose: `broker` (AOF always) + `worker`; `test_c`, `test_d` | ack только после COMMIT; падение до ack → повторная доставка без дубля | compose.yaml, pipeline_faults_events.txt | +| 3: worker недоступен → API не делает resize | `make test-worker-outage` | worker остановлен: 202, GET queued, content 409, сообщения 201; после старта worker — ready | worker_outage.txt/.json | +| 3: состояния queued/processing/ready/failed, original неизменен, thumbnail по контракту | `test_pipeline.py` | SHA-256 оригинала совпадает; thumbnail 256×160 по SOF-заголовку JPEG | pipeline_and_attachments.txt | +| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт 2-й набор | `test_e`, `test_g`, `assert_single_result` | ровно 3 строки variants и 1 событие succeeded во всех сценариях; `retry_exhausted`/`invalid_image` | pipeline_faults.txt | +| 3: демонстрация worker stop → upload → рестарт API → рестарт broker → worker → ready | `make test-worker-outage` | lag=1 до и после рестарта брокера, ready через ~0.03 с после старта worker | worker_outage.txt | +| 4: crop 128 и WebP; лимиты bytes/pixels/time/memory | `test_pipeline.py`, `python -m src.imaging` | crop 128×128, WebP по RIFF-заголовку; EXIF удалён; orientation/alpha/малые исходники; `resource_limit`, `timeout` | pipeline_and_attachments.txt, imaging_memory.txt | +| 4: outbox, crash между COMMIT и publish | `test_a` (api-fault-publish) | `fault_injected` от api-fault-publish, `published` другим процессом, attempt=1, ready | pipeline_faults_events.txt | +| 4: backoff+jitter, постоянные/временные, DLQ, безопасный replay | `test_e`, `test_f`, `test_g` | интервалы в [d/2, d]; permanent без повторов; replay → ready; повторный replay → noop | pipeline_faults.txt | +| 4: остановка worker после записи объекта до ack | `test_c`, `test_d` | `lease_takeover` своим consumer после рестарта; те же SHA-256; `redelivery_noop already_succeeded` | worker_fault_log.txt | +| 4: lease для зависшего processing, ограниченная конкуренция, очистка staging | `test_h`, `test_i` | `lease_expired` → attempt 2 у другого consumer → ready; старый `lease_lost`; staging_key=NULL | pipeline_faults_events.txt | + +## Эксперименты и отказы + +Локальная песочница: 2 vCPU, 7 GiB RAM, Linux, Python 3.13 (в образе — 3.12.7), без лимитов cgroup. Набор: синтетические изображения (`tests/fixtures/sample.png` 320×200, генерируемые 40 Мп PNG/JPEG). Повторов: сценарии отказов — 1 полный прогон, все 10 PASS; задержка — 30 последовательных загрузок. Основной compose-стенд: Windows 10, 32GB, Ryzen 5 7500f; доказательства в `/evidence`. + +| Измерение | Результат | +| --- | --- | +| upload → 202 (sample.png, 109 KiB) | p50 16 мс, p95 29 мс | +| upload → ready (опрос каждые 0.2 с) | p50 0.44 с, p95 0.45 с, max 0.45 с | +| пик RSS подпроцесса обработки | 23 MiB (320×200), 46 MiB (40 Мп JPEG), 361 MiB (40 Мп RGBA PNG) | +| время обработки 40 Мп | JPEG 0.2 с, RGBA PNG 1.8 с | + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload | 202 queued, content 409, остальной API работает | так и есть, сообщения 201 | ready после старта worker; потерь нет | +| рестарт API и broker (volumes целы) при неподтверждённой задаче | задача остаётся в потоке | lag=1 до и после | — / нет | +| API умер после COMMIT до XADD | sweeper другого процесса публикует | published другим actor через ~2 с | ≤ RELAY_FAST_PATH_GRACE + 1 с / нет | +| API умер после XADD до подтверждения outbox | дубль, no-op | `redelivery_noop already_succeeded` | — / нет, 1 набор вариантов | +| worker умер после записи S3 до COMMIT | свой PEL после рестарта, те же байты | `lease_takeover`, SHA-256 = эталон | ~1–2 с (рестарт процесса) / нет | +| worker умер после COMMIT до XACK | повторная доставка no-op | `redelivery_noop already_succeeded` | ~1–2 с / нет | +| worker «завис» дольше аренды | новая попытка, fencing старой | `lease_expired` → ready на attempt 2 → `lease_lost` | аренда + grace + ≤5 с / нет | +| временные ошибки ×2 | backoff+jitter, ready на 3-й | задержки 0.5–1 с и 1–2 с (base 1, cap 4 в стенде отказов) | — | +| временная ошибка всегда | failed retry_exhausted, DLQ, replay | 5 попыток → failed → replay → ready → replay ещё раз = noop | — | +| мусор в потоке | ack, worker жив | pending=0 | — | + + +## Ограничения и следующие шаги + +- **Один Docker-host:** брокер, PostgreSQL и S3 — по одному экземпляру. Выдерживаются рестарты и падения процессов с сохранением volumes, но не потеря диска/хоста. Redis без репликации: при потере volume брокера неопубликованные строки outbox сохранятся, а уже опубликованные, но не обработанные задачи потребуют повторной публикации (`jobs replay` работает только для failed; для queued без сообщения нужен ручной requeue — сборщик аренд покрывает только processing). +- **At-least-once + идемпотентность ≠ exactly-once:** дубли сообщений и повторная запись тех же объектов S3 возможны и ожидаемы; уникален только опубликованный результат. Детерминизм байтов вариантов гарантирован в пределах одной версии Pillow/libjpeg/libwebp; при обновлении библиотек нужна новая `pipeline_version` (версионирование нескольких pipeline — трек 5, не заявлен). +- Fencing защищает БД, но не S3: «проснувшийся» старый worker может перезаписать объект варианта теми же (детерминированными) байтами после публикации результата. +- Классификация ошибок упрощена: таймаут обработки считается временным, поэтому «тяжёлый» файл может занять до 5 попыток по 30 с. +- Маркеры отказов в filename работают **только** при `WORKER_FAULTS=true` (`compose.faults.yaml`); в обычном стенде filename на обработку не влияет. +- Старые ready-вложения lab4 остаются без вариантов (запрос варианта → 404 по контракту); фоновой догенерации нет. +- HTTP без TLS — только локальный учебный стенд (TLS — lab7). + +## Вопросы защиты — кратко + +- **Publisher confirm vs consumer ack.** Confirm — брокер принял и сохранил сообщение (у нас: ответ XADD после fsync AOF), это граница «producer → брокер»; после него relay удаляет строку outbox. Ack (XACK) — consumer сообщает, что обработка устойчиво завершена (COMMIT результата/перевода в retry/failed), это граница «брокер → consumer»; до него сообщение остаётся в PEL и будет доставлено снова. +- **Что сохранено в момент 202?** Байты в S3 staging, метаданные, задание и outbox в одной транзакции PostgreSQL. Сообщения в брокере может ещё не быть — его гарантирует outbox. +- **Почему повторная доставка нормальна и где ключ идемпотентности?** At-least-once неизбежно даёт дубли (падение до ack, повтор publish). Ключ эффекта — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` + детерминированные ключи S3; задание — `job_id` (UNIQUE `(attachment_id, pipeline_version)`); повтор успешного — no-op с ack. +- **Как отличить зависший worker от долго работающего?** Аренда с заведомо большим сроком, чем максимальная попытка (жёсткий таймаут подпроцесса + дедлайны S3). Пока аренда действует — задание не отдаётся второму worker; после истечения + grace — новая попытка, а результат старой отсекается fencing-токеном. Heartbeat контейнера проверяет, что все consumer-потоки живы. diff --git a/docs/lab5/evidence/README.md b/docs/lab5/evidence/README.md new file mode 100644 index 0000000..8df7001 --- /dev/null +++ b/docs/lab5/evidence/README.md @@ -0,0 +1,5 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. diff --git a/docs/lab5/evidence/SUMMARY.txt b/docs/lab5/evidence/SUMMARY.txt new file mode 100644 index 0000000..12c408b --- /dev/null +++ b/docs/lab5/evidence/SUMMARY.txt @@ -0,0 +1,54 @@ +# 2026-11-10 14:26:03 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Образ SeaweedFS (закреплённая версия) evidence/stand.txt +OK lab5: broker (Redis AOF) и worker evidence/stand.txt +OK lab5: конфигурация durability брокера evidence/stand.txt +OK S3 bootstrap: лог первого запуска s3-init evidence/s3_bootstrap.txt +OK S3 bootstrap: повторный запуск (идемпотентность) evidence/s3_bootstrap.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB, включая test_attachments.py) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) evidence/consistency.txt +OK Сверка БД <-> S3 (dry-run) evidence/reconcile.txt +OK Приватный бакет вне API, права ключей, закрытые админ-порты evidence/s3_private.txt +OK На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes evidence/s3_private.txt +OK API: файлы в /tmp (tmpfs) после загрузок evidence/s3_private.txt +OK Пик памяти: 10 одновременных загрузок по 9.5 MiB evidence/upload_memory.txt +OK Пик памяти: 20 одновременных загрузок evidence/upload_memory.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Недоступность S3 и восстановление evidence/s3_outage.txt +OK Варианты изображений по фактическим байтам (test_pipeline.py) evidence/pipeline.txt +OK Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready evidence/worker_outage.txt +OK Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) evidence/pipeline_faults.txt +OK Очередь после сценариев: задания, outbox, DLQ, lag evidence/pipeline_faults.txt +OK Реестр failed (DLQ) evidence/pipeline_faults.txt +OK Worker: последние строки лога evidence/worker_log_sample.txt +OK Rolling update lab5 -> lab5.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB) evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Persistence: сверка БД <-> S3 после рестарта evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab5/evidence/check.txt b/docs/lab5/evidence/check.txt new file mode 100644 index 0000000..a85d620 --- /dev/null +++ b/docs/lab5/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-11-10 14:13:49 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-10 14:13:49 UTC | Compose config | exit 0 | 0.3s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab5/evidence/consistency.txt b/docs/lab5/evidence/consistency.txt new file mode 100644 index 0000000..8ecc8f1 --- /dev/null +++ b/docs/lab5/evidence/consistency.txt @@ -0,0 +1,75 @@ +# 2026-11-10 14:15:11 UTC | Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) | exit 0 | 20.0s +$ python scripts/ops.py in-network consistency + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-fault-s3-1 Creating + Container messenger-lab5-api-fault-db-1 Creating + Container messenger-lab5-api-fault-crash-1 Creating + Container messenger-lab5-api-fault-s3-1 Created + Container messenger-lab5-api-fault-db-1 Created + Container messenger-lab5-api-fault-crash-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-db-1 Starting + Container messenger-lab5-api-fault-s3-1 Starting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-crash-1 Starting + Container messenger-lab5-api-fault-db-1 Started + Container messenger-lab5-api-fault-s3-1 Started + Container messenger-lab5-api-fault-crash-1 Started + Container messenger-lab5-api-run-e891dd567ea2 Creating + Container messenger-lab5-api-run-e891dd567ea2 Created +ready: http://api-fault-s3:8080/health/ready + Container messenger-lab5-api-run-a25e14654677 Creating + Container messenger-lab5-api-run-a25e14654677 Created +ready: http://api-fault-db:8080/health/ready + Container messenger-lab5-api-run-53cacd2097d3 Creating + Container messenger-lab5-api-run-53cacd2097d3 Created +ready: http://api-fault-crash:8080/health/ready + Container messenger-lab5-api-run-66f13fe194f4 Creating + Container messenger-lab5-api-run-66f13fe194f4 Created +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.594s + +OK diff --git a/docs/lab5/evidence/failover.txt b/docs/lab5/evidence/failover.txt new file mode 100644 index 0000000..b52781d --- /dev/null +++ b/docs/lab5/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-11-10 14:15:40 UTC | Failover #1: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP +# 2026-11-10 14:16:30 UTC | Failover #1: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 49.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 4.297} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:16:30 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 exited unhealthy messenger-api:lab5 - - + +# 2026-11-10 14:16:37 UTC | Failover #1: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-10T14:16:37.028Z restore: started ['messenger-lab5-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-10 14:16:37 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:16:54 UTC | Failover #2: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.8s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.532} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:16:54 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 exited unhealthy messenger-api:lab5 - - + +# 2026-11-10 14:17:01 UTC | Failover #2: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-10T14:17:00.997Z restore: started ['messenger-lab5-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-10 14:17:01 UTC | Failover #3: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:17:18 UTC | Failover #3: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 17.0s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.437} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:17:18 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 exited unhealthy messenger-api:lab5 - - +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:17:25 UTC | Failover #3: restore | exit 0 | 6.7s +$ python scripts/ops.py restore +2026-11-10T14:17:25.258Z restore: started ['messenger-lab5-api-1']; 2 replicas healthy and UP in LB after 6.6s + +# остановленные публичным тестом реплики: ['3a0e06f14a63', '3a0e06f14a63', '4b1faad0399c'] diff --git a/docs/lab5/evidence/failover_series.json b/docs/lab5/evidence/failover_series.json new file mode 100644 index 0000000..f6d5800 --- /dev/null +++ b/docs/lab5/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "rounds": [ + { + "target_instance": "3a0e06f14a63", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.47, + "lb_detect_down_seconds": 1.88, + "target_last_served_after_stop_seconds": 1.83, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.72, + "lb_back_up_seconds": 0.95, + "target_back_in_rotation_seconds": 6.72, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1083, + "ok": 1083, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 216.6, + "write_ok_rps": 125.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1238, + "ok": 1238, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 123.8, + "write_ok_rps": 74.0, + "latency_ms_ok": { + "p50": 31.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 531, + "ok": 531, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 106.2, + "write_ok_rps": 60.8, + "latency_ms_ok": { + "p50": 63.0, + "p95": 109.0, + "p99": 391.0 + }, + "instances": [ + "4b1faad0399c" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 29.98, + "requests": 1831, + "ok": 1831, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 183.4, + "write_ok_rps": 110.1, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 171.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.88, + [ + "api1" + ] + ], + [ + 20.95, + [ + "api1", + "api2" + ] + ], + [ + 21.92, + [ + "api1" + ] + ], + [ + 24.47, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2768, + "stored_messages": 2768, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "4b1faad0399c", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.55, + "lb_detect_down_seconds": 2.05, + "target_last_served_after_stop_seconds": 1.97, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.06, + "lb_back_up_seconds": 2.08, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1346, + "ok": 1346, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 269.2, + "write_ok_rps": 160.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1371, + "ok": 1371, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 137.1, + "write_ok_rps": 82.7, + "latency_ms_ok": { + "p50": 31.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 564, + "ok": 564, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 112.8, + "write_ok_rps": 66.8, + "latency_ms_ok": { + "p50": 62.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.38, + "requests": 1657, + "ok": 1657, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 159.7, + "write_ok_rps": 90.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.05, + [ + "api2" + ] + ], + [ + 22.08, + [ + "api1", + "api2" + ] + ], + [ + 23.05, + [ + "api2" + ] + ], + [ + 25.51, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2905, + "stored_messages": 2905, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/failover_series.txt b/docs/lab5/evidence/failover_series.txt new file mode 100644 index 0000000..003e978 --- /dev/null +++ b/docs/lab5/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-11-10 14:18:28 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 62.8s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['3a0e06f14a63', '4b1faad0399c'] +-- stopping 3a0e06f14a63 (mode=stop) under load + hook 4.47s | LB DOWN after 1.88 s | errors 0 (-) | RPS before 216.6, outage 123.8, after restore 183.4 | p95 outage 109.0 ms | back in rotation 6.72 s | data ok=True +-- stopping 4b1faad0399c (mode=stop) under load + hook 4.55s | LB DOWN after 2.05 s | errors 0 (-) | RPS before 269.2, outage 137.1, after restore 159.7 | p95 outage 94.0 ms | back in rotation 7.06 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-11-10 14:19:31 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 63.0s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['3a0e06f14a63', '4b1faad0399c'] +-- stopping 3a0e06f14a63 (mode=kill) under load + hook 1.20s | LB DOWN after 2.39 s | errors 3 (502) | RPS before 280.4, outage 89.8, after restore 171.1 | p95 outage 110.0 ms | back in rotation 7.06 s | data ok=True +-- stopping 4b1faad0399c (mode=kill) under load + hook 0.83s | LB DOWN after 2.34 s | errors 6 (502) | RPS before 260.4, outage 89.4, after restore 156.0 | p95 outage 110.0 ms | back in rotation 7.28 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab5/evidence/failover_series_kill.json b/docs/lab5/evidence/failover_series_kill.json new file mode 100644 index 0000000..1291a1e --- /dev/null +++ b/docs/lab5/evidence/failover_series_kill.json @@ -0,0 +1,298 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "rounds": [ + { + "target_instance": "3a0e06f14a63", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.2, + "lb_detect_down_seconds": 2.39, + "target_last_served_after_stop_seconds": 0.56, + "target_served_after_hook_returned": false, + "errors_during_outage": 3, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.64, + "restore_hook_seconds": 7.12, + "lb_back_up_seconds": 1.42, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1402, + "ok": 1402, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 280.4, + "write_ok_rps": 161.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 94.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 901, + "ok": 898, + "errors": 3, + "error_ratio": 0.0033, + "ok_rps": 89.8, + "write_ok_rps": 52.6, + "latency_ms_ok": { + "p50": 32.0, + "p95": 110.0, + "p99": 1015.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 518, + "ok": 518, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 103.6, + "write_ok_rps": 60.8, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 187.0 + }, + "instances": [ + "4b1faad0399c" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.44, + "requests": 1786, + "ok": 1786, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 171.1, + "write_ok_rps": 101.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.39, + [ + "api1" + ] + ], + [ + 21.42, + [ + "api1", + "api2" + ] + ], + [ + 21.53, + [ + "api1" + ] + ], + [ + 24.02, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2698, + "stored_messages": 2698, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 1, + "ok": true + } + }, + { + "target_instance": "4b1faad0399c", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 0.83, + "lb_detect_down_seconds": 2.34, + "target_last_served_after_stop_seconds": 0.09, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.28, + "restore_hook_seconds": 7.3, + "lb_back_up_seconds": 1.33, + "target_back_in_rotation_seconds": 7.28, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1302, + "ok": 1302, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 260.4, + "write_ok_rps": 153.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 900, + "ok": 894, + "errors": 6, + "error_ratio": 0.0067, + "ok_rps": 89.4, + "write_ok_rps": 52.5, + "latency_ms_ok": { + "p50": 32.0, + "p95": 110.0, + "p99": 1016.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 565, + "ok": 565, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 113.0, + "write_ok_rps": 66.6, + "latency_ms_ok": { + "p50": 62.0, + "p95": 109.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.5, + "requests": 1638, + "ok": 1638, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 156.0, + "write_ok_rps": 92.5, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.34, + [ + "api2" + ] + ], + [ + 21.33, + [ + "api1", + "api2" + ] + ], + [ + 22.2, + [ + "api2" + ] + ], + [ + 24.74, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2599, + "stored_messages": 2599, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/instances.txt b/docs/lab5/evidence/instances.txt new file mode 100644 index 0000000..3a3d302 --- /dev/null +++ b/docs/lab5/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-11-10 14:14:03 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019A78-1F90-6AF326AB268-0BC1-0008 + 2 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019A88-1F90-6AF326AB27B-0BC2-0008 + 3 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019A8E-1F90-6AF326AB27F-0BC3-0008 + 4 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019A9C-1F90-6AF326AB282-0BC4-0008 + 5 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AA4-1F90-6AF326AB296-0BC5-0008 + 6 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AA8-1F90-6AF326AB2A5-0BC6-0008 + 7 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AAA-1F90-6AF326AB2B4-0BC7-0008 + 8 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AB0-1F90-6AF326AB2C4-0BC8-0008 + 9 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ABA-1F90-6AF326AB2D3-0BC9-0008 +10 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019ABE-1F90-6AF326AB2E2-0BCA-0008 +11 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ACC-1F90-6AF326AB2E5-0BCB-0008 +12 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019ADC-1F90-6AF326AB2F2-0BCC-0008 +13 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ADE-1F90-6AF326AB300-0BCD-0008 +14 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AE4-1F90-6AF326AB303-0BCE-0008 +15 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AEA-1F90-6AF326AB305-0BCF-0008 +16 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AF6-1F90-6AF326AB320-0BD0-0008 +17 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019B06-1F90-6AF326AB323-0BD1-0008 +18 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019B0C-1F90-6AF326AB33F-0BD2-0008 +19 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019B16-1F90-6AF326AB34E-0BD3-0008 +20 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019B24-1F90-6AF326AB351-0BD4-0008 +разных реплик: 2 diff --git a/docs/lab5/evidence/lb_log_sample.txt b/docs/lab5/evidence/lb_log_sample.txt new file mode 100644 index 0000000..4a390a1 --- /dev/null +++ b/docs/lab5/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-11-10 14:26:03 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.20.0.1:36070 [10/Nov/2026:14:26:00.806] public api/api1 0/0/0/24/25 201 421 ---- 7/7/4/3/0 0/0 rid=AC1400018CE6-1F90-6AF32978326-1436-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36082 [10/Nov/2026:14:26:00.818] public api/api1 0/0/0/14/14 201 421 ---- 8/8/4/2/0 0/0 rid=AC1400018CF2-1F90-6AF32978332-1438-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36116 [10/Nov/2026:14:26:00.827] public api/api2 0/0/0/6/6 201 421 ---- 7/7/3/1/0 0/0 rid=AC1400018D14-1F90-6AF3297833B-143B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36130 [10/Nov/2026:14:26:00.831] public api/api2 0/0/0/7/8 201 421 ---- 8/8/4/1/0 0/0 rid=AC1400018D22-1F90-6AF3297833F-143D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36078 [10/Nov/2026:14:26:00.822] public api/api1 0/0/0/18/18 201 421 ---- 8/8/3/2/0 0/0 rid=AC1400018CEE-1F90-6AF3297832C-143A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36104 [10/Nov/2026:14:26:00.838] public api/api2 0/0/0/6/6 201 421 ---- 8/8/4/1/0 0/0 rid=AC1400018D08-1F90-6AF32978338-143F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36182 [10/Nov/2026:14:26:00.842] public api/api2 0/0/0/6/6 201 421 ---- 8/8/4/0/0 0/0 rid=AC1400018D56-1F90-6AF3297834A-1441-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36196 [10/Nov/2026:14:26:00.850] public api/api2 0/0/0/4/4 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018D64-1F90-6AF32978352-1443-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36122 [10/Nov/2026:14:26:00.854] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018D1A-1F90-6AF3297833D-1445-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36212 [10/Nov/2026:14:26:00.856] public api/api2 0/0/0/6/6 201 421 ---- 8/8/6/0/0 0/0 rid=AC1400018D74-1F90-6AF32978358-1447-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36224 [10/Nov/2026:14:26:00.884] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018D80-1F90-6AF3297835E-1449-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36118 [10/Nov/2026:14:26:00.828] public api/api1 0/0/0/76/76 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018D16-1F90-6AF3297833C-143C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36156 [10/Nov/2026:14:26:00.834] public api/api1 0/0/0/78/78 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018D3C-1F90-6AF32978342-143E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36276 [10/Nov/2026:14:26:00.914] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018DB4-1F90-6AF32978392-144B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36190 [10/Nov/2026:14:26:00.846] public api/api1 0/0/0/77/77 201 421 ---- 8/8/7/6/0 0/0 rid=AC1400018D5E-1F90-6AF3297834E-1442-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36168 [10/Nov/2026:14:26:00.841] public api/api1 0/0/0/82/83 201 421 ---- 7/7/6/5/0 0/0 rid=AC1400018D48-1F90-6AF32978349-1440-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36166 [10/Nov/2026:14:26:00.854] public api/api1 0/0/0/72/73 201 421 ---- 8/8/7/5/0 0/0 rid=AC1400018D46-1F90-6AF32978343-1446-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36262 [10/Nov/2026:14:26:00.906] public api/api1 0/0/0/21/21 201 421 ---- 8/8/6/4/0 0/0 rid=AC1400018DA6-1F90-6AF3297838A-144A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36238 [10/Nov/2026:14:26:00.864] public api/api1 0/0/0/64/64 201 421 ---- 7/7/5/3/0 0/0 rid=AC1400018D8E-1F90-6AF32978360-1448-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36146 [10/Nov/2026:14:26:00.853] public api/api1 0/0/1/74/75 201 421 ---- 6/6/5/3/0 0/0 rid=AC1400018D32-1F90-6AF32978340-1444-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36282 [10/Nov/2026:14:26:00.922] public api/api2 0/0/0/6/6 201 421 ---- 5/5/4/1/0 0/0 rid=AC1400018DBA-1F90-6AF3297839A-144D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36250 [10/Nov/2026:14:26:00.915] public api/api1 0/0/0/14/15 201 421 ---- 5/5/3/2/0 0/0 rid=AC1400018D9A-1F90-6AF3297837C-144C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36294 [10/Nov/2026:14:26:00.926] public api/api2 0/0/0/8/8 201 421 ---- 8/8/6/2/0 0/0 rid=AC1400018DC6-1F90-6AF3297839E-144F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36324 [10/Nov/2026:14:26:00.930] public api/api2 0/0/0/10/10 201 421 ---- 8/8/6/2/0 0/0 rid=AC1400018DE4-1F90-6AF329783A2-1451-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36378 [10/Nov/2026:14:26:00.935] public api/api2 0/0/0/7/7 201 421 ---- 8/8/6/1/0 0/0 rid=AC1400018E1A-1F90-6AF329783A7-1455-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36336 [10/Nov/2026:14:26:00.931] public api/api2 0/0/0/12/12 201 421 ---- 7/7/5/0/0 0/0 rid=AC1400018DF0-1F90-6AF329783A3-1453-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36396 [10/Nov/2026:14:26:00.945] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018E2C-1F90-6AF329783B1-1457-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36362 [10/Nov/2026:14:26:00.945] public api/api2 0/0/0/7/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018E0A-1F90-6AF329783A4-1459-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36416 [10/Nov/2026:14:26:00.977] public api/api2 0/0/0/7/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018E40-1F90-6AF329783BA-145B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36308 [10/Nov/2026:14:26:00.926] public api/api1 0/0/0/78/78 201 421 ---- 8/8/7/7/0 0/0 rid=AC1400018DD4-1F90-6AF3297839E-144E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36320 [10/Nov/2026:14:26:00.928] public api/api1 0/0/0/77/78 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018DE0-1F90-6AF329783A0-1450-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36428 [10/Nov/2026:14:26:01.006] public api/api2 0/0/0/6/6 201 420 ---- 8/8/6/0/0 0/0 rid=AC1400018E4C-1F90-6AF3297906-145D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36348 [10/Nov/2026:14:26:00.931] public api/api1 0/0/0/86/86 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018DFC-1F90-6AF329783A3-1452-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36342 [10/Nov/2026:14:26:00.930] public api/api1 0/0/0/88/88 201 421 ---- 8/8/5/5/0 0/0 rid=AC1400018DF6-1F90-6AF329783A2-1454-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36418 [10/Nov/2026:14:26:00.986] public api/api1 0/0/0/35/36 201 421 ---- 8/8/4/4/0 0/0 rid=AC1400018E42-1F90-6AF329783DA-145C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36384 [10/Nov/2026:14:26:00.942] public api/api1 0/0/0/80/81 201 421 ---- 7/7/4/3/0 0/0 rid=AC1400018E20-1F90-6AF329783AE-1456-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36392 [10/Nov/2026:14:26:00.945] public api/api1 0/0/0/77/78 201 421 ---- 6/6/3/2/0 0/0 rid=AC1400018E28-1F90-6AF329783B1-1458-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36402 [10/Nov/2026:14:26:00.952] public api/api1 0/0/0/71/71 201 421 ---- 6/6/2/1/0 0/0 rid=AC1400018E32-1F90-6AF329783B8-145A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36444 [10/Nov/2026:14:26:01.014] public api/api1 0/0/0/14/14 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E5C-1F90-6AF329790E-145E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36438 [10/Nov/2026:14:26:01.023] public api/api2 0/0/0/11/11 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E56-1F90-6AF3297908-145F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36496 [10/Nov/2026:14:26:01.025] public api/api2 0/0/0/11/11 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E90-1F90-6AF3297919-1461-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36482 [10/Nov/2026:14:26:01.025] public api/api1 0/0/0/11/11 201 420 ---- 7/7/4/2/0 0/0 rid=AC1400018E82-1F90-6AF3297919-1462-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36488 [10/Nov/2026:14:26:01.025] public api/api2 0/0/0/12/12 201 420 ---- 6/6/3/1/0 0/0 rid=AC1400018E88-1F90-6AF3297919-1463-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36512 [10/Nov/2026:14:26:01.036] public api/api2 0/0/0/10/10 201 420 ---- 8/8/7/2/0 0/0 rid=AC1400018EA0-1F90-6AF3297924-1465-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36462 [10/Nov/2026:14:26:01.038] public api/api2 0/0/0/11/11 201 420 ---- 8/8/7/2/0 0/0 rid=AC1400018E6E-1F90-6AF3297915-1467-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36524 [10/Nov/2026:14:26:01.039] public api/api2 0/0/0/9/10 201 420 ---- 7/7/6/1/0 0/0 rid=AC1400018EAC-1F90-6AF3297927-1469-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36546 [10/Nov/2026:14:26:01.048] public api/api2 0/0/0/8/8 201 420 ---- 8/8/7/1/0 0/0 rid=AC1400018EC2-1F90-6AF3297930-146B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36568 [10/Nov/2026:14:26:01.051] public api/api2 0/0/0/7/7 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018ED8-1F90-6AF3297933-146D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36592 [10/Nov/2026:14:26:01.059] public api/api2 0/0/0/4/4 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018EF0-1F90-6AF329793B-146F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36508 [10/Nov/2026:14:26:01.030] public api/api1 0/0/0/68/68 201 420 ---- 8/8/7/7/0 0/0 rid=AC1400018E9C-1F90-6AF329791E-1464-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36470 [10/Nov/2026:14:26:01.025] public api/api1 0/0/0/73/73 201 420 ---- 7/7/6/6/0 0/0 rid=AC1400018E76-1F90-6AF3297919-1460-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36612 [10/Nov/2026:14:26:01.100] public api/api2 0/0/0/5/6 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018F04-1F90-6AF3297964-1471-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36620 [10/Nov/2026:14:26:01.108] public api/api2 0/0/0/4/4 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018F0C-1F90-6AF329796C-1473-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36618 [10/Nov/2026:14:26:01.100] public api/api1 0/0/0/17/17 201 420 ---- 7/7/6/6/0 0/0 rid=AC1400018F0A-1F90-6AF3297964-1472-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36584 [10/Nov/2026:14:26:01.057] public api/api1 0/0/0/61/61 201 420 ---- 6/6/5/5/0 0/0 rid=AC1400018EE8-1F90-6AF3297939-146E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36536 [10/Nov/2026:14:26:01.040] public api/api1 0/0/0/81/81 201 420 ---- 5/5/4/4/0 0/0 rid=AC1400018EB8-1F90-6AF3297928-146A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36516 [10/Nov/2026:14:26:01.039] public api/api1 0/0/0/83/83 201 420 ---- 4/4/3/3/0 0/0 rid=AC1400018EA4-1F90-6AF3297927-1468-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36452 [10/Nov/2026:14:26:01.038] public api/api1 0/0/1/84/85 201 420 ---- 3/3/2/2/0 0/0 rid=AC1400018E64-1F90-6AF3297913-1466-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36600 [10/Nov/2026:14:26:01.065] public api/api1 0/0/0/57/58 201 420 ---- 2/2/1/1/0 0/0 rid=AC1400018EF8-1F90-6AF3297941-1470-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36562 [10/Nov/2026:14:26:01.051] public api/api1 0/0/0/72/72 201 420 ---- 1/1/0/0/0 0/0 rid=AC1400018ED2-1F90-6AF3297933-146C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" diff --git a/docs/lab5/evidence/load.json b/docs/lab5/evidence/load.json new file mode 100644 index 0000000..d999394 --- /dev/null +++ b/docs/lab5/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.094, + "concurrency": 8, + "successful_requests": 5150, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 171.13, + "successful_latency_ms": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab5/evidence/load.txt b/docs/lab5/evidence/load.txt new file mode 100644 index 0000000..7cc2794 --- /dev/null +++ b/docs/lab5/evidence/load.txt @@ -0,0 +1,31 @@ +# 2026-11-10 14:26:01 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.5s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.094, + "concurrency": 8, + "successful_requests": 5150, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 171.13, + "successful_latency_ms": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-11-10 14:26:03 UTC | CPU/RAM после нагрузки | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab5-api-6 20.42% 95.3MiB / 320MiB 29.78% +messenger-lab5-api-5 19.69% 94.15MiB / 320MiB 29.42% +messenger-lab5-worker-1 0.13% 77.54MiB / 1.25GiB 6.06% +messenger-lab5-lb-1 0.34% 13.26MiB / 128MiB 10.36% +messenger-lab5-postgres-1 3.10% 71.41MiB / 512MiB 13.95% +messenger-lab5-redis-1 0.21% 4.559MiB / 96MiB 4.75% +messenger-lab5-s3-1 2.87% 99.73MiB / 512MiB 19.48% +messenger-lab5-broker-1 2.79% 8.617MiB / 320MiB 2.69% + diff --git a/docs/lab5/evidence/local/README.md b/docs/lab5/evidence/local/README.md new file mode 100644 index 0000000..be500a2 --- /dev/null +++ b/docs/lab5/evidence/local/README.md @@ -0,0 +1,33 @@ +# Локальный прогон (без Docker) — что это и чем НЕ является + +Дата: 2026-11-09. Эти файлы получены при разработке в изолированной Linux-песочнице, +где **Docker Hub недоступен** (образы не скачиваются), поэтому compose-стенд там не +поднимался. Те же процессы запускались нативно: + +| Роль | В compose.yaml | В локальном прогоне | +| --- | --- | --- | +| API x2 | `api` (образ из Dockerfile) | `python -m src.server` x2 (INSTANCE_ID=api1/api2) | +| LB | HAProxy 2.8.30 | 40-строчный round-robin TCP-прокси на asyncio (только для теста) | +| PostgreSQL | postgres:16.4-alpine | PostgreSQL 16 (системный) | +| Кэш сессий | redis:7.4.1-alpine | Redis 7.0.15, без persistence | +| Broker | redis:7.4.1-alpine, AOF `appendfsync always`, `noeviction` | Redis 7.0.15 с теми же флагами, отдельный dir | +| S3 | SeaweedFS 4.50 | moto 5.0.28 (S3-эмулятор, in-memory) | +| Worker | `worker` (`python -m src.worker`) | тот же модуль; «restart: unless-stopped» эмулирован циклом в shell | + +Код приложения, миграции и тесты — те же, что в репозитории. Хуки рестарта/остановки +заменены локальными скриптами (kill/pg_ctl/redis SHUTDOWN), поэтому **доказательства +на compose-стенде нужно собрать самостоятельно**: `python scripts/collect_evidence.py` +(запишет evidence/*.txt и evidence/SUMMARY.txt). S3 в локальном persistence-прогоне +не перезапускался (moto хранит данные в памяти). + +| Файл | Что внутри | +| --- | --- | +| smoke.txt, student-tests.txt | `make test` и `student-tests/run_all.py` через 2 реплики: 91 тест + race, PASS | +| pipeline_and_attachments.txt | test_pipeline.py (7) и адаптированный test_attachments.py (17), PASS | +| pipeline_faults.txt | test_pipeline_faults.py: 10 сценариев отказов уровня 4, PASS | +| pipeline_faults_events.txt | журнал job_events этих сценариев (доказательство *как* система восстановилась) | +| worker_fault_log.txt | строки лога worker: внедрённые падения и восстановление своего PEL | +| worker_outage.txt / .json | сценарий защиты уровня 3 (worker stop -> 202 -> рестарт API и broker -> ready) | +| persistence.txt, failover.txt | публичные tests/resilience.py (persistence: API+worker+PG+Redis+broker) | +| latency.json | 30 загрузок: 202 p50 16 мс; до ready p50 0.44 с (с шагом опроса 0.2 с) | +| imaging_memory.txt | пиковый RSS подпроцесса обработки: 40 Мп RGBA PNG — 361 MiB, 40 Мп JPEG — 46 MiB | diff --git a/docs/lab5/evidence/local/failover.txt b/docs/lab5/evidence/local/failover.txt new file mode 100644 index 0000000..364e26c --- /dev/null +++ b/docs/lab5/evidence/local/failover.txt @@ -0,0 +1,4 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 141, "failed_attempts": 0, "first_recovery_seconds": 0.115} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +EXIT=0 diff --git a/docs/lab5/evidence/local/imaging_memory.txt b/docs/lab5/evidence/local/imaging_memory.txt new file mode 100644 index 0000000..5da3946 --- /dev/null +++ b/docs/lab5/evidence/local/imaging_memory.txt @@ -0,0 +1,5 @@ +synthetic/big.jpg 626630 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (1280, 800)} 0.20s peak RSS 46 MiB +synthetic/big.png 165443 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (1280, 800)} 1.80s peak RSS 361 MiB +tests/fixtures/sample.png 111817 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (320, 200)} 0.05s peak RSS 23 MiB +synthetic/alpha.png 86 {'thumbnail': (50, 30), 'crop': (128, 128), 'optimized': (50, 30)} 0.04s peak RSS 23 MiB +synthetic/orient.jpg 1365 {'thumbnail': (64, 256), 'crop': (128, 128), 'optimized': (100, 400)} 0.04s peak RSS 23 MiB diff --git a/docs/lab5/evidence/local/latency.json b/docs/lab5/evidence/local/latency.json new file mode 100644 index 0000000..ec3d3a5 --- /dev/null +++ b/docs/lab5/evidence/local/latency.json @@ -0,0 +1,14 @@ +{ + "n": 30, + "file": "tests/fixtures/sample.png 320x200 109 KiB", + "upload_202_s": { + "p50": 0.016, + "p95": 0.029 + }, + "upload_to_ready_s": { + "p50": 0.435, + "p95": 0.449, + "max": 0.45 + }, + "note": "local sandbox, poll interval 0.2 s, 1 worker x 2 consumers" +} diff --git a/docs/lab5/evidence/local/persistence.txt b/docs/lab5/evidence/local/persistence.txt new file mode 100644 index 0000000..baff3a9 --- /dev/null +++ b/docs/lab5/evidence/local/persistence.txt @@ -0,0 +1,3 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +EXIT=0 diff --git a/docs/lab5/evidence/local/pipeline_and_attachments.txt b/docs/lab5/evidence/local/pipeline_and_attachments.txt new file mode 100644 index 0000000..d695303 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_and_attachments.txt @@ -0,0 +1,11 @@ +....... +---------------------------------------------------------------------- +Ran 7 tests in 4.403s + +OK +................. +---------------------------------------------------------------------- +Ran 17 tests in 3.869s + +OK +done diff --git a/docs/lab5/evidence/local/pipeline_faults.txt b/docs/lab5/evidence/local/pipeline_faults.txt new file mode 100644 index 0000000..77846c7 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_faults.txt @@ -0,0 +1,18 @@ +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... ok +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... ok +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... ok +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... ok +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... ok +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... ok +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... ok +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... ok +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... /home/claude/venv/lib/python3.13/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 71.328s + +OK +EXIT=0 diff --git a/docs/lab5/evidence/local/pipeline_faults_events.txt b/docs/lab5/evidence/local/pipeline_faults_events.txt new file mode 100644 index 0000000..c47c7f8 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_faults_events.txt @@ -0,0 +1,71 @@ + filename | attempt | event | detail | actor +----------------------------------+---------+-----------------+---------------------------------------------------+------------- + a.png | 1 | accepted | | api + a.png | 1 | fault_injected | crash between COMMIT and publish | api:apipub + a.png | 1 | published | stream id 1794253144786-0 | api:apidup + a.png | 1 | claimed | | wf-1 + a.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + b.png | 1 | accepted | | api + b.png | 1 | fault_injected | crash after XADD before outbox commit | api:apidup + b.png | 1 | claimed | | wf-1 + b.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + b.png | 1 | published | stream id 1794253148012-0 | api:api1 + b.png | 1 | redelivery_noop | already_succeeded | wf-0 + fault-crash-after-effects.png | 1 | accepted | | api + fault-crash-after-effects.png | 1 | published | stream id 1794253148173-0 | api:api1 + fault-crash-after-effects.png | 1 | claimed | | wf-1 + fault-crash-after-effects.png | 1 | fault_injected | crash after S3 writes before COMMIT | wf-1 + fault-crash-after-effects.png | 1 | lease_takeover | previous owner wf-1 | wf-1 + fault-crash-after-effects.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + fault-crash-before-ack.png | 1 | accepted | | api + fault-crash-before-ack.png | 1 | published | stream id 1794253150245-0 | api:api1 + fault-crash-before-ack.png | 1 | claimed | | wf-0 + fault-crash-before-ack.png | 1 | succeeded | thumbnail,crop,optimized | wf-0 + fault-crash-before-ack.png | 1 | fault_injected | crash after COMMIT before XACK | wf-0 + fault-crash-before-ack.png | 1 | redelivery_noop | already_succeeded | wf-0 + fault-permanent.png | 1 | accepted | | api + fault-permanent.png | 1 | published | stream id 1794253155274-0 | api:api1 + fault-permanent.png | 1 | claimed | | wf-1 + fault-permanent.png | 1 | failed | invalid_image; injected permanent failure | wf-1 + fault-slow.png | 1 | accepted | | api + fault-slow.png | 1 | published | stream id 1794253167381-0 | api:api1 + fault-slow.png | 1 | claimed | | wf-1 + fault-slow.png | 1 | lease_expired | owner wf-1; next attempt 2 in 0.51s | worker:wf + fault-slow.png | 1 | redelivery_noop | stale_attempt | wf-0 + fault-slow.png | 2 | published | stream id 1794253198039-0 | worker:wf + fault-slow.png | 2 | claimed | | wf-0 + fault-slow.png | 2 | succeeded | thumbnail,crop,optimized | wf-0 + fault-slow.png | 1 | lease_lost | result not published: lease token no longer valid | wf-1 + fault-transient-until-3.png | 1 | accepted | | api + fault-transient-until-3.png | 1 | published | stream id 1794253152182-0 | api:api1 + fault-transient-until-3.png | 1 | claimed | | wf-1 + fault-transient-until-3.png | 1 | retry_scheduled | injected_transient; next attempt 2 in 0.79s | wf-1 + fault-transient-until-3.png | 2 | published | stream id 1794253153023-0 | api:api1 + fault-transient-until-3.png | 2 | claimed | | wf-0 + fault-transient-until-3.png | 2 | retry_scheduled | injected_transient; next attempt 3 in 1.90s | wf-0 + fault-transient-until-3.png | 3 | published | stream id 1794253154959-0 | worker:wf + fault-transient-until-3.png | 3 | claimed | | wf-0 + fault-transient-until-3.png | 3 | succeeded | thumbnail,crop,optimized | wf-0 + fault-transient-until-replay.png | 1 | accepted | | api + fault-transient-until-replay.png | 1 | published | stream id 1794253155506-0 | api:api1 + fault-transient-until-replay.png | 1 | claimed | | wf-0 + fault-transient-until-replay.png | 1 | retry_scheduled | injected_transient; next attempt 2 in 0.54s | wf-0 + fault-transient-until-replay.png | 2 | published | stream id 1794253156965-0 | worker:wf + fault-transient-until-replay.png | 2 | claimed | | wf-1 + fault-transient-until-replay.png | 2 | retry_scheduled | injected_transient; next attempt 3 in 1.97s | wf-1 + fault-transient-until-replay.png | 3 | published | stream id 1794253158970-0 | worker:wf + fault-transient-until-replay.png | 3 | claimed | | wf-1 + fault-transient-until-replay.png | 3 | retry_scheduled | injected_transient; next attempt 4 in 3.54s | wf-1 + fault-transient-until-replay.png | 4 | published | stream id 1794253162979-0 | worker:wf + fault-transient-until-replay.png | 4 | claimed | | wf-1 + fault-transient-until-replay.png | 4 | retry_scheduled | injected_transient; next attempt 5 in 2.54s | wf-1 + fault-transient-until-replay.png | 5 | published | stream id 1794253165987-0 | worker:wf + fault-transient-until-replay.png | 5 | claimed | | wf-1 + fault-transient-until-replay.png | 5 | failed | retry_exhausted; injected_transient | wf-1 + fault-transient-until-replay.png | 6 | replayed | operator replay; new attempt 6 | test:replay + fault-transient-until-replay.png | 6 | published | stream id 1794253166991-0 | worker:wf + fault-transient-until-replay.png | 6 | claimed | | wf-0 + fault-transient-until-replay.png | 6 | succeeded | thumbnail,crop,optimized | wf-0 + fault-transient-until-replay.png | 6 | replay_refused | already succeeded (no duplicate execution) | test:replay +(67 rows) + diff --git a/docs/lab5/evidence/local/smoke.txt b/docs/lab5/evidence/local/smoke.txt new file mode 100644 index 0000000..5c90482 --- /dev/null +++ b/docs/lab5/evidence/local/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab5/evidence/local/student-tests.txt b/docs/lab5/evidence/local/student-tests.txt new file mode 100644 index 0000000..545d8be --- /dev/null +++ b/docs/lab5/evidence/local/student-tests.txt @@ -0,0 +1,111 @@ + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... + 24 parallel POSTs, one key -> 1 message; replicas: ['api1', 'api2'] +ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 91 tests in 11.178s + +OK (skipped=33) + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +EXIT=0 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +EXIT=0 diff --git a/docs/lab5/evidence/local/worker_fault_log.txt b/docs/lab5/evidence/local/worker_fault_log.txt new file mode 100644 index 0000000..0cbbea1 --- /dev/null +++ b/docs/lab5/evidence/local/worker_fault_log.txt @@ -0,0 +1,8 @@ +2026-11-09 22:39:08,453 ERROR messenger.worker FAULT_INJECTION: worker exits after S3 writes before COMMIT (job 689dc4ad-bf5d-4d33-a7f4-d557aef94b2a) +22:39:08 wf exited (os._exit(70)); supervisor restarts it like docker restart: unless-stopped +2026-11-09 22:39:09,883 INFO messenger.worker wf-1: recovering own unacked message 1794253148173-0 +2026-11-09 22:39:10,530 ERROR messenger.worker FAULT_INJECTION: worker exits after COMMIT before XACK (job 6746ae17-f4cc-48b1-9864-c6e80507c2bf) +22:39:10 wf exited (os._exit(70)); supervisor restarts it like docker restart: unless-stopped +2026-11-09 22:39:11,945 INFO messenger.worker wf-0: recovering own unacked message 1794253150245-0 +2026-11-09 22:39:57,386 INFO messenger.worker wf-0: claimed idle message 1794253167381-0 (deliveries=2) +2026-11-09 22:40:12,653 WARNING messenger.worker wf-1: lease lost for job acf17aa8-ef8d-4338-8beb-bd9bab29be8b; result not published (fenced) diff --git a/docs/lab5/evidence/local/worker_outage.json b/docs/lab5/evidence/local/worker_outage.json new file mode 100644 index 0000000..a154d0a --- /dev/null +++ b/docs/lab5/evidence/local/worker_outage.json @@ -0,0 +1,80 @@ +[ + { + "seconds": 0.02, + "step": "worker stopped", + "t": 0.14 + }, + { + "status": 202, + "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", + "attachment_status": "queued", + "step": "upload accepted", + "t": 0.15 + }, + { + "metadata_status": "queued", + "content_status": 409, + "message_with_attachment": 201, + "plain_message": 201, + "message_id": "d589ad3f-63a3-4258-a981-701a2a7bfb5e", + "step": "while worker is down", + "t": 3.17 + }, + { + "length": 1, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 32, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "broker before restarts", + "t": 3.37 + }, + { + "seconds": 3.06, + "step": "api replicas restarted", + "t": 6.43 + }, + { + "seconds": 2.03, + "step": "broker restarted (volume kept)", + "t": 8.46 + }, + { + "metadata_status": "queued", + "length": 1, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 32, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "after restarts", + "t": 10.66 + }, + { + "seconds": 3.01, + "step": "worker started", + "t": 13.67 + }, + { + "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "thumbnail_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true, + "step": "ready", + "t": 13.7 + } +] \ No newline at end of file diff --git a/docs/lab5/evidence/local/worker_outage.txt b/docs/lab5/evidence/local/worker_outage.txt new file mode 100644 index 0000000..1db4cd3 --- /dev/null +++ b/docs/lab5/evidence/local/worker_outage.txt @@ -0,0 +1,11 @@ +{"seconds": 0.02, "step": "worker stopped", "t": 0.14} +{"status": 202, "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", "attachment_status": "queued", "step": "upload accepted", "t": 0.15} +{"metadata_status": "queued", "content_status": 409, "message_with_attachment": 201, "plain_message": 201, "message_id": "d589ad3f-63a3-4258-a981-701a2a7bfb5e", "step": "while worker is down", "t": 3.17} +{"length": 1, "lag": 1, "pending": 0, "jobs": {"succeeded": 32, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "broker before restarts", "t": 3.37} +{"seconds": 3.06, "step": "api replicas restarted", "t": 6.43} +{"seconds": 2.03, "step": "broker restarted (volume kept)", "t": 8.46} +{"metadata_status": "queued", "length": 1, "lag": 1, "pending": 0, "jobs": {"succeeded": 32, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "after restarts", "t": 10.66} +{"seconds": 3.01, "step": "worker started", "t": 13.67} +{"attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", "variants": ["thumbnail", "crop", "optimized"], "thumbnail_jpeg_size": [256, 160], "original_sha256_ok": true, "step": "ready", "t": 13.7} +PASS: accepted job survived worker stop + API restart + broker restart and became ready +EXIT=0 diff --git a/docs/lab5/evidence/persistence.txt b/docs/lab5/evidence/persistence.txt new file mode 100644 index 0000000..40b055c --- /dev/null +++ b/docs/lab5/evidence/persistence.txt @@ -0,0 +1,53 @@ +# 2026-11-10 14:24:42 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab5-api-5 Up 33 seconds (healthy) 34 seconds ago +messenger-lab5-api-6 Up 22 seconds (healthy) 22 seconds ago +messenger-lab5-broker-1 Up 3 minutes (healthy) 39 minutes ago +messenger-lab5-lb-1 Up 24 minutes (healthy) 39 minutes ago +messenger-lab5-migrate-1 Exited (0) 3 minutes ago 39 minutes ago +messenger-lab5-postgres-1 Up 25 minutes (healthy) 39 minutes ago +messenger-lab5-redis-1 Up 25 minutes (healthy) 39 minutes ago +messenger-lab5-s3-1 Up 4 minutes (healthy) 39 minutes ago +messenger-lab5-s3-init-1 Exited (0) 3 minutes ago 39 minutes ago +messenger-lab5-worker-1 Up About a minute (healthy) About a minute ago +# 2026-11-10 14:25:27 UTC | Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB) | exit 0 | 45.1s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:25:27 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab5-api-5 Up 7 seconds (healthy) About a minute ago +messenger-lab5-api-6 Up 7 seconds (healthy) About a minute ago +messenger-lab5-broker-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-lb-1 Up Less than a second (health: starting) 40 minutes ago +messenger-lab5-migrate-1 Exited (0) 4 minutes ago 40 minutes ago +messenger-lab5-postgres-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-redis-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-s3-1 Up 27 seconds (healthy) 40 minutes ago +messenger-lab5-s3-init-1 Exited (0) 4 minutes ago 40 minutes ago +messenger-lab5-worker-1 Up 7 seconds (healthy) 2 minutes ago + +# 2026-11-10 14:25:30 UTC | Persistence: сверка БД <-> S3 после рестарта | exit 0 | 2.9s +$ python scripts/ops.py reconcile + Container messenger-lab5-api-run-467cf1258171 Creating + Container messenger-lab5-api-run-467cf1258171 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "staging_to_clean": 1, + "objects_listed": 640, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.211, + "ready_rows": 637, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.229 +} + diff --git a/docs/lab5/evidence/pipeline.txt b/docs/lab5/evidence/pipeline.txt new file mode 100644 index 0000000..bbae87b --- /dev/null +++ b/docs/lab5/evidence/pipeline.txt @@ -0,0 +1,14 @@ +# 2026-11-10 14:20:25 UTC | Варианты изображений по фактическим байтам (test_pipeline.py) | exit 0 | 6.6s +$ python -m unittest discover -s student-tests -p test_pipeline.py -v +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 6.481s + +OK diff --git a/docs/lab5/evidence/pipeline_faults.txt b/docs/lab5/evidence/pipeline_faults.txt new file mode 100644 index 0000000..28753d3 --- /dev/null +++ b/docs/lab5/evidence/pipeline_faults.txt @@ -0,0 +1,152 @@ +# 2026-11-10 14:23:04 UTC | Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) | exit 0 | 126.3s +$ python scripts/ops.py pipeline-faults +2026-11-10T14:20:58.422Z pipeline-faults: recreating worker with compose.faults.yaml + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-fault-publish-1 Creating + Container messenger-lab5-api-fault-dup-1 Creating + Container messenger-lab5-api-fault-publish-1 Created + Container messenger-lab5-api-fault-dup-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-dup-1 Starting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-publish-1 Starting + Container messenger-lab5-api-fault-dup-1 Started + Container messenger-lab5-api-fault-publish-1 Started + Container messenger-lab5-api-run-f6d1d1271d68 Creating + Container messenger-lab5-api-run-f6d1d1271d68 Created +ready: http://api-fault-publish:8080/health/ready + Container messenger-lab5-api-run-0c827bfb627f Creating + Container messenger-lab5-api-run-0c827bfb627f Created +ready: http://api-fault-dup:8080/health/ready + Container messenger-lab5-api-run-6e712120049d Creating + Container messenger-lab5-api-run-6e712120049d Created +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... ok +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... ok +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... ok +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... ok +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... ok +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... ok +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... ok +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... ok +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 90.529s + +OK +2026-11-10T14:23:04.629Z pipeline-faults: worker restored to normal configuration +# 2026-11-10 14:23:06 UTC | Очередь после сценариев: задания, outbox, DLQ, lag | exit 0 | 1.7s +$ python scripts/ops.py jobs stats + Container messenger-lab5-worker-run-e232ea69a3ca Creating + Container messenger-lab5-worker-run-e232ea69a3ca Created +{ + "jobs": { + "succeeded": 158, + "failed": 2 + }, + "attachments": { + "ready": 159, + "failed": 2 + }, + "outbox": { + "rows": 0, + "due": 0, + "oldest_age_s": 0.0 + }, + "dead_letters_unreplayed": 2, + "oldest_unfinished_job_age_s": 0.0, + "broker": { + "stream": "attachments.jobs", + "length": 4, + "groups": [ + { + "name": "attachment-workers", + "consumers": 10, + "pending": 0, + "lag": 0, + "last_delivered_id": "1794320565944-0" + } + ] + } +} + +# 2026-11-10 14:23:07 UTC | Реестр failed (DLQ) | exit 0 | 1.6s +$ python scripts/ops.py jobs failed --limit 10 + Container messenger-lab5-worker-run-85c598896740 Creating + Container messenger-lab5-worker-run-85c598896740 Created +[ + { + "attachment_id": "9f72cadb-e186-43cb-8456-ee1d5f3bde45", + "job_id": "05c07c99-eda0-4dbe-b49e-98d1426f3933", + "error_code": "retry_exhausted", + "attempts": 5, + "failed_at": "2026-11-10 14:21:41.960496+00:00", + "replayed_at": "2026-11-10 14:21:42.086003+00:00", + "replay_count": 1, + "job_status": "succeeded", + "replayable": false + }, + { + "attachment_id": "390e3a82-63cd-4a64-a50c-a200f68924c4", + "job_id": "bfe81165-4f51-445a-bf37-07e61e527985", + "error_code": "invalid_image", + "attempts": 1, + "failed_at": "2026-11-10 14:21:32.124484+00:00", + "replayed_at": null, + "replay_count": 0, + "job_status": "failed", + "replayable": true + }, + { + "attachment_id": "5c05fbd5-77b0-4554-87bd-5b315aea3b82", + "job_id": "d9f58adb-d682-4d2f-a94c-16f2ce326ec7", + "error_code": "retry_exhausted", + "attempts": 5, + "failed_at": "2026-11-10 14:09:44.857997+00:00", + "replayed_at": "2026-11-10 14:09:44.981918+00:00", + "replay_count": 1, + "job_status": "succeeded", + "replayable": false + }, + { + "attachment_id": "5a2d84b3-b13f-4c34-83e5-65f42b792752", + "job_id": "41cdf50b-cfd4-4a88-864c-f5a918948014", + "error_code": "invalid_image", + "attempts": 1, + "failed_at": "2026-11-10 14:09:34.398249+00:00", + "replayed_at": null, + "replay_count": 0, + "job_status": "failed", + "replayable": true + } +] + diff --git a/docs/lab5/evidence/reconcile.txt b/docs/lab5/evidence/reconcile.txt new file mode 100644 index 0000000..184a985 --- /dev/null +++ b/docs/lab5/evidence/reconcile.txt @@ -0,0 +1,20 @@ +# 2026-11-10 14:15:14 UTC | Сверка БД <-> S3 (dry-run) | exit 0 | 2.5s +$ python scripts/ops.py reconcile + Container messenger-lab5-api-run-864f3258a8e5 Creating + Container messenger-lab5-api-run-864f3258a8e5 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "staging_to_clean": 0, + "objects_listed": 266, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.053, + "ready_rows": 265, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.066 +} diff --git a/docs/lab5/evidence/rolling_back.json b/docs/lab5/evidence/rolling_back.json new file mode 100644 index 0000000..b6273a7 --- /dev/null +++ b/docs/lab5/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 29.78, + "replicas_before": [ + "cb14462a28f6", + "cc4d20e47b36" + ], + "replicas_after": [ + "9adbb2f64abd", + "cec102b25e41" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1438, + "ok": 1438, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 287.6, + "write_ok_rps": 163.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "cb14462a28f6", + "cc4d20e47b36" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 34.78, + "requests": 8087, + "ok": 8087, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 271.5, + "write_ok_rps": 160.1, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "9adbb2f64abd", + "cb14462a28f6", + "cc4d20e47b36", + "cec102b25e41" + ] + }, + "after": { + "from_s": 34.78, + "to_s": 42.83, + "requests": 1845, + "ok": 1845, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 229.3, + "write_ok_rps": 137.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 79.0, + "p99": 140.0 + }, + "instances": [ + "9adbb2f64abd", + "cec102b25e41" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api3" + ] + ], + [ + 11.92, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.66, + [ + "api1", + "api3" + ] + ], + [ + 15.09, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.36, + [ + "api1", + "api2" + ] + ], + [ + 26.97, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 30.62, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6690, + "stored_messages": 6690, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab5/evidence/rolling_update.json b/docs/lab5/evidence/rolling_update.json new file mode 100644 index 0000000..eaf4bac --- /dev/null +++ b/docs/lab5/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 31.98, + "replicas_before": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "replicas_after": [ + "cb14462a28f6", + "cc4d20e47b36" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1190, + "ok": 1190, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 238.0, + "write_ok_rps": 141.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 36.98, + "requests": 7392, + "ok": 7392, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 231.1, + "write_ok_rps": 135.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 141.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c", + "cb14462a28f6", + "cc4d20e47b36" + ] + }, + "after": { + "from_s": 36.98, + "to_s": 45.02, + "requests": 2038, + "ok": 2038, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 253.7, + "write_ok_rps": 150.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "cb14462a28f6", + "cc4d20e47b36" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 14.41, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 15.08, + [ + "api1", + "api2" + ] + ], + [ + 17.67, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 22.42, + [ + "api2", + "api3" + ] + ], + [ + 30.97, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 33.14, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6255, + "stored_messages": 6255, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab5/evidence/rolling_update.txt b/docs/lab5/evidence/rolling_update.txt new file mode 100644 index 0000000..8be1631 --- /dev/null +++ b/docs/lab5/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-11-10 14:23:55 UTC | Rolling update lab5 -> lab5.1 под нагрузкой | exit 0 | 47.5s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['3a0e06f14a63', '4b1faad0399c'] +2026-11-10T14:23:14.819Z rolling-update: start; replicas=2 old=['4b1faad0399c', '3a0e06f14a63'] +2026-11-10T14:23:20.086Z rolling-update: images built in 5.3s (tag lab5.1) +2026-11-10T14:23:22.594Z rolling-update: migrations once in 2.5s: schema is up to date; nothing applied +2026-11-10T14:23:34.419Z rolling-update: step 1/2 new=cb14462a28f6 (messenger-api:lab5.1, ready+UP in 7.5s) replaced old=4b1faad0399c (graceful stop 4.1s) +2026-11-10T14:23:46.022Z rolling-update: step 2/2 new=cc4d20e47b36 (messenger-api:lab5.1, ready+UP in 7.0s) replaced old=3a0e06f14a63 (graceful stop 4.4s) +2026-11-10T14:23:46.260Z rolling-update: done in 31.8s; replicas now ['cb14462a28f6', 'cc4d20e47b36'] +update 32.0s rc=0 | replicas ['3a0e06f14a63', '4b1faad0399c'] -> ['cb14462a28f6', 'cc4d20e47b36'] | errors during update 0 (-) | RPS before 238.0, during 231.1, after 253.7 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-11-10 14:24:41 UTC | Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой | exit 0 | 45.9s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['cb14462a28f6', 'cc4d20e47b36'] +2026-11-10T14:24:02.303Z rolling-update: start; replicas=2 old=['cb14462a28f6', 'cc4d20e47b36'] +2026-11-10T14:24:05.202Z rolling-update: images built in 2.9s (tag lab5) +2026-11-10T14:24:07.607Z rolling-update: migrations once in 2.4s: schema is up to date; nothing applied +2026-11-10T14:24:19.412Z rolling-update: step 1/2 new=cec102b25e41 (messenger-api:lab5, ready+UP in 7.2s) replaced old=cb14462a28f6 (graceful stop 4.4s) +2026-11-10T14:24:31.138Z rolling-update: step 2/2 new=9adbb2f64abd (messenger-api:lab5, ready+UP in 7.1s) replaced old=cc4d20e47b36 (graceful stop 4.4s) +2026-11-10T14:24:31.579Z rolling-update: done in 29.5s; replicas now ['cec102b25e41', '9adbb2f64abd'] +update 29.8s rc=0 | replicas ['cb14462a28f6', 'cc4d20e47b36'] -> ['9adbb2f64abd', 'cec102b25e41'] | errors during update 0 (-) | RPS before 287.6, during 271.5, after 229.3 | p95 during 78.0 ms | data ok=True +PASS: rolling update under load + +# 2026-11-10 14:24:41 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +cec102b25e41 messenger-lab5-api-5 running healthy messenger-api:lab5 api2 UP +9adbb2f64abd messenger-lab5-api-6 running healthy messenger-api:lab5 api3 UP + diff --git a/docs/lab5/evidence/s3_bootstrap.txt b/docs/lab5/evidence/s3_bootstrap.txt new file mode 100644 index 0000000..a370a05 --- /dev/null +++ b/docs/lab5/evidence/s3_bootstrap.txt @@ -0,0 +1,225 @@ +# 2026-11-10 14:13:53 UTC | S3 bootstrap: лог первого запуска s3-init | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color s3-init +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "created", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +# 2026-11-10 14:14:03 UTC | S3 bootstrap: повторный запуск (идемпотентность) | exit 0 | 9.6s +$ docker compose --env-file .env run --rm -T s3-init + Container messenger-lab5-s3-1 Running + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-s3-init-run-ac08328e286f Creating + Container messenger-lab5-s3-init-run-ac08328e286f Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} + diff --git a/docs/lab5/evidence/s3_outage.json b/docs/lab5/evidence/s3_outage.json new file mode 100644 index 0000000..18fc191 --- /dev/null +++ b/docs/lab5/evidence/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 19.66, + "checks": [ + { + "check": "baseline upload 202 (lab5)", + "ok": true, + "detail": "202" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 5.06s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "5.06s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 5.33s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "5.33s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 5.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "5.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "19.66s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 202 after recovery (same token, no API restart)", + "ok": true, + "detail": "202" + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/s3_outage.txt b/docs/lab5/evidence/s3_outage.txt new file mode 100644 index 0000000..6c80ff1 --- /dev/null +++ b/docs/lab5/evidence/s3_outage.txt @@ -0,0 +1,24 @@ +# 2026-11-10 14:20:18 UTC | Недоступность S3 и восстановление | exit 0 | 47.3s +$ python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json +PASS baseline upload 202 (lab5) — 202 +$ docker compose --env-file .env stop s3 +PASS readiness stays 200 while S3 is down (S3 is not a readiness dependency) — 200 +PASS text message 201 during S3 outage — 201 +PASS history 200 during S3 outage — 200 +PASS attachment metadata 200 (PostgreSQL only) — 200 +PASS message referencing existing attachment 201 — 201 +PASS upload #1 -> 503 unavailable — 503 unavailable in 5.06s +PASS upload #1 failed fast (<= 15s) — 5.06s +PASS upload #1 has Retry-After +PASS upload #2 -> 503 unavailable — 503 unavailable in 5.33s +PASS upload #2 failed fast (<= 15s) — 5.33s +PASS upload #2 has Retry-After +PASS download -> 503 during outage — 503 in 5.01s +PASS download failed fast — 5.01s +PASS session still valid — 200 +$ docker compose --env-file .env start s3 +$ C:\Users\mrend\AppData\Local\Programs\Python\Python312\python.exe scripts/ops.py wait-s3 +PASS S3 read/write warm-up after start — 19.66s +PASS old file downloads byte-for-byte after recovery — 200 0.02s +PASS new upload 202 after recovery (same token, no API restart) — 202 +PASS diff --git a/docs/lab5/evidence/s3_private.txt b/docs/lab5/evidence/s3_private.txt new file mode 100644 index 0000000..efe8586 --- /dev/null +++ b/docs/lab5/evidence/s3_private.txt @@ -0,0 +1,67 @@ +# 2026-11-10 14:15:16 UTC | Приватный бакет вне API, права ключей, закрытые админ-порты | exit 0 | 2.6s +$ python scripts/ops.py s3-check + Container messenger-lab5-api-run-796deabd49b0 Creating + Container messenger-lab5-api-run-796deabd49b0 Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} +# 2026-11-10 14:15:17 UTC | На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes | exit 0 | 0.5s +$ python scripts/ops.py api-fs +messenger-lab5-api-1 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/broker_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/database_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] +messenger-lab5-api-2 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/broker_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/database_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] + +# 2026-11-10 14:15:17 UTC | API: файлы в /tmp (tmpfs) после загрузок | exit 0 | 0.3s +$ docker compose --env-file .env exec -T api sh -c df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l) +Filesystem Size Used Avail Use% Mounted on +tmpfs 16M 0 16M 0% /tmp +files_in_tmp=0 + diff --git a/docs/lab5/evidence/smoke.txt b/docs/lab5/evidence/smoke.txt new file mode 100644 index 0000000..988a689 --- /dev/null +++ b/docs/lab5/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-10 14:14:05 UTC | Публичный smoke | exit 0 | 1.8s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab5/evidence/stand.txt b/docs/lab5/evidence/stand.txt new file mode 100644 index 0000000..81e51f5 --- /dev/null +++ b/docs/lab5/evidence/stand.txt @@ -0,0 +1,61 @@ +# 2026-11-10 14:13:49 UTC | Стенд: контейнеры | exit 0 | 0.3s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab5-api-1 messenger-api:lab5 "python -m src.server" api 28 minutes ago Up 10 minutes (healthy) 8080/tcp +messenger-lab5-api-2 messenger-api:lab5 "python -m src.server" api 28 minutes ago Up 10 minutes (healthy) 8080/tcp +messenger-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 28 minutes ago Up 9 minutes (healthy) 6379/tcp +messenger-lab5-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 28 minutes ago Up 13 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab5-migrate-1 messenger-lab5-migrate "python -m src.migra…" migrate 28 minutes ago Exited (0) 4 minutes ago +messenger-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 28 minutes ago Up 14 minutes (healthy) 5432/tcp +messenger-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 28 minutes ago Up 14 minutes (healthy) 6379/tcp +messenger-lab5-s3-1 chrislusf/seaweedfs:4.50 "/usr/bin/weed serve…" s3 28 minutes ago Up 14 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +messenger-lab5-s3-init-1 messenger-lab5-s3-init "python -m src.s3_bo…" s3-init 28 minutes ago Exited (0) 4 minutes ago +messenger-lab5-worker-1 messenger-api:lab5 "python -m src.worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp +# 2026-11-10 14:13:50 UTC | Стенд: реплики и статус в LB | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:13:52 UTC | Стенд: лимиты CPU/RAM | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab5-worker-1 0.09% 76.75MiB / 1.25GiB 6.00% +messenger-lab5-lb-1 0.13% 12.85MiB / 128MiB 10.04% +messenger-lab5-api-1 0.38% 82.19MiB / 320MiB 25.68% +messenger-lab5-api-2 18.24% 77.13MiB / 320MiB 24.10% +messenger-lab5-postgres-1 3.64% 74.93MiB / 512MiB 14.63% +messenger-lab5-redis-1 0.27% 3.957MiB / 96MiB 4.12% +messenger-lab5-s3-1 2.99% 189.7MiB / 512MiB 37.06% +messenger-lab5-broker-1 0.29% 4.012MiB / 320MiB 1.25% + +# 2026-11-10 14:13:52 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.4s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +broker 6379/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp +s3 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +worker 8080/tcp + +# 2026-11-10 14:13:53 UTC | Образ SeaweedFS (закреплённая версия) | exit 0 | 0.4s +$ docker compose --env-file .env images s3 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +messenger-lab5-s3-1 chrislusf/seaweedfs 4.50 linux/amd64 c2658820e763 734MB 2 weeks ago + +# 2026-11-10 14:13:53 UTC | lab5: broker (Redis AOF) и worker | exit 0 | 0.3s +$ docker compose --env-file .env ps broker worker +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 28 minutes ago Up 10 minutes (healthy) 6379/tcp +messenger-lab5-worker-1 messenger-api:lab5 "python -m src.worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp + +# 2026-11-10 14:13:53 UTC | lab5: конфигурация durability брокера | exit 0 | 0.3s +$ docker compose --env-file .env exec -T broker sh -c REDISCLI_AUTH="$(tr -d '\r\n' < /run/secrets/broker_password)" redis-cli CONFIG GET appendfsync; REDISCLI_AUTH="$(tr -d '\r\n' < /run/secrets/broker_password)" redis-cli CONFIG GET maxmemory-policy +appendfsync +always +maxmemory-policy +noeviction + diff --git a/docs/lab5/evidence/student-tests.txt b/docs/lab5/evidence/student-tests.txt new file mode 100644 index 0000000..f333b67 --- /dev/null +++ b/docs/lab5/evidence/student-tests.txt @@ -0,0 +1,185 @@ +# 2026-11-10 14:14:27 UTC | Собственные тесты (HTTP через LB, включая test_attachments.py) | exit 0 | 21.7s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 91 tests in 19.876s + +OK (skipped=33) + + 24 parallel POSTs, one key -> 1 message; replicas: ['3a0e06f14a63', '4b1faad0399c'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-11-10 14:14:36 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 8.8s +$ python scripts/ops.py in-network storage + Container messenger-lab5-api-run-87d6a97994de Creating + Container messenger-lab5-api-run-87d6a97994de Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.408s + +OK + Container messenger-lab5-api-run-7049d7c00dfc Creating + Container messenger-lab5-api-run-7049d7c00dfc Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 2.032s + +OK + +# 2026-11-10 14:14:51 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 15.7s +$ python scripts/ops.py in-network expiry + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-short-ttl-1 Creating + Container messenger-lab5-api-short-ttl-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-short-ttl-1 Starting + Container messenger-lab5-api-short-ttl-1 Started + Container messenger-lab5-api-run-f4cd12243eab Creating + Container messenger-lab5-api-run-f4cd12243eab Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab5-api-run-249c8116f0a3 Creating + Container messenger-lab5-api-run-249c8116f0a3 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.743s + +OK + diff --git a/docs/lab5/evidence/upload_memory.txt b/docs/lab5/evidence/upload_memory.txt new file mode 100644 index 0000000..5ddd978 --- /dev/null +++ b/docs/lab5/evidence/upload_memory.txt @@ -0,0 +1,161 @@ +# 2026-11-10 14:15:28 UTC | Пик памяти: 10 одновременных загрузок по 9.5 MiB | exit 0 | 11.3s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 10 --rounds 3 --docker --label c=10, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c10.json +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:28Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 6.22, + "latency_201_s": { + "p50": 1.719, + "p95": 1.922, + "max": 2.5309999999990396, + "mean": 1.663 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 76.3, + "peak_mib": 151.7, + "growth_mib": 75.4, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 13.0, + "peak_mib": 30.2, + "growth_mib": 17.3, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 134.6, + "peak_mib": 153.6, + "growth_mib": 19.0, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 136.9, + "peak_mib": 152.8, + "growth_mib": 15.9, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 84.9, + "peak_mib": 86.6, + "growth_mib": 1.7, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 227.9, + "peak_mib": 303.6, + "growth_mib": 75.7, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.2, + "peak_mib": 4.5, + "growth_mib": 0.4, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} +# 2026-11-10 14:15:40 UTC | Пик памяти: 20 одновременных загрузок | exit 0 | 11.5s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 20 --rounds 2 --docker --label c=20, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c20.json +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:40Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "202": 40 + }, + "sha256_ok": 40, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 7.16, + "latency_201_s": { + "p50": 2.984, + "p95": 3.562, + "max": 3.577999999997701, + "mean": 2.695 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 152.5, + "peak_mib": 152.5, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 73.6, + "peak_mib": 74.0, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 155.9, + "peak_mib": 171.2, + "growth_mib": 15.3, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 168.7, + "peak_mib": 168.7, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 86.3, + "peak_mib": 86.7, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.2, + "peak_mib": 5.1, + "growth_mib": 0.9, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 348.9, + "peak_mib": 430.3, + "growth_mib": 81.4, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} + diff --git a/docs/lab5/evidence/upload_memory_c10.json b/docs/lab5/evidence/upload_memory_c10.json new file mode 100644 index 0000000..f15c89d --- /dev/null +++ b/docs/lab5/evidence/upload_memory_c10.json @@ -0,0 +1,78 @@ +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:28Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 6.22, + "latency_201_s": { + "p50": 1.719, + "p95": 1.922, + "max": 2.5309999999990396, + "mean": 1.663 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 76.3, + "peak_mib": 151.7, + "growth_mib": 75.4, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 13.0, + "peak_mib": 30.2, + "growth_mib": 17.3, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 134.6, + "peak_mib": 153.6, + "growth_mib": 19.0, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 136.9, + "peak_mib": 152.8, + "growth_mib": 15.9, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 84.9, + "peak_mib": 86.6, + "growth_mib": 1.7, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 227.9, + "peak_mib": 303.6, + "growth_mib": 75.7, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.2, + "peak_mib": 4.5, + "growth_mib": 0.4, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} diff --git a/docs/lab5/evidence/upload_memory_c20.json b/docs/lab5/evidence/upload_memory_c20.json new file mode 100644 index 0000000..a790ce9 --- /dev/null +++ b/docs/lab5/evidence/upload_memory_c20.json @@ -0,0 +1,78 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:40Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "202": 40 + }, + "sha256_ok": 40, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 7.16, + "latency_201_s": { + "p50": 2.984, + "p95": 3.562, + "max": 3.577999999997701, + "mean": 2.695 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 152.5, + "peak_mib": 152.5, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 73.6, + "peak_mib": 74.0, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 155.9, + "peak_mib": 171.2, + "growth_mib": 15.3, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 168.7, + "peak_mib": 168.7, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 86.3, + "peak_mib": 86.7, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.2, + "peak_mib": 5.1, + "growth_mib": 0.9, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 348.9, + "peak_mib": 430.3, + "growth_mib": 81.4, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} diff --git a/docs/lab5/evidence/worker_log_sample.txt b/docs/lab5/evidence/worker_log_sample.txt new file mode 100644 index 0000000..5865ce0 --- /dev/null +++ b/docs/lab5/evidence/worker_log_sample.txt @@ -0,0 +1,3 @@ +# 2026-11-10 14:23:08 UTC | Worker: последние строки лога | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 80 worker +worker-1 | 2026-11-10 14:23:00,051 INFO messenger.worker worker 853a915daea3: concurrency=2 max_attempts=5 lease=60s claim_idle=75s image_timeout=30.0s image_memory=1024MiB retry=2.0..60.0s faults=off diff --git a/docs/lab5/evidence/worker_outage.json b/docs/lab5/evidence/worker_outage.json new file mode 100644 index 0000000..7f7b8f4 --- /dev/null +++ b/docs/lab5/evidence/worker_outage.json @@ -0,0 +1,80 @@ +[ + { + "seconds": 2.06, + "step": "worker stopped", + "t": 2.42 + }, + { + "status": 202, + "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", + "attachment_status": "queued", + "step": "upload accepted", + "t": 2.44 + }, + { + "metadata_status": "queued", + "content_status": 409, + "message_with_attachment": 201, + "plain_message": 201, + "message_id": "310e494b-eccf-49b1-9424-bfec99522d7b", + "step": "while worker is down", + "t": 5.53 + }, + { + "length": 9, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 148, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "broker before restarts", + "t": 7.22 + }, + { + "seconds": 11.0, + "step": "api replicas restarted", + "t": 18.22 + }, + { + "seconds": 4.58, + "step": "broker restarted (volume kept)", + "t": 22.8 + }, + { + "metadata_status": "queued", + "length": 9, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 148, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "after restarts", + "t": 26.67 + }, + { + "seconds": 6.41, + "step": "worker started", + "t": 33.08 + }, + { + "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "thumbnail_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true, + "step": "ready", + "t": 33.12 + } +] \ No newline at end of file diff --git a/docs/lab5/evidence/worker_outage.txt b/docs/lab5/evidence/worker_outage.txt new file mode 100644 index 0000000..2b67ab7 --- /dev/null +++ b/docs/lab5/evidence/worker_outage.txt @@ -0,0 +1,12 @@ +# 2026-11-10 14:20:58 UTC | Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready | exit 0 | 33.3s +$ python student-tests/worker_outage.py --base-url http://127.0.0.1:8080 --output evidence/worker_outage.json +{"seconds": 2.06, "step": "worker stopped", "t": 2.42} +{"status": 202, "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", "attachment_status": "queued", "step": "upload accepted", "t": 2.44} +{"metadata_status": "queued", "content_status": 409, "message_with_attachment": 201, "plain_message": 201, "message_id": "310e494b-eccf-49b1-9424-bfec99522d7b", "step": "while worker is down", "t": 5.53} +{"length": 9, "lag": 1, "pending": 0, "jobs": {"succeeded": 148, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "broker before restarts", "t": 7.22} +{"seconds": 11.0, "step": "api replicas restarted", "t": 18.22} +{"seconds": 4.58, "step": "broker restarted (volume kept)", "t": 22.8} +{"metadata_status": "queued", "length": 9, "lag": 1, "pending": 0, "jobs": {"succeeded": 148, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "after restarts", "t": 26.67} +{"seconds": 6.41, "step": "worker started", "t": 33.08} +{"attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", "variants": ["thumbnail", "crop", "optimized"], "thumbnail_jpeg_size": [256, 160], "original_sha256_ok": true, "step": "ready", "t": 33.12} +PASS: accepted job survived worker stop + API restart + broker restart and became ready diff --git a/docs/runbooks.md b/docs/runbooks.md new file mode 100644 index 0000000..de9a878 --- /dev/null +++ b/docs/runbooks.md @@ -0,0 +1,45 @@ +# Runbooks (lab6) + +Локальные инструкции для alert rules из `infra/prometheus/rules/messenger.yml`. +Состояние alerts: Prometheus `http://127.0.0.1:9090/alerts` или Grafana → Alerting → Alert rules +(источник Prometheus). Доставка наружу не настроена (не требуется). + +Общий порядок: **метрика говорит «что и сколько» → trace говорит «где именно» → лог говорит «почему»**. + +## ApiHighErrorRate + +**Условие:** `messenger:http_error_ratio:rate1m > 0.05` при трафике ≥ 0.2 RPS, `for: 1m`. +SLI = 5xx / все бизнес-запросы (health исключены инструментацией, 4xx не считаются). + +1. Dashboard «Messenger lab6» → «RPS по маршруту и классу статуса»: какие `http_route` дают 5xx? + Один маршрут (например `/api/v1/chats/{chat_id}/messages`) → проблема в его зависимостях; все маршруты → общая зависимость (БД, сеть). +2. «p95 по маршруту» и «DB pool: занято / ожидают». Растёт `waiting` и p95 ≈ 3 с → пул БД исчерпан (`DB_POOL_TIMEOUT_SECONDS=3`). p95 ≈ 5 с → `statement_timeout` (`DB_STATEMENT_TIMEOUT_MS=5000`) — БД медленная или заблокирована. +3. Grafana → Explore → Tempo, TraceQL: `{ resource.service.name="messenger-api" && status=error }`. + Откройте trace: если длинный и красный — CLIENT span `INSERT`/`SELECT` (psycopg) с `QueryCanceled` → виновата БД, handler ни при чём. Если SQL-spans короткие, а SERVER span длинный → медленный код handler'а. +4. Кнопка «Logs for this span» (или Loki `{service_name="messenger-api"} | trace_id=""`) — лог `messenger.api database unavailable: QueryCanceled`. +5. Проверка БД: `docker compose --env-file .env exec postgres psql -U messenger -d messenger -c "select pid, state, wait_event_type, now()-query_start as dur, left(query,60) from pg_stat_activity where datname='messenger' order by dur desc nulls last;"` + Видна блокирующая транзакция (`LOCK TABLE ...`, `idle in transaction`) → завершить: `select pg_terminate_backend();` +6. Resolved: доля 5xx < 5% — через ~20–60 с после устранения (окно 1m). + +## QueueLagGrowing + +**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.jobs"}) > 60`, `for: 30s`. +Метрику публикуют API-реплики из PostgreSQL, поэтому она жива и при остановленном worker. + +1. Dashboard → «Глубина очереди»: растёт `queued`, а `processing` = 0 → никто не берёт задания. + `outbox (не опубликовано)` > 0 → проблема до брокера (брокер недоступен), а не в worker. +2. «Saturation worker»: серии `capacity` нет → worker не работает (`docker compose ps worker`). + busy/capacity = 100% долгое время → worker жив, но не успевает: смотреть «Длительность задания p95» и «Задания по исходу» (много `retry`/`error`?). +3. Tempo: `{ resource.service.name="messenger-worker" && kind=consumer && status=error }` — ошибки обработки; в trace видно, на каком S3/SQL span упала попытка, retry идут в той же трассе. +4. Loki: `{service_name="messenger-worker"} | event="job.processed" | outcome!="succeeded"`. +5. Действие: `python scripts/ops.py worker-start` (или `docker compose --env-file .env up -d worker`); при постоянных ошибках — `make jobs-stats`, `make jobs-failed`. +6. Resolved: после запуска worker очередь разбирается (в замере — за секунды), возраст → 0. + +## TelemetryExportFailing + +**Условие:** Collector не отправляет spans/logs (`otelcol_exporter_send_failed_*` или `enqueue_failed_*` растут), `for: 1m`. + +1. Dashboard → «Collector: принято / отправлено / потеряно» и «очередь экспорта»: `queue_size` упирается в 20 → backend недоступен. +2. `docker compose --env-file .env ps tempo loki` — какой backend лежит; `docker compose logs --tail 50 otel-collector`. +3. API и worker продолжают работать: проверить `make test`. Потери телеметрии = значение `enqueue_failed_*` + `send_failed_*` за период. +4. Действие: `docker compose --env-file .env up -d tempo loki`. После возврата очередь опустошается, `sent_*` растёт. diff --git a/evidence/README.md b/evidence/README.md index 8df7001..019c34a 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,9 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Содержимое (lab6) + +- `local-sandbox/` — прогон в песочнице без Docker Hub (те же исходники и конфиги; PostgreSQL 16.2, Redis 6.2, + S3-эмулятор moto, 1 реплика API; настоящие Collector 0.115 / Prometheus 2.55.1 / Loki 3.3.2 / Tempo 2.6.1). Описание условий — REPORT.md, раздел «Доказательства критериев». +- Результаты Docker-стенда - в корне `evidence/`. diff --git a/evidence/alerts_db.txt b/evidence/alerts_db.txt new file mode 100644 index 0000000..581f622 Binary files /dev/null and b/evidence/alerts_db.txt differ diff --git a/evidence/alerts_worker.txt b/evidence/alerts_worker.txt new file mode 100644 index 0000000..fbb4d75 Binary files /dev/null and b/evidence/alerts_worker.txt differ diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..80f5d36 Binary files /dev/null and b/evidence/check.txt differ diff --git a/evidence/incident_db.jsonl b/evidence/incident_db.jsonl new file mode 100644 index 0000000..35458b3 Binary files /dev/null and b/evidence/incident_db.jsonl differ diff --git a/evidence/incident_worker.jsonl b/evidence/incident_worker.jsonl new file mode 100644 index 0000000..1e1a3fd Binary files /dev/null and b/evidence/incident_worker.jsonl differ diff --git a/evidence/local-sandbox/alerts_db_lock_180s.txt b/evidence/local-sandbox/alerts_db_lock_180s.txt new file mode 100644 index 0000000..6fb5b28 --- /dev/null +++ b/evidence/local-sandbox/alerts_db_lock_180s.txt @@ -0,0 +1,6 @@ +17:59:03 lock start +18:02:03 lock end +2026-11-11T17:58:48Z watch start http://127.0.0.1:9090 +2026-11-11T18:00:06Z ApiHighErrorRate pending value=1e+00 active_since=2026-11-11T18:00:03.736030136Z +2026-11-11T18:01:06Z ApiHighErrorRate firing value=1e+00 active_since=2026-11-11T18:00:03.736030136Z +2026-11-11T18:02:24Z ApiHighErrorRate resolved diff --git a/evidence/local-sandbox/alerts_queue_lag_and_db_short.txt b/evidence/local-sandbox/alerts_queue_lag_and_db_short.txt new file mode 100644 index 0000000..f6418fb --- /dev/null +++ b/evidence/local-sandbox/alerts_queue_lag_and_db_short.txt @@ -0,0 +1,6 @@ +2026-11-11T17:53:27Z watch start http://127.0.0.1:9090 +2026-11-11T17:54:45Z ApiHighErrorRate pending value=6.666666666666666e-01 active_since=2026-11-11T17:54:43.736030136Z +2026-11-11T17:54:45Z QueueLagGrowing pending value=6.0948187e+01 active_since=2026-11-11T17:54:43.736030136Z +2026-11-11T17:55:15Z QueueLagGrowing firing value=9.0974623e+01 active_since=2026-11-11T17:54:43.736030136Z +2026-11-11T17:55:36Z ApiHighErrorRate resolved +2026-11-11T17:55:36Z QueueLagGrowing resolved diff --git a/evidence/local-sandbox/db_lock_span.txt b/evidence/local-sandbox/db_lock_span.txt new file mode 100644 index 0000000..83aac0e --- /dev/null +++ b/evidence/local-sandbox/db_lock_span.txt @@ -0,0 +1,6 @@ +Инцидент «медленная БД»: LOCK TABLE messages IN ACCESS EXCLUSIVE MODE на 15 с под нагрузкой. +load.py (4 потока, 12 с): successful 4, failed 8, error_ratio 0.667 +Метрика: http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/messages",http_response_status_code="503",error_type="503"} 8 +Trace: CLIENT span "INSERT" 5.01 s, status ERROR "QueryCanceled: canceling statement due to statement timeout", + event exception.type=psycopg.errors.QueryCanceled -> время ушло в SQL (statement_timeout 5 с), а не в handler. +Длинный прогон (lock 180 с) и состояния ApiHighErrorRate — alerts_db_lock_180s.txt diff --git a/evidence/local-sandbox/incident_worker_stop.jsonl b/evidence/local-sandbox/incident_worker_stop.jsonl new file mode 100644 index 0000000..dd82cd0 --- /dev/null +++ b/evidence/local-sandbox/incident_worker_stop.jsonl @@ -0,0 +1,25 @@ +{"step": "exec", "t": 0.1, "cmd": "/tmp/it/stop_worker.sh"} +{"step": "upload_while_worker_down", "t": 0.1, "status": 202, "request_id": "probe-81b73ce394ae"} +{"step": "upload_while_worker_down", "t": 5.2, "status": 202, "request_id": "probe-099629d749fd"} +{"step": "upload_while_worker_down", "t": 10.2, "status": 202, "request_id": "probe-985b23bb6d03"} +{"step": "upload_while_worker_down", "t": 15.2, "status": 202, "request_id": "probe-2d61f4b6cd93"} +{"step": "upload_while_worker_down", "t": 20.3, "status": 202, "request_id": "probe-30099c838b61"} +{"step": "upload_while_worker_down", "t": 25.3, "status": 202, "request_id": "probe-d789fddd7157"} +{"step": "upload_while_worker_down", "t": 30.3, "status": 202, "request_id": "probe-7fa609b8c10e"} +{"step": "upload_while_worker_down", "t": 35.3, "status": 202, "request_id": "probe-159dfe385e83"} +{"step": "upload_while_worker_down", "t": 40.4, "status": 202, "request_id": "probe-195c69d91227"} +{"step": "upload_while_worker_down", "t": 45.4, "status": 202, "request_id": "probe-c1c1bdea3f27"} +{"step": "upload_while_worker_down", "t": 50.4, "status": 202, "request_id": "probe-1a1a61531996"} +{"step": "upload_while_worker_down", "t": 55.5, "status": 202, "request_id": "probe-8939388f4388"} +{"step": "upload_while_worker_down", "t": 60.5, "status": 202, "request_id": "probe-a23a88d40d77"} +{"step": "upload_while_worker_down", "t": 65.5, "status": 202, "request_id": "probe-bf9e82d4b5b6"} +{"step": "upload_while_worker_down", "t": 70.5, "status": 202, "request_id": "probe-e0e98298a112"} +{"step": "upload_while_worker_down", "t": 75.6, "status": 202, "request_id": "probe-2a5a8eb7a055"} +{"step": "upload_while_worker_down", "t": 80.6, "status": 202, "request_id": "probe-3efa69f3c78e"} +{"step": "upload_while_worker_down", "t": 85.6, "status": 202, "request_id": "probe-281847650ffd"} +{"step": "upload_while_worker_down", "t": 90.7, "status": 202, "request_id": "probe-205546b70ca0"} +{"step": "upload_while_worker_down", "t": 95.7, "status": 202, "request_id": "probe-e352076974d9"} +{"step": "upload_while_worker_down", "t": 100.7, "status": 202, "request_id": "probe-64e5650653c9"} +{"step": "upload_while_worker_down", "t": 105.7, "status": 202, "request_id": "probe-6eee887affe3"} +{"step": "exec", "t": 110.7, "cmd": "/tmp/it/start_worker.sh"} +{"step": "drained", "t": 119.4, "uploads": 22, "not_ready": 0, "drain_seconds": 8.7} diff --git a/evidence/local-sandbox/metric_names.txt b/evidence/local-sandbox/metric_names.txt new file mode 100644 index 0000000..242c089 --- /dev/null +++ b/evidence/local-sandbox/metric_names.txt @@ -0,0 +1,32 @@ +# HELP http_server_active_requests Number of active HTTP server requests. +# TYPE http_server_active_requests gauge +# HELP http_server_request_duration_seconds Duration of HTTP server requests. +# TYPE http_server_request_duration_seconds histogram +# HELP messenger_attachment_time_to_ready_seconds От durable acceptance (COMMIT задания, ответ 202) до публикации ready +# TYPE messenger_attachment_time_to_ready_seconds histogram +# HELP messenger_db_pool_connections +# TYPE messenger_db_pool_connections gauge +# HELP messenger_db_pool_max +# TYPE messenger_db_pool_max gauge +# HELP messenger_db_pool_waiting Запросы, ждущие свободное соединение +# TYPE messenger_db_pool_waiting gauge +# HELP messenger_job_duration_seconds Длительность обработки задания worker'ом +# TYPE messenger_job_duration_seconds histogram +# HELP messenger_jobs_processed_total Обработанные сообщения заданий по исходу (succeeded/retry/failed/noop/...) +# TYPE messenger_jobs_processed_total counter +# HELP messenger_jobs_published_total Сообщения, опубликованные outbox relay в брокер +# TYPE messenger_jobs_published_total counter +# HELP messenger_outbox_pending Строки outbox, ещё не опубликованные в брокер +# TYPE messenger_outbox_pending gauge +# HELP messenger_queue_depth Незавершённые задания +# TYPE messenger_queue_depth gauge +# HELP messenger_queue_oldest_age_seconds Возраст старейшего незавершённого задания +# TYPE messenger_queue_oldest_age_seconds gauge +# HELP messenger_worker_busy Занятые consumer-потоки +# TYPE messenger_worker_busy gauge +# HELP messenger_worker_capacity WORKER_CONCURRENCY +# TYPE messenger_worker_capacity gauge +# HELP process_cpu_time_seconds_total CPU-время процесса и его подпроцессов +# TYPE process_cpu_time_seconds_total counter +# HELP process_memory_rss_bytes RSS процесса +# TYPE process_memory_rss_bytes gauge diff --git a/evidence/local-sandbox/smoke.txt b/evidence/local-sandbox/smoke.txt new file mode 100644 index 0000000..7e5c8e2 --- /dev/null +++ b/evidence/local-sandbox/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local-sandbox/telemetry_outage.txt b/evidence/local-sandbox/telemetry_outage.txt new file mode 100644 index 0000000..9efc090 --- /dev/null +++ b/evidence/local-sandbox/telemetry_outage.txt @@ -0,0 +1,21 @@ +Локальный прогон (песочница, без Docker): API x1, worker x1, нагрузка tests/load.py --concurrency 4 (closed loop, POST messages). + +A. Collector остановлен (SDK не может отправить ничего), 45 с: + до: 15 с, 4129 успешных, 0 ошибок, 274.9 RPS; RSS API 126.7 MiB + без коллектора: 45 с, 13663 успешных, 0 ошибок, 303.5 RPS; RSS API 135.0 -> 135.4 MiB (не растёт) + после возврата: 10 с, 2976 успешных, 0 ошибок, 297.1 RPS + SDK: OTLP-экспорт падает с ConnectionError (без повторов), батч отбрасывается; + очереди BatchSpanProcessor/BatchLogRecordProcessor ограничены 2048 записями. + В stdout API — одна JSON-строка на неудачный экспорт (143 строки за 45 с), без трейсбеков, НЕ в OTLP. + +B. Tempo и Loki остановлены, Collector работает, 120 с, ~300 RPS: + API: 36028 успешных, 0 ошибок, 300.2 RPS + Collector (GOMEMLIMIT=200MiB, memory_limiter 180 MiB, sending_queue 20 батчей): пик RSS 234 MiB + otelcol_exporter_queue_size: 20/20 (tempo, loki) — очередь полна, дальше отбрасывание + otelcol_exporter_enqueue_failed_spans 131083, enqueue_failed_log_records 22016 (потери посчитаны) + otelcol_exporter_send_failed_spans 2048, send_failed_log_records 2363 (истёк max_elapsed_time 60 с) + otelcol_receiver_refused_spans 0 + После запуска Tempo/Loki: queue_size -> 0, sent_spans/sent_log_records снова растут, + новый probe виден в Tempo и Loki. + Первый прогон с queue_size 100 дал RSS 290 MiB (> лимита 256M) -> очередь уменьшена до 20, + лимит контейнера Collector поднят до 384M. diff --git a/evidence/local-sandbox/trace_retry_chain.txt b/evidence/local-sandbox/trace_retry_chain.txt new file mode 100644 index 0000000..dae3f84 --- /dev/null +++ b/evidence/local-sandbox/trace_retry_chain.txt @@ -0,0 +1,77 @@ +Tempo search: { span.messenger.request_id = "probe-6f88bf5e5b0d" } -> trace 735b28c5cb7f27799037d11314a8f3c7, spans 76 +messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments 12.5 ms + messenger-api CLIENT messenger 0.1 ms + messenger-api CLIENT SELECT 0.3 ms + messenger-api CLIENT messenger 0.2 ms + messenger-api CLIENT INSERT 0.7 ms + messenger-api CLIENT S3.PutObject 4.7 ms + messenger-api CLIENT messenger 0.2 ms + messenger-api CLIENT UPDATE 0.5 ms + messenger-api CLIENT INSERT 0.3 ms + messenger-api CLIENT INSERT 0.2 ms + messenger-api CLIENT INSERT 0.1 ms + messenger-api CLIENT messenger 1.3 ms + messenger-api CLIENT SELECT 0.7 ms + messenger-api PRODUCER attachments.jobs publish 1.1 ms attempt=1 outcome=None + messenger-worker CONSUMER attachments.jobs process 9.6 ms attempt=1 outcome=retry + messenger-worker CLIENT messenger 0.9 ms + messenger-worker CLIENT SELECT 1.0 ms + messenger-worker CLIENT UPDATE 0.8 ms + messenger-worker CLIENT UPDATE 1.3 ms + messenger-worker CLIENT INSERT 0.5 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.4 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT UPDATE 0.2 ms + messenger-worker CLIENT INSERT 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-api PRODUCER attachments.jobs publish 2.4 ms attempt=2 outcome=None + messenger-worker CONSUMER attachments.jobs process 9.5 ms attempt=2 outcome=retry + messenger-worker CLIENT messenger 0.7 ms + messenger-worker CLIENT SELECT 0.8 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT UPDATE 0.5 ms + messenger-worker CLIENT INSERT 0.2 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT INSERT 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-api PRODUCER attachments.jobs publish 1.1 ms attempt=3 outcome=None + messenger-worker CONSUMER attachments.jobs process 296.9 ms attempt=3 outcome=succeeded + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.5 ms + messenger-worker CLIENT UPDATE 0.6 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT S3.GetObject 5.6 ms + messenger-worker CLIENT S3.HeadObject 6.7 ms + messenger-worker CLIENT S3.PutObject 4.4 ms + messenger-worker CLIENT S3.PutObject 5.3 ms + messenger-worker CLIENT S3.PutObject 13.8 ms + messenger-worker CLIENT S3.PutObject 13.4 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT UPDATE 0.6 ms + messenger-worker CLIENT INSERT 0.8 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-worker CLIENT INSERT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT S3.DeleteObject 3.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-api CLIENT DELETE 1.4 ms + messenger-api CLIENT INSERT 0.6 ms diff --git a/evidence/local-sandbox/trace_upload_chain.txt b/evidence/local-sandbox/trace_upload_chain.txt new file mode 100644 index 0000000..26e5979 --- /dev/null +++ b/evidence/local-sandbox/trace_upload_chain.txt @@ -0,0 +1,43 @@ +TRACE f2e461f369476c9ad5bab71657bd71b9 spans: 40 +messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments 21.9ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT SELECT 1.3ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT INSERT 1.3ms + messenger-api CLIENT S3.PutObject 6.5ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT UPDATE 1.1ms + messenger-api CLIENT INSERT 0.6ms + messenger-api CLIENT INSERT 0.7ms + messenger-api CLIENT INSERT 0.5ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT SELECT 0.3ms + messenger-api PRODUCER attachments.jobs publish 1.4ms + messenger-worker CONSUMER attachments.jobs process 323.1ms + messenger-worker CLIENT messenger 0.6ms + messenger-worker CLIENT SELECT 0.7ms + messenger-worker CLIENT UPDATE 0.4ms + messenger-worker CLIENT UPDATE 0.8ms + messenger-worker CLIENT INSERT 0.6ms + messenger-worker CLIENT messenger 0.3ms + messenger-worker CLIENT SELECT 0.5ms + messenger-worker CLIENT S3.GetObject 10.8ms + messenger-worker CLIENT S3.HeadObject 7.6ms + messenger-worker CLIENT S3.PutObject 15.6ms + messenger-worker CLIENT S3.PutObject 16.5ms + messenger-worker CLIENT S3.PutObject 4.7ms + messenger-worker CLIENT S3.PutObject 5.4ms + messenger-worker CLIENT messenger 0.2ms + messenger-worker CLIENT UPDATE 0.9ms + messenger-worker CLIENT INSERT 1.1ms + messenger-worker CLIENT UPDATE 0.8ms + messenger-worker CLIENT INSERT 0.4ms + messenger-worker CLIENT messenger 0.2ms + messenger-worker CLIENT SELECT 1.9ms + messenger-worker CLIENT S3.DeleteObject 3.8ms + messenger-worker CLIENT messenger 0.4ms + messenger-worker CLIENT UPDATE 0.6ms + messenger-api CLIENT DELETE 1.8ms + messenger-api CLIENT INSERT 2.2ms +total traces 65537 spans 263520 +root CLIENT spans (should be 0): 0 diff --git a/evidence/probe.txt b/evidence/probe.txt new file mode 100644 index 0000000..fd2c00e Binary files /dev/null and b/evidence/probe.txt differ diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..7fff3da Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/stand.txt b/evidence/stand.txt new file mode 100644 index 0000000..7742881 Binary files /dev/null and b/evidence/stand.txt differ diff --git a/evidence/telemetry_down.jsonl b/evidence/telemetry_down.jsonl new file mode 100644 index 0000000..4067907 Binary files /dev/null and b/evidence/telemetry_down.jsonl differ diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..f71ab65 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,944 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger lab6 — API, worker, очередь", + "tags": [ + "lab6", + "messenger" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "annotations": { + "list": [] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "API: RED (бизнес-запросы, health исключены)", + "id": 1, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "timeseries", + "title": "RPS по маршруту и классу статуса", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "rate(http_server_request_duration_seconds_count[1m]); класс статуса из http_response_status_code", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (http_route, status_class) (label_replace(rate(http_server_request_duration_seconds_count{job=\"messenger-api\"}[1m]), \"status_class\", \"${1}xx\", \"http_response_status_code\", \"(\\\\d)..\"))", + "legendFormat": "{{http_route}} {{status_class}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Доля 5xx (SLI ошибок)", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "numerator: 5xx; denominator: все бизнес-запросы. Порог алерта 5%", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "custom": { + "thresholdsStyle": { + "mode": "line" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_error_ratio:rate1m", + "legendFormat": "5xx / все, окно 1m", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Латентность p50 / p95 / p99 (из histogram)", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Квантили из buckets всех реплик (sum by le), не среднее p95 реплик", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "p95 по маршруту", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "stat", + "title": "SLO доступности, окно 1 ч (цель 99%)", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "1 - 5xx/все за 1h", + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_availability:ratio1h", + "legendFormat": "availability 1h", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "stat", + "title": "SLO задержки: доля < 0.5 с, 1 ч (цель 95%)", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_fast_ratio:ratio1h", + "legendFormat": "fast 1h", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Worker и очередь attachments.jobs", + "id": 8, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + } + }, + { + "type": "timeseries", + "title": "Задания по исходу", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "succeeded / retry / failed / noop / lease_lost / poison / error", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Длительность задания p50 / p95 / p99", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Время до ready p50 / p95 (end-to-end)", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "От COMMIT задания (ответ 202) до COMMIT ready", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Глубина очереди", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Из PostgreSQL (публикуют API-реплики, max по репликам)", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max by (state) (messenger_queue_depth{queue=\"attachments.jobs\"})", + "legendFormat": "{{state}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max(messenger_outbox_pending{queue=\"attachments.jobs\"})", + "legendFormat": "outbox (не опубликовано)", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Возраст старейшего задания", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Порог алерта QueueLagGrowing: 60 с", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "custom": { + "thresholdsStyle": { + "mode": "line" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.jobs\"})", + "legendFormat": "oldest age", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Saturation worker (busy / capacity)", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "capacity (слотов)" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "targets": [ + { + "expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)", + "legendFormat": "busy/capacity", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "capacity (слотов)", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Ресурсы: DB pool, CPU, RAM", + "id": 15, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + } + }, + { + "type": "timeseries", + "title": "DB pool: занято / ожидают / максимум", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (job, instance) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{job}} {{instance}} used", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum by (job, instance) (messenger_db_pool_waiting)", + "legendFormat": "{{job}} {{instance}} waiting", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max by (job) (messenger_db_pool_max)", + "legendFormat": "{{job}} max", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "CPU процесса (ядра)", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "rate(process_cpu_time_seconds_total{job=~\"messenger-.+\"}[1m])", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "RSS процесса", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_memory_rss_bytes{job=~\"messenger-.+\"}", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Конвейер телеметрии (Collector)", + "id": 19, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + } + }, + { + "type": "timeseries", + "title": "Collector: принято / отправлено / потеряно (spans)", + "id": 20, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(otelcol_receiver_accepted_spans[1m]))", + "legendFormat": "accepted", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_sent_spans[1m]))", + "legendFormat": "sent", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "(sum(rate(otelcol_exporter_send_failed_spans[1m])) or vector(0)) + (sum(rate(otelcol_exporter_enqueue_failed_spans[1m])) or vector(0))", + "legendFormat": "lost", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_receiver_refused_spans[1m]))", + "legendFormat": "refused (memory_limiter)", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Collector: логи и очередь экспорта", + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(otelcol_exporter_sent_log_records[1m]))", + "legendFormat": "logs sent", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "(sum(rate(otelcol_exporter_send_failed_log_records[1m])) or vector(0)) + (sum(rate(otelcol_exporter_enqueue_failed_log_records[1m])) or vector(0))", + "legendFormat": "logs lost", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "queue {{exporter}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Логи и трассы", + "id": 22, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + } + }, + { + "type": "logs", + "title": "Логи WARNING/ERROR API и worker", + "id": 23, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARNING|ERROR|CRITICAL\"", + "queryType": "range", + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + } + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "type": "table", + "title": "Трассы обработки заданий (consumer spans)", + "id": 24, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "query": "{resource.service.name=\"messenger-worker\" && kind=consumer}", + "queryType": "traceql", + "limit": 20, + "tableType": "traces", + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..3d454f6 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..7e89f18 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,50 @@ +# Grafana 11.4.0: источники данных (не хранит сигналы сама — только запросы и UI). +# Фиксированные uid: на них ссылаются dashboard и переходы trace <-> logs. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 10s + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + # trace_id хранится как structured metadata записи -> кнопка перехода в Tempo. + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: "Открыть trace" + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: true + filterBySpanID: false + customQuery: true + # Все логи API и worker этой трассы (trace_id — structured metadata, не текст строки). + query: '{service_name=~"messenger-.+"} | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..62706b6 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,104 @@ +# HAProxy 2.8 LTS — единственная публичная точка входа (lab3, дополнено в lab4). +# +# Обнаружение реплик: Docker DNS (127.0.0.11) отдаёт по имени сервиса `api` +# A-записи ВСЕХ запущенных контейнеров сервиса. server-template держит до 6 слотов +# и каждые 2 с синхронизирует их с DNS: новый контейнер (scale / rolling update) +# появляется сам, остановленный исчезает. Перезапуск LB при масштабировании не нужен. +# +# Исключение недоступного backend (активные проверки): +# GET /health/ready каждые inter=1s; после первой неудачи — fastinter=500ms; +# fall=2 неудачи подряд -> DOWN; rise=2 удачи -> UP; timeout check 1500ms. +# Время обнаружения: +# * graceful stop (реплика сразу отвечает 503): <= 1 с + 0.5 с ~ 1.5 с +# * crash / kill (соединение отклонено/сброшено): ~1.5 с +# * зависание без ответа: <= 1 + 1.5 + 0.5 + 1.5 = 4.5 с +# Пока реплика ещё не помечена DOWN, запрос к ней, не сумевший СОЕДИНИТЬСЯ +# (timeout connect 1s), повторяется на другой реплике (retries 2 + redispatch). +# Повторяются только ошибки установления соединения (retry-on conn-failure по +# умолчанию): запрос ещё не был отправлен, поэтому это безопасно и для POST. +# Уже отправленные POST LB не повторяет — это делает клиент с тем же +# Idempotency-Key (контракт запрещает прозрачные повторы POST /users, /chats). +# +# Согласование таймаутов: +# API: пул БД 3 с + statement_timeout 5 с = максимум ~8 с на запрос +# -> timeout server 10s; graceful shutdown API 10 с; drain API 3 с > 1.5 с. +# lab4: загрузка вложения (POST .../attachments) — до UPLOAD_READ_TIMEOUT_SECONDS +# 60 с чтения тела + очередь 5 с + S3 PUT (2 попытки по ~12 с) -> для этого +# пути timeout server 90s (http-request set-timeout), остальные — прежние 10 с. +# Скачивание идёт потоком, timeout server — это пауза между пакетами, не общая длительность. +# Docker stop_grace_period 20 с > drain 3 с + graceful 10 с. + +global + log stdout format raw local0 info + maxconn 2000 + +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + # 8 КБ: в ответ DNS помещаются A-записи всех слотов. + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + option redispatch 1 + retries 2 + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1500ms + default-server inter 1s fastinter 500ms downinter 2s fall 2 rise 2 + +frontend public + bind :8080 + # Сквозной идентификатор запроса: HAProxy генерирует X-Request-Id, если клиент + # не прислал свой; API отражает его в ответе и в теле ошибок, LB — в логе. + unique-id-format %{+X}o\ %ci%cp-%fp-%Ts%ms-%rt-%pid + http-request set-header X-Request-Id %[unique-id] unless { req.hdr(X-Request-Id) -m found } + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[capture.req.hdr(0)] inst=%[capture.res.hdr(0)] \"%HM %HPO\"" + http-request capture req.hdr(X-Request-Id) len 64 + declare capture response len 32 + http-response capture res.hdr(X-Instance-Id) id 0 + default_backend api + +backend api + balance roundrobin + # lab4: только загрузка вложения получает длинный timeout server (см. выше). + acl is_upload method POST + acl is_upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 90s if is_upload is_upload_path + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # Ответы, которые формирует сам LB (нет ни одной готовой реплики, таймаут), + # — в общем JSON-формате ошибок контракта, с тем же X-Request-Id. + http-error status 503 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API instance is available\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API instance timeout\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 502 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Bad gateway\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + # До 6 реплик (в т.ч. N+1 во время rolling update). init-addr none: LB стартует, + # даже если реплик ещё нет; слоты без адреса находятся в MAINT. + server-template api 1-6 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +# Внутренняя страница состояния (UP/DOWN по репликам) — только чтение. +# Публикуется на 127.0.0.1:8404 хоста; используется scripts/ops.py. +frontend stats + bind :8404 + monitor-uri /lb-health + stats enable + stats uri /stats + stats refresh 2s + stats show-legends + http-request set-log-level silent diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..974ecee --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,45 @@ +# Loki 3.3.2 (single binary) — хранилище ЛОГОВ. Приём — native OTLP (/otlp) от Collector. +# Resource-атрибуты service.name / service.instance.id становятся index labels +# (service_name, service_instance_id), атрибуты записи (event, request_id, trace_id, +# span_id, severity) — structured metadata (не labels: низкая кардинальность индекса). +auth_enabled: false + +server: + http_listen_port: 3100 + log_level: warn + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + # Защита от взрыва: не больше 2 MiB/s на tenant на учебном стенде. + ingestion_rate_mb: 2 + ingestion_burst_size_mb: 4 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..d5b9be0 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,96 @@ +# OpenTelemetry Collector (contrib 0.115.1) — получатель/обработчик/экспортёр, НЕ хранилище. +# +# API/worker --OTLP/HTTP :4318--> receivers.otlp +# traces -> memory_limiter -> batch -> otlp/tempo (Tempo, OTLP gRPC) +# metrics -> memory_limiter -> batch -> prometheus (pull: Prometheus скрейпит :8889) +# logs -> memory_limiter -> batch -> otlphttp/loki (Loki native OTLP /otlp) +# +# Ограничение памяти и буферов (поведение при недоступном backend): +# * memory_limiter: при > limit_mib collector отказывает новым данным (SDK получает +# ошибку и отбрасывает свой батч), принудительный GC; контейнер ограничен 256M. +# * sending_queue: в памяти, не больше queue_size батчей на exporter; при +# переполнении новые батчи отбрасываются (otelcol_exporter_enqueue_failed_*). +# * retry_on_failure: повтор не дольше max_elapsed_time, затем батч отбрасывается +# (otelcol_exporter_send_failed_*). Бесконечного накопления нет. +# Порты наружу не публикуются: 4317/4318/8888/8889 доступны только в сети telemetry. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + send_batch_size: 512 + send_batch_max_size: 1024 + timeout: 2s + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 20 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 20 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + prometheus: + endpoint: 0.0.0.0:8889 + # job = service.name, instance = service.instance.id; прочие resource-атрибуты — в target_info. + resource_to_telemetry_conversion: + enabled: false + # Серии остановленной реплики исчезают через 5 минут (а не висят вечно). + metric_expiration: 5m + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: normal + address: 0.0.0.0:8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..4090b88 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,24 @@ +# Prometheus v2.55.1 — хранилище МЕТРИК (TSDB на volume promdata, retention 7d) +# и вычислитель alert rules (состояние firing/resolved видно в /alerts и Grafana; +# доставка наружу не настроена — Alertmanager не нужен по заданию). +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + # Метрики приложения, которые Collector получил по OTLP. honor_labels: сохраняем + # job= и instance=, выставленные exporter'ом. + - job_name: otel-collector-app + honor_labels: true + static_configs: + - targets: ["otel-collector:8889"] + # Собственные метрики Collector: принятые/отброшенные/неотправленные spans, logs, points. + - job_name: otel-collector-self + static_configs: + - targets: ["otel-collector:8888"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..8492445 --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,66 @@ +# SLI/SLO и alert rules lab6. Единицы: доли (0..1), секунды. +# +# SLI доступности API: доля бизнес-запросов (health исключены инструментацией) +# без ответа 5xx. 4xx (неверный пароль, 404, 409) — ошибка клиента, не сервиса. +# SLO: 99% запросов без 5xx в скользящем окне 1 ч (учебный стенд; бюджет 1%). +# SLI задержки: доля запросов быстрее 0.5 с (корзина le="0.5" гистограммы); SLO 95% за 1 ч. +# SLI очереди: возраст старейшего незавершённого задания; цель < 60 с. +groups: + - name: messenger-sli + interval: 10s + rules: + - record: messenger:http_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m])) + - record: messenger:http_errors_5xx:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_response_status_code=~"5.."}[1m])) or vector(0) + - record: messenger:http_error_ratio:rate1m + expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m + - record: messenger:http_availability:ratio1h + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{job="messenger-api",http_response_status_code=~"5.."}[1h])) or vector(0)) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api"}[1h])) + ) + - record: messenger:http_fast_ratio:ratio1h + expr: | + sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api",le="0.5"}[1h])) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api"}[1h])) + + - name: messenger-alerts + rules: + - alert: ApiHighErrorRate + # >5% ответов 5xx за 1 мин при заметном трафике (>= 0.2 RPS), держится 1 мин. + expr: messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m >= 0.2 + for: 1m + labels: + severity: page + annotations: + summary: "API: доля 5xx {{ $value | humanizePercentage }} (> 5%)" + description: "Бизнес-запросы messenger-api отвечают 5xx. Окно 1m, порог 5%, for 1m." + runbook: "docs/runbooks.md#apihigherrorrate" + + - alert: QueueLagGrowing + # Старейшее незавершённое задание старше 60 с — worker не успевает или стоит. + expr: max(messenger_queue_oldest_age_seconds{queue="attachments.jobs"}) > 60 + for: 30s + labels: + severity: page + annotations: + summary: "Очередь attachments.jobs: старейшее задание {{ $value | humanizeDuration }}" + description: "Возраст старейшего queued/processing задания > 60 с в течение 30 с." + runbook: "docs/runbooks.md#queuelaggrowing" + + - alert: TelemetryExportFailing + # Collector не может отправить данные в backend (Tempo/Loki недоступны). + # Счётчики появляются лениво (enqueue_failed — после первого отказа): "or vector(0)". + expr: | + (sum(rate(otelcol_exporter_send_failed_spans[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_send_failed_log_records[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_spans[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_log_records[1m])) or vector(0)) > 0 + for: 1m + labels: + severity: ticket + annotations: + summary: "Collector теряет telemetry (backend недоступен)" + runbook: "docs/runbooks.md#telemetryexportfailing" diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..0bae958 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,32 @@ +# Tempo 2.6.1 (monolithic) — хранилище ТРАСС. Приём OTLP gRPC от Collector. +# Блоки на volume tempodata (local backend), хранение 72 ч. +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 72h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..1bfb18f --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,66 @@ +-- 001_init: базовая схема мессенджера lab2. +-- Применяется src/migrate.py ровно один раз (таблица schema_migrations), +-- внутри одной транзакции; повторный запуск миграций ничего не дублирует. + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Список чатов пользователя: WHERE user_id = ? ORDER BY (created_at, id) +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); + +-- "В каких чатах состоит пользователь" (GET /chats) +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- История чата с keyset-пагинацией: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT $4 +-- Обоснование и EXPLAIN (ANALYZE, BUFFERS): evidence/explain_history.txt +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +-- Серверные сессии. Источник истины для сессий — PostgreSQL; +-- Redis хранит только кэш проверок токена (см. REPORT.md). +-- Хранится SHA-256 от токена, сам токен в БД не попадает. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_user_id_idx ON sessions (user_id); +CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); diff --git a/migrations/002_session_cache_invalidations.sql b/migrations/002_session_cache_invalidations.sql new file mode 100644 index 0000000..1c26116 --- /dev/null +++ b/migrations/002_session_cache_invalidations.sql @@ -0,0 +1,13 @@ +-- 002: надёжная инвалидация кэша сессий. +-- Строка вставляется В ТОЙ ЖЕ транзакции, что и sessions.revoked_at, и удаляется +-- только после того, как в Redis записано «надгробие» для этого токена. +-- Пока в таблице есть строки, API не доверяет положительным записям кэша +-- и проверяет каждый токен в PostgreSQL (см. src/sessions.py). +CREATE TABLE session_cache_invalidations ( + token_hash bytea PRIMARY KEY, + revoked_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT session_cache_invalidations_hash_len CHECK (octet_length(token_hash) = 32) +); + +CREATE INDEX session_cache_invalidations_revoked_at_idx + ON session_cache_invalidations (revoked_at); diff --git a/migrations/003_idempotency_keys.sql b/migrations/003_idempotency_keys.sql new file mode 100644 index 0000000..af60c76 --- /dev/null +++ b/migrations/003_idempotency_keys.sql @@ -0,0 +1,40 @@ +-- 003 (lab3): общая идемпотентность POST /chats/{chat_id}/messages. +-- +-- Область уникальности по контракту: пользователь + HTTP-метод + путь чата + ключ. +-- Путь однозначно задаётся chat_id, метод сейчас всегда POST (колонка оставлена, +-- чтобы область ключа была записана явно и расширялась без смены PK). +-- +-- Атомарность: строка ключа и сообщение пишутся ОДНОЙ транзакцией. +-- 1. INSERT ключа (ON CONFLICT ...) — конкурентный повтор с тем же ключом +-- ждёт на уникальном индексе, пока первая транзакция не завершится; +-- 2. INSERT сообщения с заранее выбранным id; +-- 3. COMMIT. FK message_id -> messages проверяется в момент COMMIT +-- (DEFERRABLE INITIALLY DEFERRED), поэтому message_id NOT NULL с первого шага. +-- Ошибка/откат на любом шаге не оставляет ни сообщения, ни ключа: +-- транзиентный 5xx не фиксируется как окончательный результат. +-- +-- Совместимость с работающей версией (rolling update): миграция только +-- добавляет новую таблицу, старые таблицы и запросы не меняются. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + method text NOT NULL DEFAULT 'POST', + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL + REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idem_key), + CONSTRAINT idempotency_keys_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_method CHECK (method IN ('POST')), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_expiry_after_create CHECK (expires_at > created_at) +); + +-- Периодическая очистка истёкших ключей (src/idempotency.py). +CREATE INDEX idempotency_keys_expires_at_idx ON idempotency_keys (expires_at); +-- ON DELETE CASCADE от messages без полного скана. +CREATE INDEX idempotency_keys_message_id_idx ON idempotency_keys (message_id); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..20561c2 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,68 @@ +-- 004 (lab4): вложения. Метаданные — в PostgreSQL, байты — в приватном S3-бакете. +-- +-- Жизненный цикл строки attachments (внутреннее поле state, в API не видно): +-- pending — намерение записано ДО S3 PUT (одна короткая транзакция); +-- ready — объект записан в S3, затем UPDATE ... WHERE state='pending'. +-- Наружу (GET, ссылки из сообщений) доступны только ready. Строки pending, +-- оставшиеся после сбоя между S3 и COMMIT, удаляет сборщик (src/reconcile.py) +-- после grace period; сначала удаляется строка (атомарный «захват»), потом объект. +-- Миграция только расширяющая (expand): реплики lab3 продолжают работать. + +-- Сообщение с вложениями может иметь пустой текст (text = ""). +-- Запрет «пустой текст без вложений» проверяет приложение. +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); +-- Цель составного внешнего ключа message_attachments (message_id, chat_id). +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE RESTRICT, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + -- Только отображаемое имя. Никогда не используется как путь или ключ S3. + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + -- Ключ объекта назначает сервер (случайный UUID), он не равен id вложения + -- и не раскрывается в API. + object_key text NOT NULL, + state text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_state CHECK (state IN ('pending', 'ready')) +); + +-- Сборщик незавершённых загрузок: WHERE state = 'pending' AND created_at < ... +CREATE INDEX attachments_pending_created_idx ON attachments (created_at) WHERE state = 'pending'; +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +-- Связь сообщение -> вложения (порядок attachment_ids сохраняется в position). +-- Составные внешние ключи гарантируют НА УРОВНЕ БД, что вложение и сообщение +-- принадлежат одному чату. ON DELETE RESTRICT: строку вложения, на которую +-- ссылается сообщение, нельзя удалить (сборщик удаляет только pending, а +-- pending прикрепить нельзя — двойная защита). +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/005_async_processing.sql b/migrations/005_async_processing.sql new file mode 100644 index 0000000..49d3df0 --- /dev/null +++ b/migrations/005_async_processing.sql @@ -0,0 +1,116 @@ +-- 005 (lab5): асинхронная обработка вложений через брокер. +-- +-- Жизненный цикл attachments.state (pending — внутренний, в API не виден): +-- pending — строка-намерение записана ДО S3 PUT в staging (как в lab4); +-- queued — оригинал целиком лежит в staging/, в ОДНОЙ транзакции с этим +-- UPDATE записаны attachment_jobs и строка job_outbox -> 202; +-- processing — worker взял аренду (lease) задания; +-- ready — worker записал оригинал в final/ и все варианты, затем одной +-- транзакцией опубликовал результат (variants + state); +-- failed — постоянная ошибка или исчерпаны попытки (error_code). +-- Миграция расширяющая: реплики lab4 продолжают работать (pending/ready). + +ALTER TABLE attachments DROP CONSTRAINT attachments_state; +ALTER TABLE attachments ADD CONSTRAINT attachments_state + CHECK (state IN ('pending', 'queued', 'processing', 'ready', 'failed')); + +-- Ключ staging-объекта (приватный префикс staging/ того же бакета). NULL после +-- безопасной очистки. object_key — конечный ключ оригинала (final//original), +-- назначается сервером при приёме и становится читаемым только в ready. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key); +ALTER TABLE attachments ADD COLUMN error_code text; +-- Версия pipeline, результат которой опубликован (или ожидается) для вложения. +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code + CHECK ((state = 'failed') = (error_code IS NOT NULL) OR state = 'pending'); + +-- Результат обработки. Ключ эффекта — (attachment_id, pipeline_version, name): +-- повторная доставка не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_key UNIQUE (object_key), + CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp', 'image/png')), + CONSTRAINT attachment_variants_size CHECK (size_bytes > 0 AND width > 0 AND height > 0), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +-- Одно логическое задание на (вложение, версию pipeline). job_id не меняется +-- при redelivery и retry; attempt растёт только при управляемой повторной попытке. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + status text NOT NULL DEFAULT 'queued', + attempt integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL, + replay_count integer NOT NULL DEFAULT 0, + -- Аренда: токен-«забор» (fencing), владелец (consumer) и срок. + lease_token uuid, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + last_error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + started_at timestamptz, + finished_at timestamptz, + CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'succeeded', 'failed')), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND max_attempts >= 1), + CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') = (lease_token IS NOT NULL)) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_status_idx ON attachment_jobs (status, updated_at); + +-- Transactional outbox: строка пишется в той же транзакции, что и задание. +-- Relay публикует в брокер (XADD с fsync), затем удаляет строку. Сбой между +-- XADD и DELETE даёт повторную публикацию — worker идемпотентен. +-- available_at: для свежей загрузки = now()+grace (сначала быстрый путь из API), +-- для retry = время следующей попытки (backoff с jitter). +CREATE TABLE job_outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX job_outbox_available_idx ON job_outbox (available_at, id); + +-- Реестр окончательно упавших заданий (DLQ). Replay — только явный, операторский. +CREATE TABLE dead_letters ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL, + error_code text NOT NULL, + attempts integer NOT NULL, + failed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + replayed_at timestamptz, + replay_count integer NOT NULL DEFAULT 0 +); + +-- Аудит переходов задания: claim, retry, fail, replay, no-op повторы, публикации. +-- Без секретов и без содержимого файлов. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid, + attempt integer, + event text NOT NULL, + detail text, + actor text, + at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); +CREATE INDEX job_events_at_idx ON job_events (at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..188c6e3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic==2.10.4 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +# lab4: S3-клиент и независимый парсер изображений +boto3==1.35.99 +Pillow==11.1.0 +# lab6: OpenTelemetry SDK + OTLP/HTTP exporter + автоинструментация (версии закреплены вместе) +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-http==1.29.0 +opentelemetry-instrumentation-fastapi==0.50b0 +opentelemetry-instrumentation-psycopg==0.50b0 +opentelemetry-instrumentation-botocore==0.50b0 diff --git a/scripts/alerts_watch.py b/scripts/alerts_watch.py new file mode 100644 index 0000000..f7dbe45 --- /dev/null +++ b/scripts/alerts_watch.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""lab6: журнал переходов alert'ов Prometheus (inactive -> pending -> firing -> resolved). + + python scripts/alerts_watch.py --duration 600 > evidence/alerts_timeline.txt + +Опрашивает http://127.0.0.1:9090/api/v1/alerts и печатает строку при каждом +изменении состояния (UTC-время, имя, состояние, значение). «resolved» = alert +исчез из списка активных. Только стандартная библиотека. +""" +import argparse +import json +import time +from datetime import datetime, timezone +from urllib.request import urlopen + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--prometheus", default="http://127.0.0.1:9090") + p.add_argument("--duration", type=float, default=600) + p.add_argument("--interval", type=float, default=5) + args = p.parse_args() + seen = {} + end = time.monotonic() + args.duration + print("%s watch start %s" % (now(), args.prometheus), flush=True) + while time.monotonic() < end: + try: + data = json.load(urlopen(args.prometheus + "/api/v1/alerts", timeout=5))["data"]["alerts"] + except Exception as exc: + print("%s prometheus unavailable: %s" % (now(), type(exc).__name__), flush=True) + time.sleep(args.interval) + continue + current = {a["labels"]["alertname"]: a for a in data} + for name, a in current.items(): + if seen.get(name) != a["state"]: + print("%s %-22s %-8s value=%s active_since=%s" % (now(), name, a["state"], a.get("value"), + a.get("activeAt")), flush=True) + seen[name] = a["state"] + for name in [n for n in seen if n not in current]: + print("%s %-22s resolved" % (now(), name), flush=True) + del seen[name] + time.sleep(args.interval) + + +if __name__ == "__main__": + main() diff --git a/scripts/collect_evidence.py b/scripts/collect_evidence.py new file mode 100644 index 0000000..bbd2ec2 --- /dev/null +++ b/scripts/collect_evidence.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Прогон всех проверок lab5 (и унаследованных lab2-4) на запущенном compose-стенде с записью в evidence/. + + python scripts/collect_evidence.py # ~18-22 минуты + python scripts/collect_evidence.py --quick # ~10-12 минут: без kill-серии, rolling update и нагрузки + +Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`, +`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики, +перезапускает хранилища и S3 (volumes сохраняются), останавливает S3 и делает +rolling update lab5 -> lab5.1 -> lab5 (откат), останавливает worker/broker, поэтому запускайте его только на учебном стенде. +Токены и пароли в файлы не пишутся (тесты держат их в памяти). +""" +import argparse +import json +import os +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.request import Request, urlopen + +ROOT = Path(__file__).resolve().parents[1] +EV = ROOT / "evidence" +PY = sys.executable or "python" +WIN = os.name == "nt" +HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh")) +HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh")) +BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080") +RESULTS = [] + +try: + sys.stdout.reconfigure(errors="replace") +except AttributeError: + pass + + +def stamp(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def step(title, cmd, out_file, append=False, env=None, timeout=900): + """Выполнить команду, сохранить «$ команда + вывод» в evidence/.""" + print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True) + started = time.monotonic() + try: + proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + timeout=timeout, env=dict(os.environ, **(env or {})), + encoding="utf-8", errors="replace") + output, rc = proc.stdout, proc.returncode + except subprocess.TimeoutExpired as exc: + output, rc = (exc.stdout or "") + "\nTIMEOUT", 124 + elapsed = time.monotonic() - started + shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "") + text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip()) + path = EV / out_file + with path.open("a" if append else "w", encoding="utf-8") as f: + f.write(text + ("\n" if append else "")) + print(output.rstrip()[-1500:], flush=True) + RESULTS.append((title, rc, out_file)) + return rc, output + + +def instances_through_lb(n=20): + """Одна сессия, n запросов через LB -> какие реплики отвечали.""" + import base64 + import uuid + name = "inst_" + uuid.uuid4().hex[:12] + pw = "Inst-" + uuid.uuid4().hex + body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode() + urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10) + cred = base64.b64encode((name + ":" + pw).encode()).decode() + sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST", + headers={"Authorization": "Basic " + cred}), timeout=10).read()) + lines = [] + for i in range(n): + with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r: + lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % ( + i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id"))) + return lines + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--quick", action="store_true", help="без SIGKILL-серии, rolling update и нагрузки") + args = ap.parse_args() + EV.mkdir(exist_ok=True) + (EV / "ops.log").write_text("", encoding="utf-8") + dc = ["docker", "compose", "--env-file", ".env"] + + step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt") + step("Compose config", dc + ["config", "--quiet"], "check.txt", append=True) + step("Стенд: контейнеры", dc + ["ps", "-a"], "stand.txt") + step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True) + step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True) + step("Стенд: опубликованные порты (только lb)", dc + ["ps", "--format", "table {{.Service}}\t{{.Ports}}"], + "stand.txt", append=True) + step("Образ SeaweedFS (закреплённая версия)", dc + ["images", "s3"], "stand.txt", append=True) + step("lab5: broker (Redis AOF) и worker", dc + ["ps", "broker", "worker"], "stand.txt", append=True) + step("lab5: конфигурация durability брокера", dc + ["exec", "-T", "broker", "sh", "-c", + "REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET appendfsync; " + "REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET maxmemory-policy"], + "stand.txt", append=True) + + # Bucket bootstrap: первый запуск был при up; повтор — no-op. + step("S3 bootstrap: лог первого запуска s3-init", dc + ["logs", "--no-color", "s3-init"], "s3_bootstrap.txt") + step("S3 bootstrap: повторный запуск (идемпотентность)", dc + ["run", "--rm", "-T", "s3-init"], + "s3_bootstrap.txt", append=True) + + lines = instances_through_lb() + (EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n" + % (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})), + encoding="utf-8") + print("\n".join(lines)) + + step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt") + step("Собственные тесты (HTTP через LB, включая test_attachments.py)", + [PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt") + step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"], + "student-tests.txt", append=True) + step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"], + "student-tests.txt", append=True) + step("Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*)", + [PY, "scripts/ops.py", "in-network", "consistency"], "consistency.txt") + step("Сверка БД <-> S3 (dry-run)", [PY, "scripts/ops.py", "reconcile"], "reconcile.txt") + step("Приватный бакет вне API, права ключей, закрытые админ-порты", [PY, "scripts/ops.py", "s3-check"], + "s3_private.txt") + step("На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes", + [PY, "scripts/ops.py", "api-fs"], "s3_private.txt", append=True) + step("API: файлы в /tmp (tmpfs) после загрузок", + dc + ["exec", "-T", "api", "sh", "-c", "df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l)"], + "s3_private.txt", append=True) + + step("Пик памяти: 10 одновременных загрузок по 9.5 MiB", + [PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "10", "--rounds", "3", + "--docker", "--label", "c=10, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c10.json"], + "upload_memory.txt") + step("Пик памяти: 20 одновременных загрузок", + [PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "20", "--rounds", "2", + "--docker", "--label", "c=20, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c20.json"], + "upload_memory.txt", append=True) + + # Публичный тест останавливает реплику, обслужившую его первое сообщение. + # Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь + # лишними запросами, пока цель не окажется другой (README: «повторить для второй»). + targets = [] + for i in range(1, 5): + if len(set(targets)) >= 2: + break + for _ in range(i - 1): + urlopen(BASE + "/health/live", timeout=5).read() + ops_log = EV / "ops.log" + stops_before = sum(1 for l in ops_log.read_text(encoding="utf-8").splitlines() + if "stop-one: mode=" in l) if ops_log.exists() else 0 + step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1) + step("Failover #%d: make test-failover (вкл. скачивание файла через другую реплику)" % i, + [PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE], + "failover.txt", append=True) + step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"], + "failover.txt", append=True) + step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True) + stops = [l for l in ops_log.read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l] \ + if ops_log.exists() else [] + # Только остановка из ЭТОЙ попытки: если тест упал до hook'а, новой строки нет, + # и цель прошлой попытки не должна засчитываться повторно. + if len(stops) > stops_before: + targets.append(stops[-1].split("target=")[1].split()[0]) + with (EV / "failover.txt").open("a", encoding="utf-8") as f: + f.write("# остановленные публичным тестом реплики: %s\n" % targets) + + step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE, + "--output", "evidence/failover_series.json"], "failover_series.txt") + if not args.quick: + step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE, + "--mode", "kill", "--output", "evidence/failover_series_kill.json"], + "failover_series.txt", append=True) + + step("Недоступность S3 и восстановление", [PY, "student-tests/s3_outage.py", "--base-url", BASE, + "--output", "evidence/s3_outage.json"], "s3_outage.txt") + + # lab5: очередь и worker + step("Варианты изображений по фактическим байтам (test_pipeline.py)", + [PY, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_pipeline.py", "-v"], + "pipeline.txt", env={"BASE_URL": BASE}) + step("Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready", + [PY, "student-tests/worker_outage.py", "--base-url", BASE, "--output", "evidence/worker_outage.json"], + "worker_outage.txt") + step("Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)", + [PY, "scripts/ops.py", "pipeline-faults"], "pipeline_faults.txt", timeout=1200) + step("Очередь после сценариев: задания, outbox, DLQ, lag", [PY, "scripts/ops.py", "jobs", "stats"], + "pipeline_faults.txt", append=True) + step("Реестр failed (DLQ)", [PY, "scripts/ops.py", "jobs", "failed", "--limit", "10"], "pipeline_faults.txt", + append=True) + step("Worker: последние строки лога", dc + ["logs", "--no-color", "--tail", "80", "worker"], "worker_log_sample.txt") + + if not args.quick: + step("Rolling update lab5 -> lab5.1 под нагрузкой", + [PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"], + "rolling_update.txt", env={"APP_VERSION": "lab5.1"}) + step("Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой", + [PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"], + "rolling_update.txt", append=True, env={"APP_VERSION": "lab5"}) + step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True) + + fmt = ["ps", "-a", "--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"] + step("Persistence: контейнеры до", dc + fmt, "persistence.txt") + step("Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB)", + [PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE], + "persistence.txt", append=True) + step("Persistence: контейнеры после (Status = Up N seconds)", dc + fmt, "persistence.txt", append=True) + step("Persistence: сверка БД <-> S3 после рестарта", [PY, "scripts/ops.py", "reconcile"], "persistence.txt", + append=True) + + if not args.quick: + step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30", + "--concurrency", "8", "--output", "evidence/load.json"], "load.txt") + step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True) + + step("Лог LB (последние строки, без секретов)", dc + ["logs", "--no-color", "--tail", "60", "lb"], "lb_log_sample.txt") + + summary = ["# %s" % stamp()] + ["%-4s %-70s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS] + (EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8") + print("\n" + "\n".join(summary)) + sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/incident.py b/scripts/incident.py new file mode 100644 index 0000000..238b963 --- /dev/null +++ b/scripts/incident.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости. + +Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose. +Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей. + + python scripts/incident.py probe [--filename photo.png] + Один upload: печатает X-Request-Id, attachment_id и время до ready. + Затем в Grafana -> Explore -> Tempo (TraceQL): + { span.messenger.request_id = "" } + и в Loki: {service_name=~"messenger-.+"} | request_id="" + + python scripts/incident.py worker-stop [--duration 150] + Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и + возраст старейшего задания -> QueueLagGrowing firing; worker запущен -> + очередь разбирается -> resolved. + + python scripts/incident.py db-lock [--duration 180] + Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на + таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и + получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой + QueryCanceled (а не долгий handler). Остальные маршруты работают. + + python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector] + Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются + указанные сервисы; проверяется, что API отвечает, и RSS API не растёт. +""" +import argparse +import json +import shlex +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT / "student-tests")) +sys.path.insert(0, str(ROOT / "tests")) +import common # noqa: E402 +from imagegen import sample_png # noqa: E402 +from test_attachments import upload # noqa: E402 + +PY = sys.executable +T0 = time.monotonic() + + +def note(step, **kw): + kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw) + print(json.dumps(kw, ensure_ascii=False), flush=True) + + +def sh(cmd, check=True): + note("exec", cmd=cmd) + return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None) + + +def setup_chat(): + owner = common.actor("obs") + chat = common.create_chat(owner) + return owner, chat + + +def one_upload(owner, chat, filename="photo.png", data=None): + rid = "probe-" + uuid.uuid4().hex[:12] + r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename, + headers={"X-Request-Id": rid}) + return r, rid + + +def cmd_probe(args): + owner, chat = setup_chat() + started = time.monotonic() + r, rid = one_upload(owner, chat, args.filename) + note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"), + instance=r.headers.get("X-Instance-Id")) + if r.status != 202: + return 1 + ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True) + note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2)) + print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid) + print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid) + return 0 + + +def cmd_worker_stop(args): + owner, chat = setup_chat() + sh(args.stop_worker) + ids, end = [], time.monotonic() + args.duration + while time.monotonic() < end: + r, rid = one_upload(owner, chat) + ids.append(r.json["id"] if r.status == 202 else None) + note("upload_while_worker_down", status=r.status, request_id=rid) + time.sleep(args.interval) + sh(args.start_worker) + started = time.monotonic() + pending = [i for i in ids if i] + while pending and time.monotonic() - started < 300: + pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i, + token=owner["_token"]).json.get("status") != "ready"] + time.sleep(2) + note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1)) + return 0 if not pending else 1 + + +class Load: + """Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков).""" + + def __init__(self, owner, chat, threads): + self.owner, self.chat, self.stop = owner, chat, threading.Event() + self.counts, self.lock = {}, threading.Lock() + self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)] + + def run(self): + path = "/api/v1/chats/%s/messages" % self.chat["id"] + while not self.stop.is_set(): + for method, payload in (("POST", {"text": "load"}), ("GET", None)): + try: + st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status + except Exception: + st = "conn_error" + key = "%s %s" % (method, st) + with self.lock: + self.counts[key] = self.counts.get(key, 0) + 1 + time.sleep(0.05) + + def snapshot(self): + with self.lock: + snap, self.counts = self.counts, {} + return snap + + def __enter__(self): + for t in self.threads: + t.start() + return self + + def __exit__(self, *exc): + self.stop.set() + + +def cmd_db_lock(args): + owner, chat = setup_chat() + lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration + cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-v", "ON_ERROR_STOP=1", "-c", lock_sql] + with Load(owner, chat, args.threads) as load: + time.sleep(args.warmup) + note("baseline", **load.snapshot()) + note("db_lock_start", duration_s=args.duration) + proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL) + while proc.poll() is None: + time.sleep(10) + note("during_lock", **load.snapshot()) + note("db_lock_end", rc=proc.returncode) + time.sleep(args.warmup) + note("after", **load.snapshot()) + return 0 + + +def rss_of(service): + out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service], + capture_output=True, text=True).stdout.split() + if not out: + return {} + stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out, + capture_output=True, text=True).stdout.strip().splitlines() + return dict(line.split(" ", 1) for line in stats) + + +def args_project(): + import os + return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6") + + +def cmd_telemetry_down(args): + owner, chat = setup_chat() + services = args.services.split(",") + with Load(owner, chat, args.threads) as load: + time.sleep(args.warmup) + note("baseline", **load.snapshot(), memory=rss_of("api")) + subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL) + note("telemetry_stopped", services=services) + end = time.monotonic() + args.duration + while time.monotonic() < end: + time.sleep(15) + note("telemetry_down", **load.snapshot(), memory=rss_of("api")) + subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL) + note("telemetry_started", services=services) + time.sleep(args.warmup) + note("after", **load.snapshot(), memory=rss_of("api")) + return 0 + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://127.0.0.1:8080") + sub = p.add_subparsers(dest="cmd", required=True) + pr = sub.add_parser("probe") + pr.add_argument("--filename", default="photo.png") + pr.add_argument("--timeout", type=float, default=60) + ws = sub.add_parser("worker-stop") + ws.add_argument("--duration", type=float, default=150) + ws.add_argument("--interval", type=float, default=5) + ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY)) + ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY)) + for name in ("db-lock", "telemetry-down"): + sp = sub.add_parser(name) + sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90) + sp.add_argument("--threads", type=int, default=4) + sp.add_argument("--warmup", type=float, default=20) + sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env")) + if name == "telemetry-down": + sp.add_argument("--services", default="otel-collector") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + import test_attachments + test_attachments.BASE_URL = common.BASE_URL + fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock, + "telemetry-down": cmd_telemetry_down}[args.cmd] + sys.exit(fn(args)) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..2e7cbbd --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Создаёт локальные учебные секреты в ./secrets (каталог в .gitignore). + +lab5: broker_password и broker_url (Redis Streams с AOF). +Повторный запуск ничего не перезаписывает: пароль PostgreSQL задаётся только +при первой инициализации volume, и смена файла сломала бы подключение. +Значения секретов не печатаются. + +lab4: две пары S3-ключей — bootstrap-admin (только для one-shot s3-init) и +messenger-api (Read/Write/List одного бакета). s3_config.json — identities +SeaweedFS, собирается из этих ключей (пересоздаётся, если удалён: ключи S3 не +привязаны к данным в volume, их можно сменить: удалить 5 файлов s3_* и +перезапустить init_secrets.py + docker compose up -d --force-recreate s3 s3-init api). +""" +import json +import os +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIR = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + path = DIR / name + if path.exists(): + print("keep secrets/%s" % name) + return + # newline="\n": на Windows иначе пишется CRLF, и `$(cat ...)` в контейнере Redis + # получает пароль с лишним \r -> AUTH клиента не совпадает (lab5: брокер недоступен). + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(value + "\n") + # Каталог 0700, файлы 0644: внутри контейнера их читают непривилегированные + # пользователи (appuser, redis), а на хосте каталог закрыт. + os.chmod(path, 0o644) + print("create secrets/%s" % name) + + +def read(name: str) -> str: + return (DIR / name).read_text(encoding="utf-8").strip() + + +def main() -> None: + DIR.mkdir(exist_ok=True) + try: + os.chmod(DIR, 0o700) + except OSError: + pass + write("postgres_password", secrets.token_hex(24)) + write("redis_password", secrets.token_hex(24)) + write("database_url", "postgresql://messenger:%s@postgres:5432/messenger" % read("postgres_password")) + write("cache_url", "redis://:%s@redis:6379/0" % read("redis_password")) + # lab5: брокер (отдельный Redis с AOF). URL содержит пароль — только в secrets/. + write("broker_password", secrets.token_hex(24)) + write("broker_url", "redis://:%s@broker:6379/0" % read("broker_password")) + # lab6: пароль администратора Grafana (логин admin). Посмотреть: cat secrets/grafana_admin_password + write("grafana_admin_password", secrets.token_urlsafe(18)) + # lab4: S3 (SeaweedFS). Access key — ASCII 20 символов, secret — 40 (как у AWS). + write("s3_admin_access_key_id", "ADM" + secrets.token_hex(8).upper() + "X") + write("s3_admin_secret_access_key", secrets.token_urlsafe(30)[:40]) + write("s3_access_key_id", "API" + secrets.token_hex(8).upper() + "X") + write("s3_secret_access_key", secrets.token_urlsafe(30)[:40]) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + config = { + "identities": [ + { + "name": "bootstrap_admin", + "credentials": [{"accessKey": read("s3_admin_access_key_id"), + "secretKey": read("s3_admin_secret_access_key")}], + "actions": ["Admin", "Read", "List", "Tagging", "Write"], + }, + { + # Только свой бакет: чтение, запись/удаление объектов, листинг. + # Создать/удалить бакет или прочитать чужой этими ключами нельзя. + "name": "messenger_api", + "credentials": [{"accessKey": read("s3_access_key_id"), + "secretKey": read("s3_secret_access_key")}], + "actions": ["Read:" + bucket, "Write:" + bucket, "List:" + bucket], + }, + ] + # Identity "anonymous" не объявлена -> анонимные запросы получают 403. + } + write("s3_config.json", json.dumps(config, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 0000000..14341ed --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Операции над compose-стендом lab5 (база — lab4). Python 3.9+, только стандартная библиотека. + +Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает +`docker compose` и `docker` без shell-интерполяции. Секреты не печатает. + + python scripts/ops.py wait # дождаться N здоровых реплик и LB + python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB + python scripts/ops.py stop-one # TARGET_INSTANCE_ID=; оставляет остановленной + python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB + python scripts/ops.py restart # hook persistence: API + worker + PostgreSQL + Redis + broker + S3 + LB, volumes целы + python scripts/ops.py wait-s3 # дождаться, пока S3 реально читает/пишет (прогрев после рестарта) + python scripts/ops.py s3-check # приватность бакета вне API, права ключей, закрытые админ-порты + python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (по умолчанию dry-run) + python scripts/ops.py api-fs # у API read-only rootfs, только tmpfs, без volumes + python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса + python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети + python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с) + python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы + + lab5: + python scripts/ops.py worker-stop # остановить ВСЕ контейнеры worker (docker stop, graceful) + python scripts/ops.py worker-kill # SIGKILL worker (без перезапуска политикой restart) + python scripts/ops.py worker-start # запустить worker и дождаться healthy + python scripts/ops.py api-restart # перезапуск реплик API (volumes не трогаются) + python scripts/ops.py broker-restart # перезапуск broker (AOF на volume brokerdata) + python scripts/ops.py jobs stats|failed|show ID|replay ID... # src/jobs_admin.py в one-off контейнере + python scripts/ops.py pipeline-faults # внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) + +Переменные окружения: + COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab6, как `name:` в compose.yaml) + ENV_FILE файл переменных (по умолчанию .env, если существует) + API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2) + APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404) + STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение») + +Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) — +это доказательство того, какой контейнер реально остановлен/заменён. +""" +import csv +import io +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OPS_LOG = ROOT / "evidence" / "ops.log" + +try: # консоль Windows может быть не в UTF-8 + sys.stdout.reconfigure(errors="replace") + sys.stderr.reconfigure(errors="replace") +except AttributeError: + pass + + +# ---------------------------------------------------------------- settings + +def _dotenv(): + path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env"))) + values = {} + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + values[k.strip()] = v.strip().strip('"').strip("'") + return path if path.is_file() else None, values + + +ENV_PATH, DOTENV = _dotenv() + + +def setting(name, default): + return os.environ.get(name) or DOTENV.get(name) or default + + +PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab6") +REPLICAS = int(setting("API_REPLICAS", "2")) +APP_PORT = int(setting("APP_PORT", "8080")) +STATS_PORT = int(setting("LB_STATS_PORT", "8404")) +LB_URL = "http://127.0.0.1:%d" % APP_PORT +STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT + + +def now_iso(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def log(msg): + line = "%s %s" % (now_iso(), msg) + print(line, flush=True) + try: + OPS_LOG.parent.mkdir(exist_ok=True) + with OPS_LOG.open("a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + pass + + +# ---------------------------------------------------------------- docker + +def compose_cmd(*args): + cmd = ["docker", "compose", "-p", PROJECT] + if ENV_PATH: + cmd += ["--env-file", str(ENV_PATH)] + return cmd + list(args) + + +def run(cmd, check=True, capture=False, timeout=300, show=False): + """capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо.""" + out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL) + result = subprocess.run( + cmd, cwd=ROOT, check=False, timeout=timeout, + stdout=out, stderr=out, text=True, + ) + if check and result.returncode != 0: + detail = (result.stderr or "").strip()[-400:] if capture else "" + raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail)) + return result + + +def compose(*args, timeout=300): + return run(compose_cmd(*args), timeout=timeout, capture=True) + + +def api_containers(include_stopped=True): + """[{id, short, name, state, health, image, ip}] — только сервис api, без one-off.""" + cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}", + "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=api", + "--filter", "label=com.docker.compose.oneoff=False"] + if include_stopped: + cmd.insert(2, "-a") + ids = [x for x in run(cmd, capture=True).stdout.split() if x] + if not ids: + return [] + raw = run(["docker", "inspect"] + ids, capture=True).stdout + result = [] + for item in json.loads(raw): + state = item["State"] + nets = item.get("NetworkSettings", {}).get("Networks") or {} + ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")] + result.append({ + "id": item["Id"], + "short": item["Id"][:12], + "hostname": item["Config"].get("Hostname", ""), + "name": item["Name"].lstrip("/"), + "state": state.get("Status"), + "health": (state.get("Health") or {}).get("Status", "-"), + "image": item["Config"].get("Image"), + "ip": ips[0] if ips else "", + "created": item.get("Created", "")[:19], + }) + result.sort(key=lambda c: c["name"]) + return result + + +def running(containers=None): + return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"] + + +# ---------------------------------------------------------------- load balancer + +def lb_servers(): + """[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом).""" + data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace") + rows = csv.DictReader(io.StringIO(data.lstrip("# "))) + servers = [] + for r in rows: + if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"): + continue + addr = r.get("addr") or "" + status = r.get("status", "") + if status.startswith("MAINT") and not addr: + continue # пустой слот server-template + servers.append({"name": r["svname"], "status": status, "addr": addr}) + return servers + + +def lb_up_count(): + return sum(1 for s in lb_servers() if s["status"].startswith("UP")) + + +def lb_ready(): + try: + with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r: + return r.status == 200 + except (urllib.error.URLError, OSError): + return False + + +def wait_until(predicate, timeout, what, interval=0.3): + deadline = time.monotonic() + timeout + last_exc = None + while time.monotonic() < deadline: + try: + if predicate(): + return True + except (urllib.error.URLError, OSError, ValueError, KeyError) as exc: + last_exc = exc + time.sleep(interval) + raise SystemExit("timeout after %ss waiting for %s%s" % ( + timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else "")) + + +def wait_healthy(ids, timeout=90): + def ok(): + states = {c["id"]: c for c in api_containers()} + return all(states.get(i, {}).get("health") == "healthy" for i in ids) + wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids)) + + +def wait_stack(replicas, timeout=120): + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas, + timeout, "%d healthy api replicas" % replicas) + wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas) + wait_until(lb_ready, 30, "LB /health/ready = 200") + + +# ---------------------------------------------------------------- commands + +def cmd_wait(_args): + t0 = time.monotonic() + wait_stack(REPLICAS) + print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0)) + + +def cmd_ps(_args): + containers = api_containers() + try: + by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()} + except (urllib.error.URLError, OSError): + by_addr = {} + print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status")) + for c in containers: + s = by_addr.get(c["ip"], {}) + print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ( + c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-"))) + + +def find_target(target): + for c in api_containers(): + # X-Instance-Id = hostname контейнера = первые 12 символов его ID. + if target and (c["hostname"] == target or c["id"].startswith(target)): + return c + return None + + +def cmd_stop_one(_args): + target = os.environ.get("TARGET_INSTANCE_ID", "").strip() + if not target: + raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)") + mode = os.environ.get("STOP_MODE", "stop").strip().lower() + c = find_target(target) + if c is None: + raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target)) + if c["state"] != "running": + raise SystemExit("container %s is already %s" % (c["name"], c["state"])) + t0 = time.monotonic() + log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"])) + if mode == "kill": + # «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял + # контейнер сам: тест требует, чтобы реплика оставалась остановленной. + run(["docker", "update", "--restart", "no", c["id"]]) + run(["docker", "kill", "--signal", "KILL", c["id"]]) + else: + # SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с. + run(["docker", "stop", "--time", "20", c["id"]], timeout=40) + after = find_target(target) or {} + log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)" + % (c["name"], after.get("state", "removed"), time.monotonic() - t0)) + + +def cmd_restore(_args): + t0 = time.monotonic() + stopped = [c for c in api_containers() if c["state"] != "running"] + for c in stopped: # вернуть политику, снятую в режиме kill + run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False) + # --no-recreate: не пересоздавать работающие; up запускает остановленные + # и добирает число контейнеров до deploy.replicas. + compose("up", "-d", "--no-deps", "--no-recreate", "api") + wait_stack(REPLICAS) + log("restore: started %s; %d replicas healthy and UP in LB after %.1fs" + % ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0)) + + +def cmd_restart(_args): + """Hook для make test-persistence: перезапуск API, worker, ВСЕХ хранилищ + (PostgreSQL, Redis, broker, S3) и LB. + + Volumes не удаляются, данные не пересоздаются, bucket/поток не создаются заново. + Порядок: сначала LB, API и worker (новые записи не принимаются, обработка + останавливается graceful), затем хранилища и брокер, затем ожидание реальной + готовности S3, затем API, worker и LB. + """ + t0 = time.monotonic() + before = {c["name"]: c["short"] for c in running()} + stores = ("postgres", "redis", "broker", "s3") + started_before = {svc: container_started(svc) for svc in stores} + compose("stop", "lb", "api", "worker", timeout=180) + compose("restart", *stores, timeout=120) + wait_service_healthy("s3", 60) + wait_service_healthy("broker", 60) + t_warm = time.monotonic() + warm = s3_warmup(70) + log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm)) + compose("start", "api", "worker", timeout=120) + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS, + 80, "api healthy after restart") + compose("start", "lb", timeout=60) + log("restart: started_at %s; api %s restarted; worker restarted; lb restarted; %.1fs" + % ("; ".join("%s %s -> %s" % (svc, started_before[svc], container_started(svc)) for svc in stores), + sorted(before), time.monotonic() - t0)) + + +def service_ids(service, include_stopped=False): + cmd = ["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service, + "--filter", "label=com.docker.compose.oneoff=False"] + if include_stopped: + cmd.insert(2, "-a") + return run(cmd, capture=True).stdout.split() + + +def cmd_worker_stop(_args): + t0 = time.monotonic() + ids = service_ids("worker") + # SIGTERM: worker дорабатывает текущие сообщения (stop_grace_period 75 с). + compose("stop", "worker", timeout=120) + log("worker-stop: stopped %s in %.1fs" % ([i[:12] for i in ids], time.monotonic() - t0)) + + +def cmd_worker_kill(_args): + ids = service_ids("worker") + for i in ids: + run(["docker", "update", "--restart", "no", i]) + run(["docker", "kill", "--signal", "KILL", i]) + log("worker-kill: SIGKILL %s (restart policy disabled until worker-start)" % [i[:12] for i in ids]) + + +def cmd_worker_start(_args): + t0 = time.monotonic() + for i in service_ids("worker", include_stopped=True): + run(["docker", "update", "--restart", "unless-stopped", i], check=False) + compose("up", "-d", "--no-deps", "--no-recreate", "worker", timeout=120) + wait_service_healthy("worker", 90) + log("worker-start: worker healthy after %.1fs" % (time.monotonic() - t0)) + + +def cmd_api_restart(_args): + t0 = time.monotonic() + before = [c["short"] for c in running()] + compose("restart", "api", timeout=120) + wait_stack(REPLICAS) + log("api-restart: %s restarted, %d replicas UP after %.1fs" % (before, REPLICAS, time.monotonic() - t0)) + + +def cmd_broker_restart(_args): + t0 = time.monotonic() + before = container_started("broker") + compose("restart", "broker", timeout=60) + wait_service_healthy("broker", 60) + log("broker-restart: started_at %s -> %s, healthy after %.1fs (volume brokerdata kept)" + % (before, container_started("broker"), time.monotonic() - t0)) + + +def cmd_jobs(args): + run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.jobs_admin") + list(args), + show=True, timeout=300) + + +def faults_compose(*args): + return run(["docker", "compose", "-p", PROJECT] + (["--env-file", str(ENV_PATH)] if ENV_PATH else []) + + ["-f", "compose.yaml", "-f", "compose.faults.yaml"] + list(args), capture=True, timeout=300) + + +def cmd_pipeline_faults(_args): + """Внедрённые отказы pipeline. Worker пересоздаётся с compose.faults.yaml + (маркеры в filename, короткая аренда), поднимаются api-fault-publish/api-fault-dup; + тесты идут внутри сети (нужен доступ к PostgreSQL для журнала job_events). + В конце worker возвращается к обычной конфигурации.""" + log("pipeline-faults: recreating worker with compose.faults.yaml") + faults_compose("up", "-d", "--no-deps", "--force-recreate", "worker") + wait_service_healthy("worker", 90) + prof = compose_cmd("--profile", "faults") + fault_api = ["api-fault-publish", "api-fault-dup"] + run(prof + ["up", "-d", "--no-recreate"] + fault_api, show=True, timeout=300) + mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests") + # Тест читает публичную фикстуру tests/fixtures/sample.png; каталога tests/ + # в образе нет (Dockerfile копирует только src и migrations) -> монтируем ro. + fixtures = "%s:/app/tests/fixtures:ro" % (ROOT / "tests" / "fixtures") + base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount, "-v", fixtures, + "-e", "BASE_URL=http://lb:8080", + "-e", "FAULT_PUBLISH_URL=http://api-fault-publish:8080", + "-e", "FAULT_DUP_URL=http://api-fault-dup:8080") + try: + for name in fault_api: + run(base + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name], + show=True, timeout=120) + run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_pipeline_faults.py", "-v"], show=True, timeout=900) + finally: + run(prof + ["rm", "-sf"] + fault_api, check=False, timeout=120) + compose("up", "-d", "--no-deps", "--force-recreate", "worker", timeout=180) + wait_service_healthy("worker", 90) + log("pipeline-faults: worker restored to normal configuration") + + +def wait_service_healthy(service, timeout): + def ok(): + ids = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() + if not ids: + return False + out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip() + return out == "healthy" + wait_until(ok, timeout, service + " healthy") + + +def s3_warmup(timeout=70): + """One-off контейнер образа API в сетях стенда: put/get/delete пробы + чтение старого объекта.""" + out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--wait-ready", str(timeout)), capture=True, timeout=timeout + 60) + lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")] + return lines[-1] if lines else "" + + +def cmd_wait_s3(_args): + wait_service_healthy("s3", 60) + print(s3_warmup(90)) + + +def cmd_s3_check(_args): + """Проверка приватного бакета ВНЕ API: анонимные запросы к S3 из сети storage.""" + run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--check-only", "--sample-from-db"), show=True, timeout=180) + + +def cmd_api_fs(_args): + """Доказательство «на API нет локального хранилища оригиналов».""" + for c in api_containers(): + info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0] + mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])] + print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % ( + c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none")) + + +def cmd_reconcile(args): + run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.reconcile") + list(args), + show=True, timeout=600) + + +def container_started(service): + ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() + if not ids: + return "-" + return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23] + + +def cmd_rolling_update(args): + """Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой. + + В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только + расширяющие (expand): старая версия продолжает работать на новой схеме. + """ + t_all = time.monotonic() + if [c for c in api_containers() if c["state"] != "running"]: + raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first") + old = running() + n = len(old) + if n < 1: + raise SystemExit("no running api replicas") + log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old])) + if "--no-build" not in args: + t = time.monotonic() + compose("build", "api", "migrate", timeout=900) + log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab5"))) + t = time.monotonic() + out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines() + log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else "")) + for i, victim in enumerate(old, 1): + before = {c["id"] for c in api_containers()} + t = time.monotonic() + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180) + new = [c for c in api_containers() if c["id"] not in before] + if len(new) != 1: + raise SystemExit("expected exactly one new container, got %d" % len(new)) + fresh = new[0] + wait_healthy([fresh["id"]], timeout=90) + wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy") + t_ready = time.monotonic() - t + t = time.monotonic() + run(["docker", "stop", "--time", "20", victim["id"]], timeout=40) + run(["docker", "rm", victim["id"]]) + log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)" + % (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t)) + wait_stack(n) + log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()])) + + +def cmd_in_network(args): + """Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl. + + Запускаются в one-off контейнере образа API внутри сети backend: порты БД и + реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`, + поэтому LB его не видит. + """ + which = args[0] if args else "" + mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests") + base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount) + if which == "storage": + for pattern in ("test_storage_*.py", "test_session_cache.py"): + run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"], + show=True, timeout=600) + elif which == "consistency": + prof = compose_cmd("--profile", "faults") + faults = ["api-fault-s3", "api-fault-db", "api-fault-crash"] + run(prof + ["up", "-d", "--no-recreate"] + faults, show=True, timeout=300) + env = ["-e", "BASE_URL=http://lb:8080", + "-e", "FAULT_S3_URL=http://api-fault-s3:8080", + "-e", "FAULT_DB_URL=http://api-fault-db:8080", + "-e", "FAULT_CRASH_URL=http://api-fault-crash:8080"] + try: + for name in faults: + run(base + env + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name], + show=True, timeout=120) + run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_attachment_consistency.py", "-v"], show=True, timeout=600) + finally: + run(prof + ["rm", "-sf"] + faults, check=False, timeout=120) + elif which == "expiry": + prof = compose_cmd("--profile", "ttl") + run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300) + env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"] + try: + run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"], + show=True, timeout=120) + run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_expiry.py", "-v"], show=True, timeout=300) + finally: + run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120) + else: + raise SystemExit("usage: python scripts/ops.py in-network storage|expiry|consistency") + + +COMMANDS = { + "wait": cmd_wait, + "ps": cmd_ps, + "stop-one": cmd_stop_one, + "restore": cmd_restore, + "restart": cmd_restart, + "rolling-update": cmd_rolling_update, + "in-network": cmd_in_network, + "wait-s3": cmd_wait_s3, + "s3-check": cmd_s3_check, + "reconcile": cmd_reconcile, + "api-fs": cmd_api_fs, + "worker-stop": cmd_worker_stop, + "worker-kill": cmd_worker_kill, + "worker-start": cmd_worker_start, + "api-restart": cmd_api_restart, + "broker-restart": cmd_broker_restart, + "jobs": cmd_jobs, + "pipeline-faults": cmd_pipeline_faults, +} + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: + print(__doc__) + sys.exit(2) + COMMANDS[sys.argv[1]](sys.argv[2:]) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..5429f02 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант restart.sh: API + PostgreSQL + Redis + S3 + LB, volumes сохраняются. +rem python tests\resilience.py --mode persistence --action-script scripts\restart.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py restart +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..921ed9c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Hook для make test-persistence: перезапускает API (все реплики), PostgreSQL, +# Redis, S3 (SeaweedFS) и LB, сохраняя volumes pgdata и s3data. Никаких down -v / +# flush / повторного seed / пересоздания бакета. После рестарта S3 ждёт, пока +# реально работают запись и чтение (прогрев SeaweedFS 15–30 с), затем поднимает API. +# Возвращает 0, когда API снова healthy; готовность LB тест ждёт сам. +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py restart diff --git a/scripts/sql/explain_history.sql b/scripts/sql/explain_history.sql new file mode 100644 index 0000000..d2ee413 --- /dev/null +++ b/scripts/sql/explain_history.sql @@ -0,0 +1,93 @@ +-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на >= 10 000 сообщений. +-- Всё выполняется в ОДНОЙ транзакции и откатывается (ROLLBACK): синтетические +-- данные и временное удаление индекса не остаются в БД. +-- Запуск (compose): +-- docker compose exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 \ +-- < scripts/sql/explain_history.sql > evidence/explain_history.txt +-- Внимание: DROP INDEX внутри транзакции берёт эксклюзивную блокировку messages +-- до ROLLBACK (несколько секунд) — запускать на учебном стенде. + +\set ON_ERROR_STOP 1 +\timing off +SET TimeZone = 'UTC'; +-- Убираем мёртвые версии строк от прошлых запусков (VACUUM нельзя внутри транзакции). +VACUUM (ANALYZE) messages, chat_members, chats, users; +BEGIN; + +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-4000-8000-00000000e001', 'explain_alice', 'Explain', 'x'), + ('00000000-0000-4000-8000-00000000e002', 'explain_bob', 'Explain', 'x'); + +-- 1 большой чат (12 000 сообщений) + 40 чатов по 1 000 = 52 000 сообщений. +INSERT INTO chats (id, title, created_at) +SELECT CASE WHEN g = 0 THEN '00000000-0000-4000-8000-0000000c0000'::uuid ELSE gen_random_uuid() END, + 'explain-' || g, timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM generate_series(0, 40) AS g; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT c.id, u.id, u.pos +FROM chats c +CROSS JOIN (VALUES ('00000000-0000-4000-8000-00000000e001'::uuid, 0), + ('00000000-0000-4000-8000-00000000e002'::uuid, 1)) AS u(id, pos) +WHERE c.title LIKE 'explain-%'; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), c.id, + CASE WHEN g % 2 = 0 THEN '00000000-0000-4000-8000-00000000e001'::uuid + ELSE '00000000-0000-4000-8000-00000000e002'::uuid END, + 'synthetic message ' || g, + timestamptz '2026-01-02 00:00:00+00' + g * interval '1 millisecond' + (c.created_at - timestamptz '2026-01-01 00:00:00+00') +FROM chats c +CROSS JOIN LATERAL generate_series(1, CASE WHEN c.title = 'explain-0' THEN 12000 ELSE 1000 END) AS g +WHERE c.title LIKE 'explain-%'; + +ANALYZE users, chats, chat_members, messages; + +\echo '=== dataset' +SELECT count(*) AS total_messages, + count(*) FILTER (WHERE chat_id = '00000000-0000-4000-8000-0000000c0000') AS big_chat_messages +FROM messages; + +-- Позиция курсора после 10 000-го сообщения большого чата (как в next_cursor). +SELECT created_at AS cur_ts, id AS cur_id +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== 1. first page (no cursor), LIMIT 51 = limit+1' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 2. deep page: keyset cursor after message #10000 (query used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 3. for comparison: OFFSET pagination to the same page (NOT used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +OFFSET 10000 LIMIT 51; + +\echo '=== 4. same keyset query WITHOUT messages_chat_created_id_idx (index dropped inside the transaction)' +DROP INDEX messages_chat_created_id_idx; +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +ROLLBACK; +\echo '=== rolled back: synthetic data removed, index restored' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..1bd3049 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант stop-one.sh: TARGET_INSTANCE_ID -> один контейнер api, остаётся остановленным. +rem python tests\resilience.py --mode failover --action-script scripts\stop-one.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py stop-one +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..a0b23e5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Hook для make test-failover: останавливает ОДНУ API-реплику, которую тест передал +# в TARGET_INSTANCE_ID (= X-Instance-Id = hostname контейнера = короткий ID), и +# оставляет её остановленной. По умолчанию graceful (docker stop: SIGTERM -> +# осушение 3 с -> graceful shutdown); STOP_MODE=kill — «падение» (SIGKILL). +# БД, Redis, S3, LB и volumes не трогает. Восстановление: python3 scripts/ops.py restore +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py stop-one diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..468a02b --- /dev/null +++ b/src/broker.py @@ -0,0 +1,221 @@ +"""Брокер заданий: Redis Streams с consumer group (lab5). + +Почему это durable at-least-once очередь (подробно — REPORT.md): + +* сервис `broker` — отдельный Redis с AOF и `appendfsync always`: ответ на XADD + приходит только после fsync записи в журнал. Успешный XADD — наш аналог + publisher confirm: сообщение переживёт рестарт/падение брокера с volume; +* `maxmemory-policy noeviction`: поток никогда не вытесняется (при нехватке + памяти XADD получит ошибку, строка outbox останется и будет опубликована позже); +* consumer group: XREADGROUP выдаёт сообщение одному consumer и кладёт его в + PEL (pending entries list) до XACK. Consumer ack — отдельная граница: worker + делает XACK только ПОСЛЕ устойчивого эффекта (COMMIT результата в PostgreSQL); +* упавший consumer: его сообщения остаются в PEL. Тот же consumer после + рестарта контейнера (то же имя) читает свой PEL с id 0; иначе любой живой + consumer забирает их XAUTOCLAIM после JOB_CLAIM_IDLE_SECONDS простоя. + +В сообщении — только JSON по contracts/jobs.schema.json (ID и служебные поля): +ни байтов файла, ни credentials, ни URL, ни filename. +""" +import json +import logging +from typing import Dict, List, Optional, Tuple + +import redis + +log = logging.getLogger("messenger.broker") + +FIELD = "job" + + +class BrokerUnavailable(Exception): + """Брокер недоступен/таймаут/ошибка записи.""" + + +class Delivery: + __slots__ = ("message_id", "raw", "deliveries") + + def __init__(self, message_id: str, raw: Optional[str], deliveries: int = 1) -> None: + self.message_id = message_id + self.raw = raw + self.deliveries = deliveries + + def payload(self) -> Optional[dict]: + if self.raw is None: + return None + try: + data = json.loads(self.raw) + except (TypeError, ValueError): + return None + return data if isinstance(data, dict) else None + + +def _s(value) -> str: + return value.decode("utf-8", "replace") if isinstance(value, (bytes, bytearray)) else str(value) + + +class Broker: + def __init__(self, url: str, stream: str, group: str, timeout: float = 2.0, + block_ms: int = 2000) -> None: + self.stream = stream + self.group = group + self.timeout = timeout + self.block_ms = block_ms + common = dict(socket_connect_timeout=timeout, health_check_interval=15, + retry_on_timeout=False, decode_responses=False) + # Обычные команды: короткий таймаут сокета -> быстрый BrokerUnavailable. + self.client = redis.Redis.from_url(url, socket_timeout=timeout, **common) + # Блокирующий XREADGROUP: таймаут сокета больше времени блокировки. + self.blocking = redis.Redis.from_url(url, socket_timeout=timeout + block_ms / 1000.0, **common) + self._group_ready = False + + def close(self) -> None: + for c in (self.client, self.blocking): + try: + c.close() + except Exception: + pass + + # -------------------------------------------------------------- producer + + def ensure_group(self) -> None: + """Идемпотентно: поток + группа с позиции 0 (сообщения, опубликованные до + создания группы, тоже будут доставлены).""" + if self._group_ready: + return + try: + self.client.xgroup_create(self.stream, self.group, id="0", mkstream=True) + log.info("broker: created stream %s group %s", self.stream, self.group) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise BrokerUnavailable() from exc + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + self._group_ready = True + + def publish(self, payload: Dict) -> str: + """XADD; возврат = подтверждение приёма брокером (AOF fsync always).""" + body = json.dumps(payload, separators=(",", ":"), sort_keys=True) + try: + self.ensure_group() + return _s(self.client.xadd(self.stream, {FIELD: body})) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + # -------------------------------------------------------------- consumer + + @staticmethod + def _parse(entries) -> List[Delivery]: + out = [] + for message_id, fields in entries or []: + if fields is None: # запись удалена из потока, но осталась в PEL + out.append(Delivery(_s(message_id), None)) + continue + raw = fields.get(FIELD.encode()) if isinstance(fields, dict) else None + out.append(Delivery(_s(message_id), _s(raw) if raw is not None else None)) + return out + + def read(self, consumer: str, own_pending: bool = False) -> List[Delivery]: + """Одно сообщение (prefetch = 1 на consumer). own_pending: свой PEL с id 0 — + сообщения, полученные прошлой инкарнацией этого consumer и не подтверждённые.""" + try: + self.ensure_group() + if own_pending: + resp = self.client.xreadgroup(self.group, consumer, {self.stream: "0"}, count=1) + else: + resp = self.blocking.xreadgroup(self.group, consumer, {self.stream: ">"}, + count=1, block=self.block_ms) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + result = [] + for _stream, entries in resp or []: + result.extend(self._parse(entries)) + return result + + def autoclaim(self, consumer: str, min_idle_ms: int) -> List[Delivery]: + """Забрать одно сообщение, которое другой consumer держит дольше min_idle_ms.""" + try: + self.ensure_group() + resp = self.client.xautoclaim(self.stream, self.group, consumer, min_idle_ms, + start_id="0-0", count=1) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + entries = resp[1] if resp and len(resp) > 1 else [] + deliveries = self._parse(entries) + for d in deliveries: + d.deliveries = self.delivery_count(d.message_id) + return deliveries + + def delivery_count(self, message_id: str) -> int: + try: + info = self.client.xpending_range(self.stream, self.group, message_id, message_id, 1) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + return int(info[0]["times_delivered"]) if info else 1 + + def ack(self, message_id: str) -> None: + try: + self.client.xack(self.stream, self.group, message_id) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + # -------------------------------------------------------- housekeeping + + def trim_acknowledged(self) -> int: + """XTRIM MINID: удалить только записи, которые уже доставлены И подтверждены + (id меньше самого старого pending и не больше last-delivered-id группы). + Недоставленные и неподтверждённые записи никогда не удаляются.""" + try: + groups = self.client.xinfo_groups(self.stream) + if not groups: + return 0 + bounds = [] + for g in groups: + last = _s(g.get("last-delivered-id") or g.get(b"last-delivered-id") or "0-0") + bounds.append(_increment(last)) + pending = self.client.xpending(self.stream, _s(g["name"])) + if pending and pending.get("pending"): + bounds.append(_s(pending["min"])) + minid = min(bounds, key=_id_key) + if minid == "0-0" or minid == "0-1": + return 0 + return int(self.client.xtrim(self.stream, minid=minid, approximate=False) or 0) + except redis.ResponseError: + return 0 + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + def stats(self) -> Dict: + """Длина потока, pending (в обработке/не подтверждено), lag (ещё не выдано).""" + try: + length = int(self.client.xlen(self.stream)) + out = {"stream": self.stream, "length": length, "groups": []} + for g in self.client.xinfo_groups(self.stream): + out["groups"].append({ + "name": _s(g["name"]), + "consumers": int(g["consumers"]), + "pending": int(g["pending"]), + "lag": g.get("lag"), + "last_delivered_id": _s(g["last-delivered-id"]), + }) + return out + except redis.ResponseError: + return {"stream": self.stream, "length": 0, "groups": []} + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + def ping(self) -> bool: + try: + return bool(self.client.ping()) + except redis.RedisError: + return False + + +def _id_key(message_id: str) -> Tuple[int, int]: + ms, _, seq = message_id.partition("-") + return int(ms), int(seq or 0) + + +def _increment(message_id: str) -> str: + ms, seq = _id_key(message_id) + return "%d-%d" % (ms, seq + 1) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..eca740d --- /dev/null +++ b/src/config.py @@ -0,0 +1,244 @@ +"""Конфигурация из переменных окружения. + +Секреты (строки подключения с паролями) читаются из файлов *_FILE, +которые Compose монтирует как secrets в /run/secrets/. Для локального +запуска без Docker допускаются и прямые переменные DATABASE_URL / CACHE_URL. +""" +import os +from dataclasses import dataclass +from typing import Optional + + +def _read_secret(name: str) -> Optional[str]: + path = os.environ.get(name + "_FILE") + if path: + with open(path, "r", encoding="utf-8") as handle: + value = handle.read().strip() + return value or None + value = os.environ.get(name, "").strip() + return value or None + + +def _int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return int(raw) + + +def _float(name: str, default: float) -> float: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return float(raw) + + +def _bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class Settings: + app_env: str + app_version: str + instance_id: str + database_url: str + cache_url: Optional[str] + session_store: str + session_ttl_seconds: int + session_cache_ttl_seconds: int + db_pool_min: int + db_pool_max: int + db_pool_timeout: float + db_connect_timeout: int + db_statement_timeout_ms: int + cache_timeout_seconds: float + cache_retry_after_seconds: float + migrate_wait_seconds: int + run_migrations_on_start: bool + idempotency_ttl_seconds: int + idempotency_cleanup_interval_seconds: int + ready_db_timeout_seconds: float + drain_seconds: float + graceful_timeout_seconds: int + keep_alive_seconds: int + password_hash_concurrency: int + # lab4: S3 и вложения + s3_endpoint: Optional[str] + s3_region: str + s3_bucket: str + s3_access_key_id: Optional[str] + s3_secret_access_key: Optional[str] + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + s3_max_attempts: int + s3_download_deadline_seconds: float + s3_upload_deadline_seconds: float + s3_circuit_open_seconds: float + max_upload_bytes: int + max_image_pixels: int + max_concurrent_uploads: int + upload_queue_timeout_seconds: float + upload_read_timeout_seconds: float + image_decode_concurrency: int + attachment_gc_interval_seconds: int + attachment_gc_grace_seconds: int + orphan_sweep_interval_seconds: int + fault_injection: str + # lab5: брокер, outbox и worker + broker_url: Optional[str] + broker_stream: str + broker_group: str + broker_timeout_seconds: float + pipeline_version: int + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_grace_seconds: int + job_claim_idle_seconds: int + job_max_deliveries: int + job_retry_base_seconds: float + job_retry_max_seconds: float + worker_image_timeout_seconds: float + worker_image_memory_mb: int + worker_faults: bool + staging_retention_seconds: int + relay_enabled: bool + relay_poll_interval_seconds: float + relay_fast_path_grace_seconds: float + relay_batch: int + + +# Жёсткие пределы контракта lab4/lab5; конфигурация может их только ужесточить. +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 +# Grace period сборщика должен быть заметно больше самой долгой загрузки +# (чтение тела + S3 PUT с повтором), см. REPORT.md. +MIN_GC_GRACE_SECONDS = 300 + + +def load_settings() -> Settings: + database_url = _read_secret("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE must be set") + + ttl = _int("SESSION_TTL_SECONDS", 3600) + if ttl < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") + + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() + # Источник истины для сессий в этой реализации всегда PostgreSQL; + # Redis (CACHE_URL) — только кэш проверок токена. + if session_store != "postgres": + raise RuntimeError("only SESSION_STORE=postgres is supported") + + idem_ttl = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + if idem_ttl < 86400: + # Контракт: повтор в течение 24 часов возвращает тот же результат. + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be >= 86400") + + # X-Instance-Id: по умолчанию hostname контейнера (= короткий ID контейнера + # Docker), стабилен в пределах жизни процесса. INSTANCE_ID — для запуска без Docker. + from .runtime import INSTANCE_ID as instance_id + + max_upload = _int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES) + max_pixels = _int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS) + if not 1 <= max_upload <= CONTRACT_MAX_UPLOAD_BYTES: + raise RuntimeError("MAX_UPLOAD_BYTES must be within 1..10485760") + if not 1 <= max_pixels <= CONTRACT_MAX_IMAGE_PIXELS: + raise RuntimeError("MAX_IMAGE_PIXELS must be within 1..40000000") + gc_grace = _int("ATTACHMENT_GC_GRACE_SECONDS", 900) + # Меньше минимума — только для явных тестов сверки (ALLOW_SHORT_GC_GRACE=true). + if gc_grace < MIN_GC_GRACE_SECONDS and not _bool("ALLOW_SHORT_GC_GRACE", False): + raise RuntimeError("ATTACHMENT_GC_GRACE_SECONDS must be >= %d" % MIN_GC_GRACE_SECONDS) + + job_timeout = _int("JOB_TIMEOUT_SECONDS", 60) + image_timeout = _float("WORKER_IMAGE_TIMEOUT_SECONDS", 30.0) + # Аренда обязана покрывать самую долгую попытку: обработка изображения + + # чтение staging и запись 4 объектов (дедлайны S3) + COMMIT. + if job_timeout < image_timeout + 20: + raise RuntimeError("JOB_TIMEOUT_SECONDS must be >= WORKER_IMAGE_TIMEOUT_SECONDS + 20") + max_attempts = _int("JOB_MAX_ATTEMPTS", 5) + if not 1 <= max_attempts <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS must be within 1..20") + claim_idle = _int("JOB_CLAIM_IDLE_SECONDS", job_timeout + 15) + if claim_idle <= job_timeout: + raise RuntimeError("JOB_CLAIM_IDLE_SECONDS must be > JOB_TIMEOUT_SECONDS") + + return Settings( + app_env=os.environ.get("APP_ENV", "lab"), + app_version=os.environ.get("APP_VERSION", "lab5"), + instance_id=instance_id, + database_url=database_url, + cache_url=_read_secret("CACHE_URL"), + session_store=session_store, + session_ttl_seconds=ttl, + session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60), + db_pool_min=_int("DB_POOL_MIN", 1), + db_pool_max=_int("DB_POOL_MAX", 10), + db_pool_timeout=_float("DB_POOL_TIMEOUT_SECONDS", 3.0), + db_connect_timeout=_int("DB_CONNECT_TIMEOUT_SECONDS", 3), + db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.25), + cache_retry_after_seconds=_float("CACHE_RETRY_AFTER_SECONDS", 5.0), + migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60), + # lab3: миграции выполняет отдельный one-shot сервис `migrate`, + # реплики API их не запускают (нет гонки N реплик за схему). + run_migrations_on_start=_bool("RUN_MIGRATIONS_ON_START", False), + idempotency_ttl_seconds=idem_ttl, + idempotency_cleanup_interval_seconds=_int("IDEMPOTENCY_CLEANUP_INTERVAL_SECONDS", 300), + ready_db_timeout_seconds=_float("READY_DB_TIMEOUT_SECONDS", 1.0), + drain_seconds=_float("DRAIN_SECONDS", 3.0), + graceful_timeout_seconds=_int("GRACEFUL_TIMEOUT_SECONDS", 10), + keep_alive_seconds=_int("KEEP_ALIVE_SECONDS", 30), + password_hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 4), + s3_endpoint=os.environ.get("S3_ENDPOINT", "").strip() or None, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments").strip(), + s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID"), + s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY"), + s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 2.0), + s3_read_timeout_seconds=_float("S3_READ_TIMEOUT_SECONDS", 10.0), + s3_max_attempts=_int("S3_MAX_ATTEMPTS", 2), + # Полный бюджет вызова S3 из запроса (DNS + connect + повторы). Скачивание + # обязано уложиться в timeout server HAProxy (10 с), загрузка — 10 с (PUT 10 MiB локально ~0.1 с). + s3_download_deadline_seconds=_float("S3_DOWNLOAD_DEADLINE_SECONDS", 5.0), + s3_upload_deadline_seconds=_float("S3_UPLOAD_DEADLINE_SECONDS", 10.0), + s3_circuit_open_seconds=_float("S3_CIRCUIT_OPEN_SECONDS", 5.0), + max_upload_bytes=max_upload, + max_image_pixels=max_pixels, + max_concurrent_uploads=max(1, _int("MAX_CONCURRENT_UPLOADS", 4)), + upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0), + upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 60.0), + image_decode_concurrency=max(1, _int("IMAGE_DECODE_CONCURRENCY", 2)), + attachment_gc_interval_seconds=_int("ATTACHMENT_GC_INTERVAL_SECONDS", 300), + attachment_gc_grace_seconds=gc_grace, + orphan_sweep_interval_seconds=_int("ORPHAN_SWEEP_INTERVAL_SECONDS", 3600), + # Только для тестового экземпляра api-faults (профиль faults), см. REPORT.md. + fault_injection=os.environ.get("FAULT_INJECTION", "").strip().lower(), + broker_url=_read_secret("BROKER_URL"), + broker_stream=os.environ.get("BROKER_STREAM", "attachments.jobs").strip(), + broker_group=os.environ.get("BROKER_GROUP", "attachment-workers").strip(), + broker_timeout_seconds=_float("BROKER_TIMEOUT_SECONDS", 2.0), + pipeline_version=max(1, _int("PIPELINE_VERSION", 1)), + worker_concurrency=max(1, _int("WORKER_CONCURRENCY", 2)), + job_max_attempts=max_attempts, + job_timeout_seconds=job_timeout, + job_lease_grace_seconds=_int("JOB_LEASE_GRACE_SECONDS", 10), + job_claim_idle_seconds=claim_idle, + job_max_deliveries=max(2, _int("JOB_MAX_DELIVERIES", 10)), + job_retry_base_seconds=_float("JOB_RETRY_BASE_SECONDS", 2.0), + job_retry_max_seconds=_float("JOB_RETRY_MAX_SECONDS", 60.0), + worker_image_timeout_seconds=image_timeout, + worker_image_memory_mb=_int("WORKER_IMAGE_MEMORY_MB", 1024), + # Только для стенда отказов (compose.faults.yaml): маркеры в filename. + worker_faults=_bool("WORKER_FAULTS", False), + staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400), + relay_enabled=_bool("RELAY_ENABLED", True), + relay_poll_interval_seconds=_float("RELAY_POLL_INTERVAL_SECONDS", 1.0), + relay_fast_path_grace_seconds=_float("RELAY_FAST_PATH_GRACE_SECONDS", 2.0), + relay_batch=max(1, _int("RELAY_BATCH", 100)), + ) diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..d8d413b --- /dev/null +++ b/src/db.py @@ -0,0 +1,130 @@ +"""Пул соединений PostgreSQL и версионируемые миграции.""" +import logging +import time +from pathlib import Path +from typing import Optional + +import psycopg +from psycopg_pool import ConnectionPool + +from .config import Settings + +log = logging.getLogger("messenger.db") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + +# Произвольная константа: один процесс за раз применяет миграции, +# даже если одновременно стартуют несколько реплик API (lab3). +MIGRATION_LOCK_ID = 2026_0002 + + +def conninfo_kwargs(settings: Settings) -> dict: + return { + "connect_timeout": settings.db_connect_timeout, + # Все timestamptz приходят в UTC; любой запрос ограничен по времени. + "options": "-c TimeZone=UTC -c statement_timeout=%d" + % settings.db_statement_timeout_ms, + "application_name": "messenger-api", + } + + +def create_pool(settings: Settings) -> ConnectionPool: + pool = ConnectionPool( + conninfo=settings.database_url, + kwargs=conninfo_kwargs(settings), + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + # Сколько запрос ждёт свободное соединение, прежде чем получить 503. + timeout=settings.db_pool_timeout, + # Битые после рестарта БД соединения отбрасываются при выдаче из пула. + check=ConnectionPool.check_connection, + max_idle=300, + reconnect_timeout=3600, + open=False, + name="messenger", + ) + # wait=False: процесс поднимается, даже если БД ещё/уже недоступна; + # readiness это покажет, а запросы получат 503. + pool.open(wait=False) + return pool + + +def _migration_files(): + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + if not files: + raise RuntimeError("no migrations found in %s" % MIGRATIONS_DIR) + return files + + +def required_schema_version() -> str: + """Последняя миграция, которую знает этот образ. + + Реплика готова (readiness) только если она уже применена: миграции + запускает отдельный сервис `migrate`, а не сама реплика. + """ + return _migration_files()[-1].stem + + +def schema_has_version(pool: ConnectionPool, version: str, timeout: float) -> bool: + with pool.connection(timeout=timeout) as conn: + row = conn.execute( + "SELECT to_regclass('schema_migrations') IS NOT NULL" + ).fetchone() + if not row[0]: + return False + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = %s)", + (version,), + ).fetchone()[0] + + +def apply_migrations(settings: Settings) -> list: + """Применяет недостающие миграции; возвращает список применённых версий.""" + applied_now = [] + with psycopg.connect( + settings.database_url, autocommit=True, **conninfo_kwargs(settings) + ) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + done = { + row[0] + for row in conn.execute("SELECT version FROM schema_migrations") + } + for path in _migration_files(): + version = path.stem + if version in done: + continue + sql = path.read_text(encoding="utf-8") + # DDL и отметка о версии — одна транзакция: либо всё, либо ничего. + with conn.transaction(): + conn.execute(sql) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + applied_now.append(version) + log.info("applied migration %s", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,)) + return applied_now + + +def wait_and_migrate(settings: Settings) -> list: + deadline = time.monotonic() + settings.migrate_wait_seconds + last_error: Optional[Exception] = None + while time.monotonic() < deadline: + try: + return apply_migrations(settings) + except psycopg.OperationalError as exc: + last_error = exc + log.warning("database not ready for migrations, retrying") + time.sleep(1) + raise RuntimeError("database unavailable for migrations") from last_error diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..569a655 --- /dev/null +++ b/src/images.py @@ -0,0 +1,284 @@ +"""Проверка загружаемых изображений по содержимому, с ограниченной памятью. + +Порядок для каждого файла (лимит байт уже применён при чтении тела): +1. Сигнатура: PNG (\\x89PNG\\r\\n\\x1a\\n) или JPEG (FF D8 FF). Иное -> 415. +2. Сигнатура должна совпасть с заявленным Content-Type (подмена MIME -> 415). +3. Размеры читаются из заголовка (PNG IHDR / JPEG SOFn) ДО распаковки пикселей; + width*height > MAX_IMAGE_PIXELS -> 413. Распаковка не начинается. +4. Целостность без полного декодирования в память: + * PNG: обход всех чанков с проверкой CRC, порядка IHDR..IDAT..IEND и + потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»: + объём распакованных данных должен в точности равняться ожидаемому + (height*(1+rowbytes), с учётом Adam7). Память — O(64 KiB) при любом + размере картинки; «zip-бомба» обрывается, как только вывод превысил ожидание. + * JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI). + Baseline дополнительно декодируется Pillow в режиме draft (масштаб 1/8, + ~5 MiB даже для 40 Мп). Progressive Pillow не декодируется: libjpeg держит + буфер коэффициентов всего кадра (~120 MiB на 40 Мп, замер в REPORT.md), + для него — только структурная проверка. +5. Pillow открывает файл независимым парсером: формат и размеры должны совпасть. +""" +import io +import struct +import zlib +from typing import Tuple + +from PIL import Image, ImageFile + +# Свою проверку пикселей делаем сами (шаг 3) — до вызова Pillow. +Image.MAX_IMAGE_PIXELS = None +ImageFile.LOAD_TRUNCATED_IMAGES = False + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +JPEG_SIG = b"\xff\xd8\xff" +INFLATE_CHUNK = 64 * 1024 + +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# SOF0..SOF15 без DHT(C4), JPG(C8), DAC(CC) +SOF_MARKERS = set(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} +PROGRESSIVE_SOF = {0xC2, 0xC6, 0xCA, 0xCE} + + +class ImageRejected(Exception): + def __init__(self, status: int, code: str, message: str) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def unsupported(message: str = "Unsupported or corrupted image") -> ImageRejected: + return ImageRejected(415, "unsupported_media_type", message) + + +def too_many_pixels() -> ImageRejected: + return ImageRejected(413, "payload_too_large", "Image has too many pixels") + + +def sniff(data) -> str: + head = bytes(data[:8]) + if head.startswith(PNG_SIG): + return "image/png" + if head.startswith(JPEG_SIG): + return "image/jpeg" + return "" + + +# ------------------------------------------------------------------- PNG + +def _png_expected_raw(width: int, height: int, depth: int, color: int, interlace: int) -> int: + bpp = depth * PNG_CHANNELS[color] + + def rows(w: int, h: int) -> int: + if w == 0 or h == 0: + return 0 + return h * (1 + (w * bpp + 7) // 8) + + if interlace == 0: + return rows(width, height) + total = 0 + for x0, y0, dx, dy in ADAM7: + pw = (width - x0 + dx - 1) // dx if width > x0 else 0 + ph = (height - y0 + dy - 1) // dy if height > y0 else 0 + total += rows(pw, ph) + return total + + +def inspect_png(data, max_pixels: int) -> Tuple[int, int, bool]: + view = memoryview(data) + size = len(view) + pos = 8 + width = height = 0 + expected = 0 + inflater = None + produced = 0 + seen_ihdr = seen_plte = idat_done = False + idat_started = False + color = 0 + + while True: + if pos + 12 > size: + raise unsupported("Truncated PNG") + length, ctype = struct.unpack(">I4s", view[pos:pos + 8]) + if length > 0x7FFFFFFF or pos + 12 + length > size: + raise unsupported("Truncated PNG") + if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype): + raise unsupported("Invalid PNG chunk") + body = view[pos + 8:pos + 8 + length] + (crc,) = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length]) + if zlib.crc32(body, zlib.crc32(ctype)) & 0xFFFFFFFF != crc: + raise unsupported("PNG CRC mismatch") + pos += 12 + length + + if not seen_ihdr: + if ctype != b"IHDR" or length != 13: + raise unsupported("PNG must start with IHDR") + width, height, depth, color, comp, flt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF): + raise unsupported("Invalid PNG dimensions") + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or comp or flt or interlace > 1: + raise unsupported("Invalid PNG header") + # Лимит пикселей — до любой распаковки. + if width * height > max_pixels: + raise too_many_pixels() + expected = _png_expected_raw(width, height, depth, color, interlace) + seen_ihdr = True + continue + + if ctype == b"IHDR": + raise unsupported("Duplicate IHDR") + if ctype == b"PLTE": + if idat_started: + raise unsupported("PLTE after IDAT") + seen_plte = True + elif ctype == b"IDAT": + if idat_done: + raise unsupported("Non-contiguous IDAT") + if color == 3 and not seen_plte: + raise unsupported("Missing PLTE") + if not idat_started: + idat_started = True + inflater = zlib.decompressobj() + produced += _inflate(inflater, body, expected - produced) + elif ctype == b"IEND": + if length != 0 or not idat_started: + raise unsupported("Invalid PNG end") + produced += _inflate(inflater, b"", expected - produced, final=True) + if not inflater.eof or produced != expected: + raise unsupported("Corrupted PNG image data") + # Данные после IEND не используются декодерами; допускаем. + return width, height, bool(interlace) + else: + if idat_started: + idat_done = True + if ctype[0] & 0x20 == 0: + # Неизвестный КРИТИЧЕСКИЙ чанк (заглавная первая буква): декодер обязан + # отказаться от такого файла. Вспомогательные (строчная буква) пропускаем. + raise unsupported("Unknown critical PNG chunk") + + +def _inflate(inflater, data, budget: int, final: bool = False) -> int: + """Распаковать data, не держа в памяти больше ~2 x INFLATE_CHUNK; вернуть объём вывода. + + Вход подаётся срезами memoryview по INFLATE_CHUNK: unconsumed_tail — копия + остатка входа, поэтому подача всего IDAT целиком давала бы копию размером с + файл на каждом шаге (замер: пик 2x размера файла и квадратичное копирование). + """ + produced = 0 + view = memoryview(data) + try: + for start in range(0, len(view), INFLATE_CHUNK): + buf = view[start:start + INFLATE_CHUNK] + while True: + out = inflater.decompress(buf, INFLATE_CHUNK) + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + buf = inflater.unconsumed_tail + if not buf: + break + if final: + while True: + out = inflater.flush(INFLATE_CHUNK) if not inflater.eof else b"" + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + if not out: + break + except zlib.error: + raise unsupported("Corrupted PNG image data") + return produced + + +# ------------------------------------------------------------------ JPEG + +def inspect_jpeg(data, max_pixels: int) -> Tuple[int, int, bool]: + raw = data if isinstance(data, (bytes, bytearray)) else bytes(data) + size = len(raw) + pos = 2 + width = height = 0 + progressive = False + seen_sof = False + while True: + # Маркер: один или несколько 0xFF, затем код. + if pos >= size or raw[pos] != 0xFF: + raise unsupported("Corrupted JPEG structure") + while pos < size and raw[pos] == 0xFF: + pos += 1 + if pos >= size: + raise unsupported("Truncated JPEG") + marker = raw[pos] + pos += 1 + if marker == 0xD9: # EOI + if not seen_sof: + raise unsupported("JPEG without frame") + return width, height, progressive + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if marker == 0xD8 or marker == 0x00: + raise unsupported("Corrupted JPEG structure") + if pos + 2 > size: + raise unsupported("Truncated JPEG") + (length,) = struct.unpack(">H", raw[pos:pos + 2]) + if length < 2 or pos + length > size: + raise unsupported("Truncated JPEG") + segment = raw[pos + 2:pos + length] + pos += length + if marker in SOF_MARKERS: + if seen_sof or len(segment) < 6: + raise unsupported("Invalid JPEG frame") + _precision, height, width, ncomp = struct.unpack(">BHHB", segment[:6]) + if width == 0 or height == 0 or ncomp not in (1, 3, 4) or len(segment) < 6 + 3 * ncomp: + raise unsupported("Invalid JPEG frame") + if width * height > max_pixels: + raise too_many_pixels() + progressive = marker in PROGRESSIVE_SOF + seen_sof = True + elif marker == 0xDA: # SOS: дальше энтропийные данные до следующего маркера + if not seen_sof: + raise unsupported("JPEG scan before frame") + while True: + nxt = raw.find(b"\xff", pos) + if nxt < 0 or nxt + 1 >= size: + raise unsupported("Truncated JPEG") + code = raw[nxt + 1] + if code == 0x00 or 0xD0 <= code <= 0xD7 or code == 0xFF: + pos = nxt + 1 if code == 0xFF else nxt + 2 + continue + pos = nxt + break + + +# -------------------------------------------------------------- common + +def inspect(data, declared_type: str, max_pixels: int) -> Tuple[str, int, int]: + """Вернуть (content_type, width, height) или бросить ImageRejected.""" + detected = sniff(data) + if not detected: + raise unsupported("Only PNG and JPEG images are accepted") + if detected != declared_type: + raise unsupported("Content-Type does not match image content") + if detected == "image/png": + width, height, _ = inspect_png(data, max_pixels) + pil_format = "PNG" + decode = False # целостность IDAT уже проверена потоковой распаковкой + else: + width, height, progressive = inspect_jpeg(data, max_pixels) + pil_format = "JPEG" + decode = not progressive + # Независимый парсер Pillow: только заголовок (open ленивый), затем при + # необходимости декодирование уменьшенной копии. + try: + with Image.open(io.BytesIO(data)) as img: + if img.format != pil_format or img.size != (width, height): + raise unsupported("Inconsistent image header") + if decode: + img.draft(None, (max(1, width // 8), max(1, height // 8))) + img.load() + except ImageRejected: + raise + except (OSError, SyntaxError, ValueError, struct.error, Image.DecompressionBombError, EOFError): + raise unsupported("Corrupted image") + return detected, width, height diff --git a/src/imaging.py b/src/imaging.py new file mode 100644 index 0000000..9aaf13b --- /dev/null +++ b/src/imaging.py @@ -0,0 +1,274 @@ +"""Варианты изображения (lab5) и их изолированное вычисление. + +Варианты pipeline_version=1 (параметры фиксированы, вывод детерминирован для +одной версии Pillow/libjpeg/libwebp, поэтому повтор той же job пишет те же байты): + + thumbnail JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, + размеры округляются вниз (минимум 1 px): 320x200 -> 256x160. + quality=85, optimize, baseline, 4:2:0. + crop JPEG 128x128: центрированный квадрат min(w,h) оригинала, масштаб + до 128 (маленькие исходники увеличиваются). quality=85. + optimized WebP, вписан в 1280x1280 без увеличения, quality=80, method=4. + Уменьшение размера каждого маленького файла НЕ обещается. + +Для всех вариантов: EXIF orientation применяется до вычисления размеров; +прозрачность (RGBA/LA/P с tRNS) компонуется на белый фон; EXIF и XMP НЕ +копируются (Pillow сохраняет их только при явной передаче exif=/xmp=), ICC-профиль +сохраняется, чтобы не исказить цвет широкого охвата. + +Ограничения ресурсов worker (недоверенный файл, даже прошедший проверки API): + * bytes: staging-объект больше MAX_UPLOAD_BYTES не читается (worker.py); + * pixels: размеры читаются из заголовка до декодирования; > MAX_IMAGE_PIXELS -> отказ; + * память: обработка идёт в отдельном процессе с RLIMIT_AS = WORKER_IMAGE_MEMORY_MB; + MemoryError/убийство процесса -> постоянная ошибка resource_limit; + * время: процесс убивается через WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка, + повтор ограничен JOB_MAX_ATTEMPTS); RLIMIT_CPU — страховка. + Процессы создаёт forkserver: чистый однопоточный родитель, без унаследованных + соединений PostgreSQL/Redis и потоков worker. +""" +import hashlib +import io +import multiprocessing +import os +import resource +import signal +from typing import Dict, Tuple + +from PIL import Image, ImageFile, ImageOps + +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMB_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +VARIANT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} +VARIANT_EXT = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + +# EXIF orientation 5..8 поворачивает изображение на 90°: ширина и высота меняются. +SWAPPING_ORIENTATIONS = {5, 6, 7, 8} + + +class ImageFailure(Exception): + """Постоянная ошибка: повтор того же файла даст тот же результат.""" + + def __init__(self, code: str, message: str = "") -> None: + super().__init__(message or code) + self.code = code + + +class ImageTimeout(Exception): + """Обработка не уложилась в WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка).""" + + +def fit_within(width: int, height: int, box: int) -> Tuple[int, int]: + """Вписать в box x box без увеличения; целочисленно, округление вниз, минимум 1.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def _orientation(img: Image.Image) -> int: + try: + return int(img.getexif().get(0x0112, 1) or 1) + except Exception: + return 1 + + +def _has_alpha(img: Image.Image) -> bool: + return img.mode in ("RGBA", "LA", "PA") or "transparency" in img.info + + +def _working_mode(img: Image.Image) -> Image.Image: + """RGB или RGBA без лишних копий: уже подходящий режим не конвертируется + (convert в тот же режим всё равно копирует кадр — на 40 Мп это +160 MiB).""" + if img.mode in ("RGB", "RGBA") and not (img.mode == "RGB" and "transparency" in img.info): + return img + return img.convert("RGBA" if _has_alpha(img) else "RGB") + + +def _flatten(img: Image.Image) -> Image.Image: + """Уже УМЕНЬШЕННЫЙ вариант -> RGB: альфа компонуется на белый фон. + Масштабирование RGBA в Pillow идёт с premultiplied alpha, ореолов нет.""" + if img.mode != "RGBA": + return img + background = Image.new("RGB", img.size, (255, 255, 255)) + background.paste(img, mask=img.getchannel("A")) + return background + + +def _encode(img: Image.Image, fmt: str, icc) -> bytes: + out = io.BytesIO() + extra = {"icc_profile": icc} if icc else {} + if fmt == "JPEG": + img.save(out, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=False, + subsampling="4:2:0", **extra) + else: + img.save(out, "WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, lossless=False, **extra) + return out.getvalue() + + +def render(data: bytes, max_pixels: int) -> Dict[str, Tuple[bytes, int, int, str]]: + """{name: (bytes, width, height, content_type)} для thumbnail, crop, optimized.""" + Image.MAX_IMAGE_PIXELS = None # свой лимит ниже, до декодирования + try: + img = Image.open(io.BytesIO(data)) + if img.format not in ("PNG", "JPEG"): + raise ImageFailure("invalid_image", "unsupported format") + raw_w, raw_h = img.size + if raw_w < 1 or raw_h < 1 or raw_w * raw_h > max_pixels: + raise ImageFailure("invalid_image", "too many pixels") + orientation = _orientation(img) + icc = img.info.get("icc_profile") + # Целевые размеры считаются по ОРИЕНТИРОВАННОМУ оригиналу. + w, h = (raw_h, raw_w) if orientation in SWAPPING_ORIENTATIONS else (raw_w, raw_h) + thumb_size = fit_within(w, h, THUMB_BOX) + opt_size = fit_within(w, h, OPTIMIZED_BOX) + if img.format == "JPEG": + # Декодирование сразу в уменьшенном масштабе (1/2..1/8): память и время + # пропорциональны нужному размеру. draft выбирает масштаб >= запрошенного. + want = (opt_size[1], opt_size[0]) if orientation in SWAPPING_ORIENTATIONS else opt_size + img.draft("RGB", want) + img.load() + img = ImageOps.exif_transpose(img) # копия только если orientation != 1 + base = _working_mode(img) + del img + bw, bh = base.size + + result: Dict[str, Tuple[bytes, int, int, str]] = {} + # Сначала масштабирование (reducing_gap: быстрый целочисленный reduce, затем + # LANCZOS), потом компоновка альфы на белый — уже на маленьком кадре. + thumb = _flatten(base.resize(thumb_size, Image.LANCZOS, reducing_gap=3.0) + if base.size != thumb_size else base) + result["thumbnail"] = (_encode(thumb, "JPEG", icc),) + thumb.size + ("image/jpeg",) + del thumb + + side = min(bw, bh) + left, top = (bw - side) // 2, (bh - side) // 2 + crop = _flatten(base.resize((CROP_SIDE, CROP_SIDE), Image.LANCZOS, box=(left, top, left + side, top + side), + reducing_gap=3.0 if side > CROP_SIDE else None)) + result["crop"] = (_encode(crop, "JPEG", icc),) + crop.size + ("image/jpeg",) + del crop + + opt = _flatten(base.resize(opt_size, Image.LANCZOS, reducing_gap=3.0) if base.size != opt_size else base) + result["optimized"] = (_encode(opt, "WEBP", icc),) + opt.size + ("image/webp",) + return result + except ImageFailure: + raise + except MemoryError: + raise ImageFailure("resource_limit", "memory limit exceeded") + except (OSError, SyntaxError, ValueError, EOFError, Image.DecompressionBombError) as exc: + raise ImageFailure("invalid_image", type(exc).__name__) + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +# ---------------------------------------------------------------- sandbox + +def _child(conn, memory_mb: int, cpu_seconds: int, max_pixels: int) -> None: + """Тело дочернего процесса: лимиты -> получить байты -> вычислить -> отправить.""" + try: + limit = memory_mb * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1)) + except (ValueError, OSError): + pass + signal.signal(signal.SIGTERM, signal.SIG_DFL) + try: + data = conn.recv_bytes() + result = render(data, max_pixels) + del data + peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + conn.send(("ok", (result, peak_mb))) + except ImageFailure as exc: + conn.send(("fail", exc.code)) + except MemoryError: + try: + conn.send(("fail", "resource_limit")) + except Exception: + pass + except Exception as exc: # неожиданное — тоже постоянная ошибка обработки + try: + conn.send(("fail", "processing_failed:" + type(exc).__name__)) + except Exception: + pass + finally: + conn.close() + + +class Sandbox: + def __init__(self, memory_mb: int, timeout: float, max_pixels: int) -> None: + self.memory_mb = memory_mb + self.timeout = timeout + self.max_pixels = max_pixels + self.last_peak_mb = 0.0 + method = "forkserver" if "forkserver" in multiprocessing.get_all_start_methods() else "spawn" + self.ctx = multiprocessing.get_context(method) + if method == "forkserver": + self.ctx.set_forkserver_preload(["src.imaging"]) + + def render(self, data: bytes) -> Dict[str, Tuple[bytes, int, int, str]]: + parent, child = self.ctx.Pipe(duplex=True) + proc = self.ctx.Process( + target=_child, + args=(child, self.memory_mb, int(self.timeout) + 5, self.max_pixels), + daemon=True, + ) + proc.start() + child.close() + try: + parent.send_bytes(data) + if not parent.poll(self.timeout): + raise ImageTimeout() + try: + status, value = parent.recv() + except (EOFError, OSError): + # Процесс умер без ответа: OOM/RLIMIT (SIGKILL/SIGSEGV при нехватке памяти). + proc.join(1) + raise ImageFailure("resource_limit", "worker subprocess died (exit %s)" % proc.exitcode) + if status == "ok": + result, self.last_peak_mb = value + return result + code = str(value) + if code.startswith("processing_failed"): + raise ImageFailure("processing_failed", code) + raise ImageFailure(code) + finally: + parent.close() + if proc.is_alive(): + proc.kill() + proc.join(5) + + +def main() -> None: + """Самопроверка: python -m src.imaging FILE [--memory-mb N] [--timeout S].""" + import argparse + import json + parser = argparse.ArgumentParser() + parser.add_argument("file") + parser.add_argument("--memory-mb", type=int, default=1024) + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument("--max-pixels", type=int, default=40_000_000) + args = parser.parse_args() + with open(args.file, "rb") as f: + data = f.read() + try: + box = Sandbox(args.memory_mb, args.timeout, args.max_pixels) + out = box.render(data) + print(json.dumps({**{k: {"bytes": len(v[0]), "width": v[1], "height": v[2], "type": v[3]} + for k, v in out.items()}, "peak_rss_mb": round(box.last_peak_mb, 1)})) + except ImageFailure as exc: + print(json.dumps({"failed": exc.code})) + except ImageTimeout: + print(json.dumps({"failed": "timeout"})) + + +if __name__ == "__main__": + main() diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..9ced989 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,473 @@ +"""Переходы состояний задания обработки вложения (lab5). Только PostgreSQL. + +Таблица переходов (contracts/attachment-states.md) и где она реализована: + + нет -> queued Storage.accept_attachment (API, одна транзакция с outbox) + queued -> processing claim(): аренда = lease_token + lease_owner + lease_until + processing -> ready complete(): fenced по lease_token; варианты + state одной транзакцией + processing -> queued retry(): временная ошибка, attempt+1, backoff+jitter через outbox + processing -> failed fail(): постоянная ошибка / попытки исчерпаны; запись в dead_letters + processing -> queued reap_expired(): аренда истекла (worker умер/завис) — через outbox + failed -> queued replay(): явная операторская команда, аудит в job_events + +Fencing: любая запись результата worker'а выполняется условием +`lease_token = <мой токен> AND status = 'processing'`. Worker, у которого аренду +уже отобрали (истекла и задание перезапущено), не может ни опубликовать результат, +ни перевести задание в retry/failed — его транзакция ничего не изменит. + +Ключ идемпотентности эффекта — (attachment_id, pipeline_version, variant): +PRIMARY KEY attachment_variants + детерминированные ключи объектов S3. +""" +import json +import random +import uuid +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple +from uuid import UUID + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from . import telemetry + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 + + +def iso_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int) -> Dict: + """Сообщение брокера по contracts/jobs.schema.json: только ID и служебные поля. + + lab6: + W3C traceparent/tracestate ТЕКУЩЕГО span (HTTP upload в API, consumer + span при retry в worker). Строка outbox хранит контекст в PostgreSQL, поэтому + связь upload -> publish -> process сохраняется, даже если публикует sweeper + другого процесса через минуту. Без активного span (reaper) полей нет. + """ + return telemetry.inject({ + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": iso_now(), + }) + + +def validate_payload(data: Optional[Dict]) -> Optional[Dict]: + """Строгая проверка входящего сообщения; None — сообщение отравлено (poison).""" + if not isinstance(data, dict): + return None + allowed = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", + "attempt", "created_at", "traceparent", "tracestate"} + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not required <= set(data) or not set(data) <= allowed: + return None + try: + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + return None + if type(data["attempt"]) is not int or data["attempt"] < 1: + return None + if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1: + return None + return { + "job_id": UUID(str(data["job_id"])), + "attachment_id": UUID(str(data["attachment_id"])), + "pipeline_version": data["pipeline_version"], + "attempt": data["attempt"], + } + except (ValueError, TypeError, AttributeError): + return None + + +def backoff_seconds(attempt: int, base: float, cap: float, rng=random) -> float: + """Экспоненциальный backoff с «equal jitter»: d = min(cap, base*2^(n-1)), + задержка равномерно в [d/2, d]. Нижняя граница d/2 гарантирует паузу, + случайная часть разносит повторы многих заданий во времени.""" + d = min(cap, base * (2 ** max(0, attempt - 1))) + return d / 2 + rng.uniform(0, d / 2) + + +class Claim: + __slots__ = ("job_id", "attachment_id", "attempt", "max_attempts", "replay_count", + "pipeline_version", "lease_token", "takeover") + + def __init__(self, row: Dict, takeover: bool) -> None: + self.job_id = row["job_id"] + self.attachment_id = row["attachment_id"] + self.attempt = row["attempt"] + self.max_attempts = row["max_attempts"] + self.replay_count = row["replay_count"] + self.pipeline_version = row["pipeline_version"] + self.lease_token = row["lease_token"] + self.takeover = takeover + + +class JobStore: + def __init__(self, pool: ConnectionPool, settings) -> None: + self.pool = pool + self.settings = settings + + # ------------------------------------------------------------ helpers + + @staticmethod + def _event(conn, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None: + conn.execute( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) " + "VALUES (%s, %s, %s, %s, %s, %s)", + (job_id, attachment_id, attempt, event, detail, actor), + ) + + def event(self, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None: + with self.pool.connection() as conn: + self._event(conn, job_id, attachment_id, attempt, event, detail, actor) + + def has_event(self, job_id, event: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM job_events WHERE job_id = %s AND event = %s)", + (job_id, event), + ).fetchone()[0] + + # ------------------------------------------------------------ claim + + def claim(self, msg: Dict, owner: str) -> Tuple[Optional[Claim], str]: + """queued -> processing. Возвращает (Claim, "") или (None, причина no-op). + + Захват разрешён, если сообщение относится к ТЕКУЩЕЙ попытке и задание + queued, либо processing с истёкшей арендой, либо processing с арендой + этого же consumer (его прошлая инкарнация умерла: consumer обрабатывает + не больше одного сообщения за раз, поэтому живой аренды с его именем быть не может). + """ + token = uuid.uuid4() + with self.pool.connection() as conn: + with conn.transaction(): + cur = conn.cursor(row_factory=dict_row).execute( + """ + SELECT job_id, attachment_id, status, attempt, max_attempts, replay_count, + pipeline_version, lease_owner, + (lease_until IS NOT NULL AND lease_until < clock_timestamp()) AS lease_expired + FROM attachment_jobs WHERE job_id = %s + FOR UPDATE + """, + (msg["job_id"],), + ).fetchone() + if cur is None or cur["attachment_id"] != msg["attachment_id"]: + return None, "unknown_job" + same_attempt = cur["attempt"] == msg["attempt"] and cur["pipeline_version"] == msg["pipeline_version"] + takeover = cur["status"] == "processing" and (cur["lease_expired"] or cur["lease_owner"] == owner) + if same_attempt and (cur["status"] == "queued" or takeover): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', lease_token = %s, lease_owner = %s, + lease_until = clock_timestamp() + make_interval(secs => %s), + started_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s + """, + (token, owner, self.settings.job_timeout_seconds, cur["job_id"]), + ) + conn.execute( + "UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing')", + (cur["attachment_id"],), + ) + self._event(conn, cur["job_id"], cur["attachment_id"], cur["attempt"], + "lease_takeover" if takeover else "claimed", + "previous owner %s" % cur["lease_owner"] if takeover else None, owner) + cur["lease_token"] = token + return Claim(cur, takeover), "" + if cur["status"] == "succeeded": + reason = "already_succeeded" + elif cur["status"] == "failed": + reason = "already_failed" + elif cur["attempt"] > msg["attempt"]: + reason = "stale_attempt" + elif cur["status"] == "processing": + reason = "leased_by_other" + else: + reason = "attempt_mismatch" + # Повторная доставка — нормальна; фиксируем no-op в аудите. + self._event(conn, cur["job_id"], cur["attachment_id"], msg["attempt"], + "redelivery_noop", reason, owner) + return None, reason + + def attachment_source(self, attachment_id: UUID) -> Optional[Dict]: + """Доверенные метаданные для worker'а: только из БД, не из сообщения.""" + with self.pool.connection() as conn: + return conn.cursor(row_factory=dict_row).execute( + "SELECT id, content_type, size_bytes, sha256, width, height, object_key, " + "staging_key, filename, state FROM attachments WHERE id = %s", + (attachment_id,), + ).fetchone() + + # ------------------------------------------------------------ outcomes + + def complete(self, claim: Claim, variants: List[Dict], original_key: str, actor: str) -> Optional[float]: + """processing -> ready. None — аренда потеряна (fencing), ничего не изменено. + + lab6: возвращает секунды от создания задания (COMMIT при приёме, ответ 202) + до COMMIT ready — end-to-end time to ready.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'succeeded', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + finished_at = clock_timestamp(), updated_at = clock_timestamp(), last_error_code = NULL + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt, EXTRACT(EPOCH FROM clock_timestamp() - created_at) + """, + (claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return None + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + [(claim.attachment_id, claim.pipeline_version, v["name"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"], v["object_key"]) + for v in variants], + ) + # Никогда не откатываем более новую версию: pipeline_version <= текущей задачи. + conn.execute( + """ + UPDATE attachments + SET state = 'ready', error_code = NULL, object_key = %s, + pipeline_version = %s, updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version <= %s AND state <> 'pending' + """, + (original_key, claim.pipeline_version, claim.attachment_id, claim.pipeline_version), + ) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "succeeded", + ",".join(v["name"] for v in variants), actor) + return float(row[1]) + + def retry_or_fail(self, claim: Claim, code: str, actor: str) -> Tuple[str, float]: + """Временная ошибка: следующая попытка через outbox (backoff+jitter) или + failed retry_exhausted. Возвращает ("retry", delay) | ("failed", 0) | ("lost", 0).""" + if claim.attempt >= claim.max_attempts: + return ("failed", 0.0) if self.fail(claim, "retry_exhausted", actor, detail=code) else ("lost", 0.0) + delay = backoff_seconds(claim.attempt, self.settings.job_retry_base_seconds, + self.settings.job_retry_max_seconds) + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, last_error_code = %s, updated_at = clock_timestamp(), + next_attempt_at = clock_timestamp() + make_interval(secs => %s) + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt + """, + (code, delay, claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return "lost", 0.0 + self._requeue_side_effects(conn, claim.job_id, claim.attachment_id, claim.pipeline_version, + row[0], delay) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "retry_scheduled", + "%s; next attempt %d in %.2fs" % (code, row[0], delay), actor) + return "retry", delay + + def _requeue_side_effects(self, conn, job_id, attachment_id, pipeline_version, attempt, delay) -> None: + conn.execute( + "UPDATE attachments SET state = 'queued', updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing', 'failed')", + (attachment_id,), + ) + conn.execute( + "INSERT INTO job_outbox (job_id, payload, available_at) " + "VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", + (job_id, Jsonb(job_payload(job_id, attachment_id, pipeline_version, attempt)), delay), + ) + + def fail(self, claim: Claim, code: str, actor: str, detail: Optional[str] = None) -> bool: + """processing -> failed (fenced) + запись в реестр dead_letters.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt + """, + (code, claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return False + self._mark_failed(conn, claim.job_id, claim.attachment_id, code, row[0]) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "failed", + code + ("; " + detail if detail else ""), actor) + return True + + @staticmethod + def _mark_failed(conn, job_id, attachment_id, code, attempts) -> None: + conn.execute( + "UPDATE attachments SET state = 'failed', error_code = %s, updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing')", + (code, attachment_id), + ) + conn.execute( + """ + INSERT INTO dead_letters (job_id, attachment_id, error_code, attempts) + VALUES (%s, %s, %s, %s) + ON CONFLICT (job_id) DO UPDATE + SET error_code = EXCLUDED.error_code, attempts = EXCLUDED.attempts, + failed_at = clock_timestamp() + """, + (job_id, attachment_id, code, attempts), + ) + + def fail_poison(self, job_id: UUID, code: str, actor: str) -> bool: + """Сообщение доставлялось слишком много раз (worker падает на нём) — + перевести задание в failed без аренды, если оно ещё не завершено.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s AND status IN ('queued', 'processing') + RETURNING attachment_id, attempt + """, + (code, job_id), + ).fetchone() + if row is None: + return False + self._mark_failed(conn, job_id, row[0], code, row[1]) + self._event(conn, job_id, row[0], row[1], "failed", code, actor) + return True + + # ------------------------------------------------------------ recovery + + def reap_expired(self, actor: str, limit: int = 100) -> List[Tuple[str, str]]: + """processing с истёкшей арендой (+grace) -> queued attempt+1 через outbox, + или failed processing_timeout, если попытки исчерпаны. SKIP LOCKED: несколько + relay не мешают друг другу.""" + done = [] + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT job_id, attachment_id, attempt, max_attempts, pipeline_version, lease_owner + FROM attachment_jobs + WHERE status = 'processing' + AND lease_until < clock_timestamp() - make_interval(secs => %s) + ORDER BY lease_until LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (self.settings.job_lease_grace_seconds, limit), + ).fetchall() + for r in rows: + if r["attempt"] >= r["max_attempts"]: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = 'processing_timeout', finished_at = clock_timestamp(), + updated_at = clock_timestamp() + WHERE job_id = %s + """, + (r["job_id"],), + ) + self._mark_failed(conn, r["job_id"], r["attachment_id"], "processing_timeout", r["attempt"]) + self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "failed", + "processing_timeout (lease expired, owner %s)" % r["lease_owner"], actor) + done.append((str(r["job_id"]), "failed")) + continue + delay = backoff_seconds(r["attempt"], self.settings.job_retry_base_seconds, + self.settings.job_retry_max_seconds) + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, last_error_code = 'lease_expired', updated_at = clock_timestamp(), + next_attempt_at = clock_timestamp() + make_interval(secs => %s) + WHERE job_id = %s + """, + (delay, r["job_id"]), + ) + self._requeue_side_effects(conn, r["job_id"], r["attachment_id"], r["pipeline_version"], + r["attempt"] + 1, delay) + self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "lease_expired", + "owner %s; next attempt %d in %.2fs" % (r["lease_owner"], r["attempt"] + 1, delay), + actor) + done.append((str(r["job_id"]), "requeued")) + return done + + def replay(self, attachment_id: UUID, actor: str, pipeline_version: Optional[int] = None) -> Tuple[str, str]: + """failed -> queued: явный replay. Успешное задание НЕ исполняется повторно. + + Новая попытка получает attempt+1 и свежий бюджет JOB_MAX_ATTEMPTS. Логический + эффект защищён тем же ключом (attachment_id, pipeline_version, variant). + """ + with self.pool.connection() as conn: + with conn.transaction(): + job = conn.cursor(row_factory=dict_row).execute( + """ + SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key, a.state + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = %s AND (%s::int IS NULL OR j.pipeline_version = %s) + ORDER BY j.pipeline_version DESC LIMIT 1 + FOR UPDATE OF j, a + """, + (attachment_id, pipeline_version, pipeline_version), + ).fetchone() + if job is None: + return "refused", "no job for this attachment" + if job["status"] == "succeeded" or job["state"] == "ready": + self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused", + "already succeeded (no duplicate execution)", actor) + return "noop", "already succeeded; nothing to replay" + if job["status"] != "failed": + return "refused", "job is %s, only failed jobs can be replayed" % job["status"] + if job["staging_key"] is None: + self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused", + "staging source already removed (retention)", actor) + return "refused", "staging source was removed by retention; re-upload required" + attempt = job["attempt"] + 1 + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, replay_count = replay_count + 1, + last_error_code = NULL, next_attempt_at = NULL, finished_at = NULL, + updated_at = clock_timestamp() + WHERE job_id = %s AND status = 'failed' + """, + (attempt, attempt - 1 + self.settings.job_max_attempts, job["job_id"]), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() " + "WHERE id = %s AND state = 'failed'", + (attachment_id,), + ) + conn.execute( + "UPDATE dead_letters SET replayed_at = clock_timestamp(), replay_count = replay_count + 1 " + "WHERE job_id = %s", + (job["job_id"],), + ) + conn.execute( + "INSERT INTO job_outbox (job_id, payload) VALUES (%s, %s)", + (job["job_id"], Jsonb(job_payload(job["job_id"], attachment_id, job["pipeline_version"], attempt))), + ) + self._event(conn, job["job_id"], attachment_id, attempt, "replayed", + "operator replay; new attempt %d" % attempt, actor) + return "replayed", "attempt %d queued" % attempt + + +def dumps(obj) -> str: + return json.dumps(obj, ensure_ascii=False, default=str) diff --git a/src/jobs_admin.py b/src/jobs_admin.py new file mode 100644 index 0000000..c09084f --- /dev/null +++ b/src/jobs_admin.py @@ -0,0 +1,156 @@ +"""Операторские команды очереди (lab5). Секреты и токены не печатаются. + + python -m src.jobs_admin stats # состояния заданий, outbox, брокер (длина/pending/lag) + python -m src.jobs_admin failed [--limit 50] # реестр dead_letters (DLQ) + python -m src.jobs_admin show ATTACHMENT_ID # задание + журнал событий + python -m src.jobs_admin replay ATTACHMENT_ID [...] # явный replay failed-задания + python -m src.jobs_admin replay --all-failed [--error-code retry_exhausted] [--dry-run] + +Replay безопасен: успешное задание не исполняется повторно (no-op с записью в +аудит), replay без staging-источника отклоняется, новая попытка получает attempt+1. +""" +import argparse +import json +import sys +from uuid import UUID + +from psycopg.rows import dict_row + +from .broker import Broker, BrokerUnavailable +from .config import load_settings +from .db import create_pool +from .jobs import JobStore + + +def _print(obj) -> None: + print(json.dumps(obj, ensure_ascii=False, indent=2, default=str)) + + +def cmd_stats(pool, settings, _args) -> int: + with pool.connection() as conn: + jobs = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall()) + atts = dict(conn.execute("SELECT state, count(*) FROM attachments GROUP BY state").fetchall()) + outbox = conn.execute( + "SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()), " + "COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) FROM job_outbox" + ).fetchone() + dlq = conn.execute("SELECT count(*) FROM dead_letters WHERE replayed_at IS NULL").fetchone()[0] + oldest_queued = conn.execute( + "SELECT COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) " + "FROM attachment_jobs WHERE status IN ('queued', 'processing')" + ).fetchone()[0] + out = { + "jobs": jobs, "attachments": atts, + "outbox": {"rows": outbox[0], "due": outbox[1], "oldest_age_s": round(float(outbox[2]), 1)}, + "dead_letters_unreplayed": dlq, + "oldest_unfinished_job_age_s": round(float(oldest_queued), 1), + } + if settings.broker_url: + broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group, + settings.broker_timeout_seconds) + try: + out["broker"] = broker.stats() + except BrokerUnavailable: + out["broker"] = "unavailable" + broker.close() + _print(out) + return 0 + + +def cmd_failed(pool, _settings, args) -> int: + with pool.connection() as conn: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT d.attachment_id, d.job_id, d.error_code, d.attempts, d.failed_at, d.replayed_at, + d.replay_count, j.status AS job_status, (a.staging_key IS NOT NULL) AS replayable + FROM dead_letters d + JOIN attachment_jobs j ON j.job_id = d.job_id + JOIN attachments a ON a.id = d.attachment_id + ORDER BY d.failed_at DESC LIMIT %s + """, + (args.limit,), + ).fetchall() + _print(rows) + return 0 + + +def cmd_show(pool, _settings, args) -> int: + with pool.connection() as conn: + att = conn.cursor(row_factory=dict_row).execute( + "SELECT id, state, error_code, pipeline_version, size_bytes, sha256, " + "(staging_key IS NOT NULL) AS has_staging, created_at, updated_at FROM attachments WHERE id = %s", + (args.attachment_id,), + ).fetchone() + job = conn.cursor(row_factory=dict_row).execute( + "SELECT job_id, status, attempt, max_attempts, replay_count, lease_owner, lease_until, " + "last_error_code, next_attempt_at, created_at, finished_at FROM attachment_jobs " + "WHERE attachment_id = %s ORDER BY pipeline_version", + (args.attachment_id,), + ).fetchall() + variants = conn.cursor(row_factory=dict_row).execute( + "SELECT pipeline_version, name, content_type, width, height, size_bytes, sha256 " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name", + (args.attachment_id,), + ).fetchall() + events = conn.cursor(row_factory=dict_row).execute( + "SELECT at, attempt, event, detail, actor FROM job_events WHERE attachment_id = %s " + "OR job_id IN (SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", + (args.attachment_id, args.attachment_id), + ).fetchall() + _print({"attachment": att, "jobs": job, "variants": variants, "events": events}) + return 0 if att else 1 + + +def cmd_replay(pool, settings, args) -> int: + jobs = JobStore(pool, settings) + ids = [UUID(x) for x in args.attachment_ids] + if args.all_failed: + with pool.connection() as conn: + q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'" + params = [] + if args.error_code: + q += " AND last_error_code = %s" + params.append(args.error_code) + ids += [r[0] for r in conn.execute(q + " ORDER BY updated_at", params).fetchall()] + if not ids: + print("nothing to replay") + return 0 + results = [] + for att in ids: + if args.dry_run: + results.append({"attachment_id": att, "result": "dry-run"}) + continue + outcome, detail = jobs.replay(att, actor="operator:jobs_admin") + results.append({"attachment_id": att, "result": outcome, "detail": detail}) + _print(results) + return 0 if all(r["result"] in ("replayed", "noop", "dry-run") for r in results) else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Attachment job queue operations") + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("stats") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("show") + p.add_argument("attachment_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("attachment_ids", nargs="*") + p.add_argument("--all-failed", action="store_true") + p.add_argument("--error-code") + p.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + if args.cmd == "replay" and not (args.attachment_ids or args.all_failed): + parser.error("replay: give ATTACHMENT_ID(s) or --all-failed") + settings = load_settings() + pool = create_pool(settings) + pool.wait(timeout=15) + try: + return {"stats": cmd_stats, "failed": cmd_failed, "show": cmd_show, "replay": cmd_replay}[args.cmd]( + pool, settings, args) + finally: + pool.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..01cd44e --- /dev/null +++ b/src/main.py @@ -0,0 +1,894 @@ +import asyncio +import base64 +import hashlib +import json +import logging +import os +import re +import threading +import time +import unicodedata +import uuid as uuid_mod +from contextlib import asynccontextmanager +from datetime import datetime +from typing import Callable, Dict, Optional, Tuple +from uuid import UUID, uuid4 + +import psycopg +from opentelemetry import trace +from fastapi import BackgroundTasks, Depends, FastAPI, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import JSONResponse, StreamingResponse +from psycopg_pool import PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect +from starlette.middleware.base import BaseHTTPMiddleware + +from . import telemetry +from .config import load_settings +from .db import create_pool, required_schema_version, schema_has_version, wait_and_migrate +from .images import ImageRejected, inspect as inspect_image +from .models import ( + Attachment, + Chat, + ChatCreate, + ChatPage, + Message, + MessageCreate, + MessagePage, + Session, + User, + UserCreate, +) +from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable +from .broker import Broker +from .jobs import job_payload +from .outbox import Relay +from .reconcile import STAGING_PREFIX, gc_loop +from .runtime import DRAIN, INSTANCE_ID +from .security import ( + configure_kdf_concurrency, + hash_password, + parse_basic, + parse_bearer, + verify_password, +) +from .sessions import SessionCache, SessionService +from .storage import ( + AttachmentFailed, + AttachmentNotFound, + IdempotencyConflict, + IdempotencyRace, + NotChatMember, + Storage, + UnknownMember, + UsernameTaken, +) + +# lab6: OTLP traces/metrics/logs + JSON-логи. До create_pool (инструментация psycopg). +telemetry.setup("messenger-api") +log = logging.getLogger("messenger.api") +access_log = logging.getLogger("messenger.http") + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +# Контракт lab3: Idempotency-Key — ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def _idempotency_cleanup_loop(storage: Storage, interval: float, stop: threading.Event) -> None: + """Фоновая очистка истёкших ключей. Advisory lock — одна реплика за раз.""" + while not stop.wait(interval): + try: + deleted = storage.delete_expired_idempotency_keys() + if deleted: + log.info("deleted %d expired idempotency keys", deleted) + except Exception as exc: # БД недоступна — попробуем в следующий раз + log.warning("idempotency cleanup skipped: %s", type(exc).__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = load_settings() + if settings.run_migrations_on_start: + # Только для локальной разработки. В compose миграции выполняет + # отдельный one-shot сервис `migrate` до старта реплик. + applied = wait_and_migrate(settings) + log.info("migrations applied at startup: %s", applied or "none") + configure_kdf_concurrency(settings.password_hash_concurrency) + pool = create_pool(settings) + storage = Storage(pool) + cache = SessionCache(settings) + app.state.settings = settings + app.state.storage = storage + app.state.cache = cache + app.state.sessions = SessionService(storage, cache, settings) + app.state.required_schema = required_schema_version() + # lab6: DB pool и очередь (из PostgreSQL) — метрики живут, даже когда worker остановлен. + telemetry.register_pool_metrics(pool) + telemetry.register_queue_metrics(pool, settings.broker_stream) + app.state.schema_ok = False + stop_cleanup = threading.Event() + cleanup = threading.Thread( + target=_idempotency_cleanup_loop, + args=(storage, settings.idempotency_cleanup_interval_seconds, stop_cleanup), + name="idempotency-cleanup", + daemon=True, + ) + cleanup.start() + # lab4: S3 и ограничители ресурсов загрузок (на реплику). + app.state.objects = ObjectStore(settings) if ObjectStore.configured(settings) else None + app.state.upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads) + app.state.decode_slots = threading.BoundedSemaphore(settings.image_decode_concurrency) + if app.state.objects is not None: + threading.Thread( + target=gc_loop, + args=(pool, app.state.objects, settings, stop_cleanup), + name="attachment-gc", + daemon=True, + ).start() + else: + log.warning("S3 is not configured: attachment endpoints return 503") + # lab5: outbox relay в каждой реплике (быстрый путь после 202 + sweeper). + # Брокер НЕ входит в синхронный путь: при его недоступности upload всё равно + # отвечает 202, задание ждёт в job_outbox (PostgreSQL). + app.state.relay = None + if settings.broker_url and settings.relay_enabled: + broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group, + settings.broker_timeout_seconds) + app.state.relay = Relay(pool, broker, settings, "api:" + INSTANCE_ID) + threading.Thread(target=app.state.relay.loop, args=(stop_cleanup,), name="outbox-relay", + daemon=True).start() + elif not settings.broker_url: + log.warning("BROKER_URL is not configured: jobs stay in job_outbox until a relay publishes them") + log.info( + "attachments: s3=%s bucket=%s max_bytes=%d max_pixels=%d uploads<=%d decode<=%d gc_grace=%ss fault=%s", + "on" if app.state.objects is not None else "off", + settings.s3_bucket, + settings.max_upload_bytes, + settings.max_image_pixels, + settings.max_concurrent_uploads, + settings.image_decode_concurrency, + settings.attachment_gc_grace_seconds, + settings.fault_injection or "none", + ) + log.info( + "started: instance=%s version=%s session_ttl=%ss cache=%s pool=%s..%s schema>=%s", + INSTANCE_ID, + settings.app_version, + settings.session_ttl_seconds, + "redis" if cache.client is not None else "disabled", + settings.db_pool_min, + settings.db_pool_max, + app.state.required_schema, + ) + try: + yield + finally: + log.info("shutdown: instance=%s closing pool", INSTANCE_ID) + stop_cleanup.set() + pool.close() + if cache.client is not None: + cache.client.close() + telemetry.shutdown() + + +app = FastAPI( + title="MTUSI Messenger (lab6: observability)", + version="6.0.0", + lifespan=lifespan, +) + + +# Request ID + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable): + incoming = request.headers.get("X-Request-Id", "") + request_id = incoming if REQUEST_ID_RE.match(incoming) else str(uuid4()) + request.state.request_id = request_id + # lab6: X-Request-Id в server span — связь «ответ с ошибкой -> trace -> лог». + span = trace.get_current_span() + if span.is_recording(): + span.set_attribute("messenger.request_id", request_id) + started = time.perf_counter() + response = await call_next(request) + path = request.url.path + if not path.startswith("/health/"): + route = request.scope.get("route") + # Только шаблон маршрута, без реальных UUID и query string. + access_log.info( + "%s %s -> %d", request.method, getattr(route, "path", "unmatched"), response.status_code, + extra={"event": "http.request", "http_method": request.method, + "http_route": getattr(route, "path", "unmatched"), + "http_status_code": response.status_code, "request_id": request_id, + "duration_ms": round((time.perf_counter() - started) * 1000, 1)}, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + if DRAIN.active: + # Во время осушения не держим keep-alive: следующий запрос клиент/LB + # откроет новым соединением — уже к другой реплике. + response.headers["Connection"] = "close" + return response + + +app.add_middleware(RequestIdMiddleware) +# Добавляется последним -> самый внешний middleware: RequestIdMiddleware и +# обработчики работают внутри server span (логи получают trace_id/span_id). +telemetry.instrument_fastapi(app) + + +# errors + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", "Authentication required", {"WWW-Authenticate": challenge}) + + +def bad_request(message: str = "Invalid request") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Not found") + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, + headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) or str(uuid4()) + all_headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID} + all_headers.update(headers or {}) + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=all_headers, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError) -> JSONResponse: + return error_response(request, exc.status_code, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Invalid request") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + status_to_code = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + } + status_code = exc.status_code if exc.status_code in status_to_code else 500 + code = status_to_code.get(status_code, "internal_error") + message = str(exc.detail) if status_code != 500 else "Internal server error" + return error_response(request, status_code, code, message) + + +async def dependency_unavailable(request: Request, exc: Exception) -> JSONResponse: + # Наружу — только общий 503, без SQL, адресов и текста драйвера. + log.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "Service temporarily unavailable") + + +app.add_exception_handler(psycopg.OperationalError, dependency_unavailable) +app.add_exception_handler(psycopg.InterfaceError, dependency_unavailable) +app.add_exception_handler(PoolTimeout, dependency_unavailable) + + +@app.exception_handler(Exception) +async def unexpected_exception_handler(request: Request, exc: Exception) -> JSONResponse: + log.exception("unexpected error") + return error_response(request, 500, "internal_error", "Internal server error") + + +# helpers + +def require_json_content_type(request: Request) -> None: + content_type = request.headers.get("content-type", "") + if not content_type.lower().startswith("application/json"): + raise ApiError(415, "unsupported_media_type", "Unsupported media type") + + +def ensure_not_blank(value: str) -> None: + if not value.strip(): + raise bad_request() + + +def storage_of(request: Request) -> Storage: + return request.app.state.storage + + +def current_user(request: Request) -> User: + """Только Authorization: Bearer. Basic и X-User-Id здесь не дают доступа.""" + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise unauthorized() + user = request.app.state.sessions.resolve(token) + if user is None: + raise unauthorized() + request.state.token = token + return user + + +def encode_cursor(resource_id: UUID, created_at: datetime, item_id: UUID) -> str: + payload = { + "resource_id": str(resource_id), + "created_at": created_at.isoformat(), + "item_id": str(item_id), + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: Optional[str], expected_resource_id: UUID) -> Optional[Tuple[datetime, UUID]]: + if cursor is None: + return None + try: + padding = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + padding) + payload = json.loads(raw.decode("utf-8")) + resource_id = UUID(payload["resource_id"]) + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("invalid_cursor") + except (ValueError, KeyError, TypeError, AttributeError, json.JSONDecodeError): + raise bad_request("Invalid cursor") + if resource_id != expected_resource_id: + raise bad_request("Invalid cursor") + return created_at, item_id + + +# health + +@app.get("/health/live", response_model=Dict[str, str]) +def liveness() -> Dict[str, str]: + return {"status": "ok"} + + +@app.get("/health/ready", response_model=Dict[str, str]) +def readiness(request: Request) -> Dict[str, str]: + """Готовность к обязательному синхронному пути. + + * SIGTERM получен (фаза осушения) -> 503: LB выводит реплику из ротации; + * PostgreSQL недоступна за READY_DB_TIMEOUT_SECONDS -> 503; + * схема ещё не мигрирована до версии, которую знает образ -> 503 + (миграции выполняет отдельный сервис `migrate`). + Redis не критичен: это кэш с обходом (см. REPORT.md). + """ + if DRAIN.active: + raise ApiError(503, "unavailable", "Instance is shutting down") + state = request.app.state + timeout = state.settings.ready_db_timeout_seconds + try: + if state.schema_ok: + with state.storage.pool.connection(timeout=timeout) as conn: + conn.execute("SELECT 1") + elif schema_has_version(state.storage.pool, state.required_schema, timeout): + state.schema_ok = True + else: + raise ApiError(503, "unavailable", "Database schema is not migrated") + except (psycopg.Error, PoolTimeout): + raise ApiError(503, "unavailable", "Database is not available") + return {"status": "ok"} + + +# users + +@app.post("/api/v1/users", response_model=User, status_code=201) +def register_user(request: Request, data: UserCreate) -> User: + require_json_content_type(request) + ensure_not_blank(data.display_name) + ensure_not_blank(data.password) + password_hash = hash_password(data.password) + try: + return storage_of(request).create_user(data.username, data.display_name, password_hash) + except UsernameTaken: + raise ApiError(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(user: User = Depends(current_user)) -> User: + return user + + +# sessions + +async def require_empty_body(request: Request) -> None: + """Вход — только заголовок Authorization: Basic, без тела (контракт). + + Любые байты в теле (JSON или нет) — 400. Проверка выполняется до + дорогой проверки пароля и до создания сессии. + """ + if await request.body(): + raise bad_request("Request body is not allowed") + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201, + dependencies=[Depends(require_empty_body)]) +def create_session(request: Request) -> Session: + credentials = parse_basic(request.headers.get("Authorization")) + if credentials is None: + raise unauthorized(BASIC_CHALLENGE) + username, password = credentials + found = storage_of(request).get_credentials(username) + user, password_hash = found if found else (None, None) + # verify_password выполняется и для несуществующего username (с хешем-пустышкой). + if not verify_password(password_hash, password) or user is None: + raise unauthorized(BASIC_CHALLENGE) + token, expires_at = request.app.state.sessions.create(user) + return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_current_session(request: Request, user: User = Depends(current_user)) -> Response: + if not request.app.state.sessions.revoke(request.state.token): + raise unauthorized() + return Response(status_code=204) + + +# chats + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, data: ChatCreate, user: User = Depends(current_user)) -> Chat: + require_json_content_type(request) + ensure_not_blank(data.title) + if len(data.member_ids) != len(set(data.member_ids)): + raise bad_request("Duplicate member ids") + if user.id not in data.member_ids: + raise bad_request("Caller must be a chat member") + try: + return storage_of(request).create_chat(data.title, data.member_ids) + except UnknownMember: + raise bad_request("Unknown member") + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> ChatPage: + after = decode_cursor(cursor, user.id) + items, has_more = storage_of(request).list_user_chats(user.id, limit, after) + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(user.id, items[-1].created_at, items[-1].id) + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID, user: User = Depends(current_user)) -> Chat: + chat = storage_of(request).get_chat_for_member(chat_id, user.id) + if chat is None: + raise not_found() + return chat + + +# messages + +def request_fingerprint(data: MessageCreate) -> bytes: + """SHA-256 канонического JSON тела. + + Канонизация: проверенная модель -> JSON с сортировкой ключей и без пробелов. + Поэтому порядок ключей и форматирование исходного JSON не влияют на hash. + lab4: attachment_ids имеет значение по умолчанию [], поэтому отсутствующее + поле и явный [] дают одинаковый hash (контракт). Порядок id значим. + """ + canonical = json.dumps( + data.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +def create_message( + request: Request, + response: Response, + chat_id: UUID, + data: MessageCreate, + user: User = Depends(current_user), +) -> Message: + # Порядок: аутентификация (dependency) -> ключ -> тело -> доступ к чату. + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad_request("Missing or invalid Idempotency-Key") + require_json_content_type(request) + # lab4: пустой/пробельный текст допустим только вместе с вложениями. + if not data.attachment_ids: + ensure_not_blank(data.text) + if len(set(data.attachment_ids)) != len(data.attachment_ids): + raise bad_request("Duplicate attachment ids") + settings = request.app.state.settings + try: + message, replayed = storage_of(request).create_message( + chat_id, + user.id, + data.text, + key, + request_fingerprint(data), + settings.idempotency_ttl_seconds, + data.attachment_ids, + ) + except (NotChatMember, AttachmentNotFound): + # Чужое, неизвестное или ещё не принятое (pending) вложение — как и чужой чат: 404. + raise not_found() + except AttachmentFailed: + raise ApiError(409, "conflict", "Attachment processing failed; it cannot be attached") + except IdempotencyConflict: + raise ApiError(409, "conflict", "Idempotency-Key was used with a different request") + except IdempotencyRace: + raise ApiError(503, "unavailable", "Please retry with the same Idempotency-Key") + if replayed: + # Информационный заголовок (не часть контракта): ответ — повтор. + response.headers["Idempotency-Replayed"] = "true" + return message + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages( + request: Request, + chat_id: UUID, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> MessagePage: + after = decode_cursor(cursor, chat_id) + result = storage_of(request).list_messages(chat_id, user.id, limit, after) + if result is None: + raise not_found() + items, has_more = result + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(chat_id, items[-1].created_at, items[-1].id) + return MessagePage(items=items, next_cursor=next_cursor) + + +# attachments (lab4) + +ALLOWED_UPLOAD_TYPES = ("image/png", "image/jpeg") +UPLOAD_CHUNK = 64 * 1024 +KNOWN_VARIANTS = ("thumbnail", "crop", "optimized") + + +async def s3_call(store: ObjectStore, deadline: float, fn, *args): + """Вызов S3 из запроса с жёстким общим дедлайном. + + connect/read timeout boto3 не покрывают DNS: в Docker остановленный + контейнер s3 пропадает из DNS, и getaddrinfo висит секундами (замер на + Docker Desktop: 10–13 с, у скачивания это давало 504 от HAProxy). Дедлайн + ограничивает всё вместе; по истечении — 503 и открытый circuit breaker. + Поток с зависшим вызовом доработает в фоне; для PUT это может дать + объект-сироту, его уберёт сборщик (src/reconcile.py, проход B). + """ + try: + return await asyncio.wait_for(run_in_threadpool(fn, *args), timeout=deadline) + except asyncio.TimeoutError: + store.breaker.failure() + log.warning("s3 call exceeded %.1fs deadline", deadline) + raise ObjectStoreUnavailable() + + +def objects_of(request: Request) -> ObjectStore: + store = request.app.state.objects + if store is None: + raise ApiError(503, "unavailable", "Attachment storage is not available") + return store + + +def s3_unavailable() -> ApiError: + return ApiError(503, "unavailable", "Attachment storage is temporarily unavailable", + {"Retry-After": "2"}) + + +def validate_filename(name: Optional[str]) -> str: + """Отображаемое имя: 1–255 code points, без /, \\, NUL и управляющих символов. + + Имя никогда не становится путём или ключом S3; ограничения защищают + клиентов, которые покажут/сохранят файл под этим именем. + """ + if name is None or not name.strip() or len(name) > 255: + raise bad_request("Invalid filename") + if name in (".", ".."): + raise bad_request("Invalid filename") + for ch in name: + if ch in "/\\\x00" or unicodedata.category(ch) in ("Cc", "Cs") \ + or ch in "

‮‭‪‫‬⁦⁧⁨⁩": + raise bad_request("Invalid filename") + return name + + +def upload_media_type(request: Request) -> str: + raw = request.headers.get("content-type", "") + media = raw.split(";", 1)[0].strip().lower() + if media not in ALLOWED_UPLOAD_TYPES: + raise ApiError(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted") + return media + + +def declared_length(request: Request) -> Optional[int]: + raw = request.headers.get("content-length") + if raw is None: + return None + if not raw.strip().isdigit(): + raise bad_request("Invalid Content-Length") + return int(raw) + + +async def drain(request: Request, limit: int) -> None: + """Дочитать и выбросить тело (не больше limit байт), чтобы клиент увидел 413, + а не обрыв соединения посреди отправки. Память — O(chunk).""" + seen = 0 + try: + async for chunk in request.stream(): + seen += len(chunk) + if seen > limit: + break + except ClientDisconnect: + pass + + +async def read_limited_body(request: Request, limit: int): + """Потоковое чтение тела кусками; не больше limit байт в памяти. + + Возвращает (bytes, md5, sha256). Превышение лимита -> 413 сразу, как только + счётчик перешёл limit (остаток тела отбрасывается, не накапливается). + """ + chunks = [] + size = 0 + md5 = hashlib.md5(usedforsecurity=False) + sha = hashlib.sha256() + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > limit: + chunks.clear() + await drain(request, limit) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % limit) + chunks.append(chunk) + md5.update(chunk) + sha.update(chunk) + # Одна склейка в bytes точного размера; список кусков сразу освобождается. + # Дальше bytes не копируются: BytesIO(bytes) и memoryview разделяют буфер. + data = b"".join(chunks) + chunks.clear() + return data, md5.digest(), sha.hexdigest() + + +def _validate_image(request: Request, data: bytes, declared: str): + settings = request.app.state.settings + with request.app.state.decode_slots: # потолок одновременных разборов на реплику + return inspect_image(data, declared, settings.max_image_pixels) + + +def _inject(settings, point: str) -> None: + """Внедрение отказов — ТОЛЬКО для экземпляра api-faults (профиль compose faults).""" + fault = settings.fault_injection + if not fault: + return + if fault == "crash_after_s3_put" and point == "after_put": + log.error("FAULT_INJECTION: process exit between S3 PUT and DB COMMIT") + os._exit(70) + if fault == "db_error_after_s3_put" and point == "after_put": + raise psycopg.OperationalError("injected: database failure after S3 PUT") + if fault == "s3_error_on_put" and point == "before_put": + raise ObjectStoreUnavailable() + + +@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202) +async def upload_attachment( + request: Request, + background: BackgroundTasks, + chat_id: UUID, + filename: Optional[str] = Query(default=None), + user: User = Depends(current_user), +) -> Attachment: + """lab5: приём оригинала -> 202 queued; обработку выполняет worker. + + Порядок проверок (как в lab4): аутентификация -> filename (400) -> + Content-Type (415) -> Content-Length (413 без чтения тела) -> членство (404) -> + слот загрузки (503) -> потоковое чтение с лимитом (413) -> содержимое (415/413). + Затем durable handoff: + [1] INSERT attachments(state=pending, staging_key) COMMIT + [2] S3 PUT staging/ (Content-MD5, SHA-256 в метаданных) + [3] UPDATE pending->queued + INSERT attachment_jobs + job_outbox COMMIT -> 202 + В момент 202 устойчиво сохранены: все байты оригинала (S3 staging), метаданные и + намерение обработать (задание + outbox в PostgreSQL). Публикация в брокер — после + ответа (background task) и/или sweeper'ом; остановка API после 202 работу не теряет. + """ + settings = request.app.state.settings + storage = storage_of(request) + name = validate_filename(filename) + declared = upload_media_type(request) + length = declared_length(request) + if length is not None and length > settings.max_upload_bytes: + if length <= 2 * settings.max_upload_bytes: + await drain(request, length) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % settings.max_upload_bytes, + {"Connection": "close"} if length > 2 * settings.max_upload_bytes else None) + if not await run_in_threadpool(storage.is_chat_member, chat_id, user.id): + raise not_found() + store = objects_of(request) + + slots: asyncio.Semaphore = request.app.state.upload_slots + try: + await asyncio.wait_for(slots.acquire(), timeout=settings.upload_queue_timeout_seconds) + except asyncio.TimeoutError: + raise ApiError(503, "unavailable", "Too many concurrent uploads", {"Retry-After": "1"}) + try: + try: + data, md5_digest, sha256_hex = await asyncio.wait_for( + read_limited_body(request, settings.max_upload_bytes), + timeout=settings.upload_read_timeout_seconds, + ) + except asyncio.TimeoutError: + raise bad_request("Upload was not completed in time") + except ClientDisconnect: + # Клиент оборвал загрузку: в БД и S3 ничего не записано. + log.info("upload aborted by client before completion") + raise bad_request("Upload was interrupted") + + try: + content_type, width, height = await run_in_threadpool(_validate_image, request, data, declared) + except ImageRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) + + attachment_id = uuid_mod.uuid4() + job_id = uuid_mod.uuid4() + # Ключи назначает сервер: не связаны с filename. staging — случайный; + # конечный ключ оригинала детерминирован от id (идемпотентная публикация worker'ом). + staging_key = "%s%s" % (STAGING_PREFIX, uuid_mod.uuid4().hex) + final_key = "final/%s/original" % attachment_id + # [1] намерение + created = await run_in_threadpool( + storage.create_pending_attachment, attachment_id, chat_id, user.id, name, + content_type, len(data), sha256_hex, width, height, final_key, staging_key, + settings.pipeline_version, + ) + if not created: + raise not_found() + # [2] байты в устойчивый staging + try: + _inject(settings, "before_put") + await s3_call(store, settings.s3_upload_deadline_seconds, + store.put, staging_key, data, content_type, md5_digest, sha256_hex) + except ObjectStoreUnavailable: + await _compensate(storage, store, attachment_id, None) + raise s3_unavailable() + finally: + del data + # [3] задание + outbox одной транзакцией + try: + _inject(settings, "after_put") + accepted = await run_in_threadpool( + storage.accept_attachment, attachment_id, job_id, + job_payload(job_id, attachment_id, settings.pipeline_version, 1), + settings.job_max_attempts, settings.relay_fast_path_grace_seconds, + ) + except (psycopg.Error, PoolTimeout): + await _compensate(storage, store, attachment_id, staging_key) + raise + if accepted is None: + # Строку уже забрал сборщик (загрузка шла дольше grace) — объект лишний. + await _compensate(storage, store, attachment_id, staging_key) + raise ApiError(503, "unavailable", "Upload expired, please retry") + attachment, outbox_id = accepted + relay = request.app.state.relay + if relay is not None: + # Выполняется ПОСЛЕ отправки ответа 202. + background.add_task(relay.publish_now, outbox_id) + return attachment + finally: + slots.release() + + +async def _compensate(storage: Storage, store: ObjectStore, attachment_id: UUID, object_key: Optional[str]) -> None: + """Best-effort откат в рамках запроса. Если не удалось — уберёт сборщик.""" + if object_key is not None: + try: + await s3_call(store, 5.0, store.delete, object_key) + except Exception as exc: + log.warning("compensation: object delete deferred to gc (%s)", type(exc).__name__) + try: + await run_in_threadpool(storage.delete_pending_attachment, attachment_id) + except Exception as exc: + log.warning("compensation: pending row left for gc (%s)", type(exc).__name__) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID, user: User = Depends(current_user)) -> Attachment: + found = storage_of(request).get_attachment_for_member(attachment_id, user.id) + if found is None: + raise not_found() + return found[0] + + +VARIANT_CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} + + +def content_disposition(filename: str) -> str: + from urllib.parse import quote + ascii_name = "".join(c if 32 <= ord(c) < 127 and c not in ('"', "\\", ";") else "_" for c in filename) + return "inline; filename=\"%s\"; filename*=UTF-8''%s" % (ascii_name, quote(filename, safe="")) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + request: Request, + attachment_id: UUID, + variant: Optional[str] = Query(default=None), + user: User = Depends(current_user), +): + """Байты оригинала или варианта через авторизованный API (без redirect), потоком из S3. + + lab5: queued/processing/failed -> 409 conflict; у ready нет такого варианта -> 404. + """ + if variant is not None and variant != "original" and variant not in KNOWN_VARIANTS: + raise bad_request("Invalid variant") + storage = storage_of(request) + found = await run_in_threadpool(storage.get_attachment_for_member, attachment_id, user.id) + if found is None: + raise not_found() + attachment, object_key, pipeline_version = found + if attachment.status != "ready": + raise ApiError(409, "conflict", "Attachment is not ready (status: %s)" % attachment.status) + if variant in KNOWN_VARIANTS: + meta = await run_in_threadpool(storage.variant_object, attachment_id, pipeline_version, variant) + if meta is None: + raise not_found() + object_key, media_type, size_bytes, etag = meta + disposition_name = "%s-%s.%s" % (variant, attachment.filename.rsplit(".", 1)[0][:200], + "webp" if media_type == "image/webp" else "jpg") + else: + media_type, size_bytes, etag = attachment.content_type, attachment.size_bytes, attachment.sha256 + disposition_name = attachment.filename + store = objects_of(request) + try: + chunks, length = await s3_call(store, request.app.state.settings.s3_download_deadline_seconds, + store.open, object_key) + except ObjectStoreUnavailable: + raise s3_unavailable() + except ObjectMissing: + log.error("ready attachment %s has no object in S3 (see python -m src.reconcile)", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + if length != size_bytes: + chunks.close() # закрыть поток S3 + log.error("attachment %s: object size differs from metadata", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + return StreamingResponse( + chunks, + media_type=media_type, + headers={ + "Content-Length": str(size_bytes), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": content_disposition(disposition_name), + "ETag": '"%s"' % etag, + }, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..0963b22 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,25 @@ +"""Однократный запуск миграций: python -m src.migrate + +lab3: это команда one-shot сервиса `migrate` в compose.yaml. Реплики API +миграции не применяют (RUN_MIGRATIONS_ON_START=false) и стартуют только после +успешного завершения `migrate` (depends_on: service_completed_successfully). +Повторный запуск ничего не меняет; одновременный запуск нескольких копий +сериализуется pg_advisory_lock (см. src/db.py). +""" +import logging + +from .config import load_settings +from .db import wait_and_migrate + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + applied = wait_and_migrate(load_settings()) + if applied: + print("applied migrations: " + ", ".join(applied)) + else: + print("schema is up to date; nothing applied") + + +if __name__ == "__main__": + main() diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..9794559 --- /dev/null +++ b/src/models.py @@ -0,0 +1,113 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class UserCreate(StrictModel): + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-z0-9_]{3,32}$", + ) + display_name: str = Field( + min_length=1, + max_length=100, + ) + # lab2: пароль обязателен; длина в Unicode code points (len() в Python). + password: str = Field( + min_length=12, + max_length=128, + ) + + +class User(StrictModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(StrictModel): + token: str + token_type: str = "Bearer" + expires_at: datetime + user: User + + +class ChatCreate(StrictModel): + title: str = Field( + min_length=1, + max_length=100, + ) + member_ids: List[UUID] = Field( + min_length=1, + max_length=100, + ) + + +class Chat(StrictModel): + id: UUID + title: str + member_ids: List[UUID] + created_at: datetime + + +class MessageCreate(StrictModel): + # lab4: text="" допустим, если есть хотя бы одно вложение (проверка в main.py). + text: str = Field( + min_length=0, + max_length=4000, + ) + # Отсутствующее поле == [] (контракт); значение по умолчанию попадает и в + # канонический JSON отпечатка идемпотентности. + attachment_ids: List[UUID] = Field(default_factory=list, max_length=10) + + +class Message(StrictModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: List[UUID] = Field(default_factory=list) + + +class Variant(StrictModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(StrictModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + # lab5: queued | processing | ready | failed (внутренний pending в API не виден). + status: str = "queued" + variants: List[Variant] = Field(default_factory=list) + error_code: Optional[str] = None + created_at: datetime + updated_at: datetime + + +class ChatPage(StrictModel): + items: List[Chat] + next_cursor: Optional[str] + + +class MessagePage(StrictModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/objectstore.py b/src/objectstore.py new file mode 100644 index 0000000..ca01012 --- /dev/null +++ b/src/objectstore.py @@ -0,0 +1,226 @@ +"""Приватное S3-совместимое хранилище оригиналов (lab4: SeaweedFS). + +API работает с отдельными credentials, у которых есть только Read/Write/List +на один бакет (см. infra/seaweedfs/README.md): создать/удалить бакет или +прочитать другой бакет этими ключами нельзя. Административные ключи есть +только у one-shot сервиса s3-init (src/s3_bootstrap.py). + +Все вызовы ограничены по времени (connect/read timeout, не более +S3_MAX_ATTEMPTS попыток), поэтому недоступность S3 превращается в быстрый 503 +для операций с вложениями, а не в зависший запрос. +""" +import base64 +import hashlib +import io +import logging +import threading +import time +from datetime import datetime +from typing import Iterator, List, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Settings + +log = logging.getLogger("messenger.s3") + +CHUNK = 64 * 1024 + + +class ObjectStoreUnavailable(Exception): + """S3 недоступен, таймаут или 5xx -> 503 для клиента.""" + + +class ObjectMissing(Exception): + """Объекта нет в бакете (404/NoSuchKey).""" + + +def _code(exc: ClientError) -> str: + return str(exc.response.get("Error", {}).get("Code", "")) + + +def _status(exc: ClientError) -> int: + return int(exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) or 0) + + +def make_client(endpoint: str, region: str, access_key: str, secret_key: str, + connect_timeout: float = 2.0, read_timeout: float = 10.0, max_attempts: int = 2): + config = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"max_attempts": max_attempts, "mode": "standard"}, + # Соединения на реплику: не больше, чем одновременных загрузок + скачиваний. + max_pool_connections=32, + ) + return boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=config, + ) + + +class CircuitBreaker: + """После отказа S3 следующие open_seconds операции сразу получают + ObjectStoreUnavailable, не тратя время на DNS/connect/повторы. Затем — + одна пробная попытка (half-open): успех закрывает, отказ снова открывает.""" + + def __init__(self, open_seconds: float) -> None: + self.open_seconds = open_seconds + self._lock = threading.Lock() + self._open_until = 0.0 + + def check(self) -> None: + with self._lock: + now = time.monotonic() + if now < self._open_until: + raise ObjectStoreUnavailable() + if self._open_until: + # half-open: пропускаем одну попытку, остальные пока отклоняем + self._open_until = now + self.open_seconds + + def failure(self) -> None: + with self._lock: + if not self._open_until or time.monotonic() >= self._open_until - self.open_seconds: + log.warning("s3 circuit opened for %.1fs", self.open_seconds) + self._open_until = time.monotonic() + self.open_seconds + + def success(self) -> None: + with self._lock: + self._open_until = 0.0 + + @property + def is_open(self) -> bool: + return time.monotonic() < self._open_until + + +class ObjectStore: + def __init__(self, settings: Settings) -> None: + self.bucket = settings.s3_bucket + self.breaker = CircuitBreaker(settings.s3_circuit_open_seconds) + self.client = make_client( + settings.s3_endpoint, + settings.s3_region, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_connect_timeout_seconds, + settings.s3_read_timeout_seconds, + settings.s3_max_attempts, + ) + + @staticmethod + def configured(settings: Settings) -> bool: + return bool(settings.s3_endpoint and settings.s3_access_key_id and settings.s3_secret_access_key) + + def _wrap(self, exc: Exception, op: str) -> Exception: + if isinstance(exc, ClientError): + code, status = _code(exc), _status(exc) + if status == 404 or code in ("NoSuchKey", "NotFound", "404"): + self.breaker.success() # S3 ответил — он доступен + return ObjectMissing() + log.warning("s3 %s failed: %s/%s", op, status, code) + else: + log.warning("s3 %s failed: %s", op, type(exc).__name__) + self.breaker.failure() + return ObjectStoreUnavailable() + + def put(self, key: str, data, content_type: str, md5_digest: bytes, sha256_hex: str) -> None: + """PUT с Content-MD5: S3 сам отклонит повреждённую по пути передачу (BadDigest). + + Тело передаётся как BytesIO поверх bytes (без копирования): botocore + отправляет файловый объект блоками. С bytes он склеивает заголовки и тело + в один буфер — лишняя копия размером с файл на каждую загрузку (замер в REPORT.md). + """ + self.breaker.check() + try: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=io.BytesIO(data), + ContentType=content_type, + ContentMD5=base64.b64encode(md5_digest).decode("ascii"), + Metadata={"sha256": sha256_hex}, + ) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "put") + self.breaker.success() + + def open(self, key: str) -> Tuple[Iterator[bytes], int]: + """Потоковое чтение: (итератор кусков по 64 KiB, длина). Память — O(chunk).""" + self.breaker.check() + try: + obj = self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "get") + self.breaker.success() + body = obj["Body"] + + def chunks() -> Iterator[bytes]: + try: + for chunk in body.iter_chunks(CHUNK): + yield chunk + finally: + body.close() + + return chunks(), int(obj.get("ContentLength", -1)) + + def read_all(self, key: str, limit: int) -> bytes: + """Для сверки/тестов: прочитать объект целиком (не больше limit байт).""" + it, _ = self.open(key) + out = bytearray() + for chunk in it: + out += chunk + if len(out) > limit: + raise ValueError("object exceeds limit") + return bytes(out) + + def delete(self, key: str) -> None: + """Идемпотентно: удаление отсутствующего объекта — не ошибка.""" + self.breaker.check() + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "delete") + if not isinstance(err, ObjectMissing): + raise err + + def head(self, key: str) -> Optional[dict]: + self.breaker.check() + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "head") + if isinstance(err, ObjectMissing): + return None + raise err + + def list_page(self, prefix: str, token: Optional[str] = None, + page_size: int = 1000) -> Tuple[List[Tuple[str, datetime, int]], Optional[str]]: + kwargs = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size} + if token: + kwargs["ContinuationToken"] = token + self.breaker.check() + try: + resp = self.client.list_objects_v2(**kwargs) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "list") + items = [(o["Key"], o["LastModified"], int(o.get("Size", 0))) for o in resp.get("Contents", [])] + next_token = resp.get("NextContinuationToken") if resp.get("IsTruncated") else None + return items, next_token + + def ping(self) -> bool: + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except (ClientError, BotoCoreError): + return False + + +def md5_and_sha256(data: bytes) -> Tuple[bytes, str]: + return hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest() diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 0000000..53635c6 --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,187 @@ +"""Transactional outbox relay (lab5): PostgreSQL job_outbox -> брокер. + +Почему нет разрыва «COMMIT в БД прошёл, а publish не случился»: +API пишет задание и строку outbox в ОДНОЙ транзакции с переводом вложения в +queued. Публикация — отдельный шаг, который можно безопасно повторять: + + быстрый путь API сразу после ответа 202 (background task) публикует свою строку: + SELECT ... FOR UPDATE SKIP LOCKED -> XADD -> DELETE -> COMMIT. + sweeper Каждый процесс API и worker раз в RELAY_POLL_INTERVAL_SECONDS + публикует строки с available_at <= now(). У свежей строки + available_at = created_at + RELAY_FAST_PATH_GRACE_SECONDS, чтобы + не соревноваться с быстрым путём; у retry — время следующей попытки. + +Окна отказа: + * процесс умер после COMMIT до XADD -> строка осталась -> sweeper опубликует; + * умер после XADD до DELETE/COMMIT -> строка осталась -> повторная публикация, + дубль в брокере; worker идемпотентен (redelivery_noop); + * брокер недоступен -> ROLLBACK, publish_failures+1, повтор позже. +Строка блокируется FOR UPDATE SKIP LOCKED: несколько relay публикуют разные строки. +Успешный XADD = подтверждение приёма брокером (AOF appendfsync always). +""" +import logging +import os +import threading +import time +from typing import Optional + +import psycopg +from psycopg_pool import ConnectionPool, PoolTimeout + +from opentelemetry.trace import SpanKind, Status, StatusCode + +from . import telemetry +from .broker import Broker, BrokerUnavailable +from .jobs import JobStore + +log = logging.getLogger("messenger.outbox") + + +class Relay: + def __init__(self, pool: ConnectionPool, broker: Broker, settings, actor: str) -> None: + self.pool = pool + self.broker = broker + self.settings = settings + self.actor = actor + self.jobs = JobStore(pool, settings) + self.fault = settings.fault_injection + self.published = 0 + + def _publish_rows(self, conn, rows, fast: bool = False) -> int: + sent = [] + for outbox_id, job_id, payload in rows: + stream_id = self._publish_traced(payload) # BrokerUnavailable -> ROLLBACK всего пакета + sent.append((outbox_id, job_id, payload, stream_id)) + if fast and self.fault == "crash_after_publish": + log.error("FAULT_INJECTION: relay exits after XADD %s before outbox DELETE/COMMIT", stream_id) + os._exit(70) + conn.execute("DELETE FROM job_outbox WHERE id = ANY(%s)", ([s[0] for s in sent],)) + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) " + "VALUES (%s, %s, %s, 'published', %s, %s)", + [(job_id, payload.get("attachment_id"), payload.get("attempt"), "stream id " + stream_id, self.actor) + for _oid, job_id, payload, stream_id in sent], + ) + self.published += len(sent) + return len(sent) + + def _publish_traced(self, payload: dict) -> str: + """lab6: PRODUCER span — ребёнок span'а, сохранённого в строке outbox + (HTTP upload или consumer при retry). В брокер уходит traceparent уже + этого producer span: цепочка upload -> publish -> process.""" + parent = telemetry.extract(payload) + with telemetry.tracer().start_as_current_span( + "%s publish" % self.broker.stream, context=parent, kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "redis", + "messaging.operation.type": "publish", + "messaging.destination.name": self.broker.stream, + "messenger.job.type": payload.get("type", ""), + "messenger.job.attempt": int(payload.get("attempt") or 0), + "messenger.relay.actor": self.actor.split(":", 1)[0], + }, + ) as span: + message = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")} + telemetry.inject(message) + try: + stream_id = self.broker.publish(message) + except BrokerUnavailable as exc: + span.set_status(Status(StatusCode.ERROR, "broker unavailable")) + span.record_exception(exc) + raise + span.set_attribute("messaging.message.id", stream_id) + telemetry.INSTRUMENTS.published.add(1, {"job.type": payload.get("type", "")}) + return stream_id + + def publish_now(self, outbox_id: int) -> bool: + """Быстрый путь для одной строки (вызывается после ответа 202). Ошибки не + пробрасываются: строку опубликует sweeper.""" + try: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + "SELECT id, job_id, payload FROM job_outbox WHERE id = %s FOR UPDATE SKIP LOCKED", + (outbox_id,), + ).fetchall() + if not rows: + return False + if self.fault == "crash_after_commit": + # Внедрённый разрыв: COMMIT задания и outbox уже есть, 202 отправлен, + # процесс умирает до XADD. Восстановление — sweeper другого процесса. + self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected", + "crash between COMMIT and publish", self.actor) + log.error("FAULT_INJECTION: process exit between COMMIT and publish (outbox %s)", outbox_id) + os._exit(70) + if self.fault == "crash_after_publish": + self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected", + "crash after XADD before outbox commit", self.actor) + return self._publish_rows(conn, rows, fast=True) > 0 + except (BrokerUnavailable, psycopg.Error, PoolTimeout) as exc: + log.warning("fast publish deferred to sweeper: %s", type(exc).__name__) + return False + + def sweep_once(self) -> int: + """Опубликовать все готовые строки пакетами; вернуть число опубликованных.""" + total = 0 + while True: + with self.pool.connection() as conn: + try: + with conn.transaction(): + rows = conn.execute( + """ + SELECT id, job_id, payload FROM job_outbox + WHERE available_at <= clock_timestamp() + ORDER BY available_at, id + LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (self.settings.relay_batch,), + ).fetchall() + if not rows: + return total + total += self._publish_rows(conn, rows) + except BrokerUnavailable: + # Транзакция откатилась (строки целы). Отметим неудачу отдельно. + with conn.transaction(): + conn.execute( + "UPDATE job_outbox SET publish_failures = publish_failures + 1 " + "WHERE id = ANY(%s)", ([r[0] for r in rows],)) + raise + if len(rows) < self.settings.relay_batch: + return total + + def loop(self, stop: threading.Event, wake: Optional[threading.Event] = None) -> None: + """Фоновый цикл: sweeper + сборщик просроченных аренд.""" + interval = self.settings.relay_poll_interval_seconds + last_reap = 0.0 + broker_down_logged = False + while not stop.is_set(): + try: + n = self.sweep_once() + if n: + log.info("relay %s: published %d outbox rows", self.actor, n) + broker_down_logged = False + if time.monotonic() - last_reap >= 5: + last_reap = time.monotonic() + for job_id, outcome in self.jobs.reap_expired(self.actor): + log.warning("lease expired: job %s %s", job_id, outcome) + except BrokerUnavailable: + if not broker_down_logged: + log.warning("relay %s: broker unavailable; outbox rows are kept", self.actor) + broker_down_logged = True + except (psycopg.Error, PoolTimeout) as exc: + log.warning("relay %s: database unavailable (%s)", self.actor, type(exc).__name__) + except Exception: + log.exception("relay %s: unexpected error", self.actor) + if wake is not None: + wake.wait(interval) + wake.clear() + else: + stop.wait(interval) + + +def start_relay_thread(pool, broker, settings, actor: str, stop: threading.Event) -> Relay: + relay = Relay(pool, broker, settings, actor) + threading.Thread(target=relay.loop, args=(stop,), name="outbox-relay", daemon=True).start() + return relay diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..6396928 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,295 @@ +"""Согласованность PostgreSQL (метаданные) и S3 (байты): сборка мусора и сверка. + +lab5: префиксы бакета + staging/ — принятый оригинал до обработки (API); + final//original — опубликованный оригинал (worker); + final//p<версия>/<вариант>. — варианты (worker); + originals/ — оригиналы lab4 (до перехода на lab5). +Строка pending ссылается на staging_key (объект появляется ПОСЛЕ строки). +Добавлен проход D — очистка staging: ready-вложения (staging больше не нужен, +результат опубликован) и failed старше STAGING_RETENTION_SECONDS (после этого +replay невозможен — нужен повторный upload). Текущую работу (queued/processing) +сборщик не трогает никогда. + +Почему неучтённых объектов «навсегда» не бывает (подробно — REPORT.md): + + загрузка: [1] INSERT attachments(state=pending, object_key) COMMIT + [2] S3 PUT object_key + [3] UPDATE state=ready WHERE state=pending COMMIT -> 201 + Ключ объекта записан в БД ДО того, как объект появится в бакете. + + Сбой/обрыв между шагами: + после [1] -> строка pending без объекта; + после [2] -> строка pending + объект (S3 прошёл, COMMIT [3] — нет); + ошибка S3 в [2] -> API сам компенсирует: DELETE pending; + ошибка БД в [3] -> API сам компенсирует: DELETE object (pending уберёт GC). + Процесс может умереть в любой точке, поэтому компенсация в запросе — + оптимизация, а гарантию даёт сборщик: + + A. stale pending: строки pending старше GRACE. Сначала строка удаляется + (DELETE ... WHERE state='pending' RETURNING object_key — атомарный захват: + если запрос всё-таки успел сделать [3], строка уже ready и не удаляется), + затем удаляется объект. Если удаление объекта не удалось — объект стал + «сиротой» и будет убран проходом B. + B. orphan objects: объекты в бакете (префикс originals/), для ключа которых + в БД нет строки, и LastModified старше GRACE. Строка всегда создаётся + раньше объекта и после удаления не возрождается (ключ одноразовый), + поэтому «нет строки» + «старше GRACE» = объект не принадлежит живой загрузке. + C. missing objects: строки ready без объекта (или с другим размером) — только + отчёт: восстановить байты нельзя, удалять метаданные автоматически опасно. + +GRACE (ATTACHMENT_GC_GRACE_SECONDS, по умолчанию 900 с, минимум 300 с) много +больше самой долгой загрузки: чтение тела <= UPLOAD_READ_TIMEOUT_SECONDS (60 с) ++ проверка + S3 PUT (connect 2 с + read 10 с) x S3_MAX_ATTEMPTS (2). +Строки ready и вложения, на которые ссылаются сообщения, сборщик не трогает +никогда (FK message_attachments -> attachments ON DELETE RESTRICT). + +CLI (по умолчанию dry-run, ничего не меняет): + python -m src.reconcile # отчёт + python -m src.reconcile --apply # исправить A и B + python -m src.reconcile --apply --grace-seconds 0 # только на учебном стенде +""" +import argparse +import json +import logging +import time +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional + +from psycopg_pool import ConnectionPool + +from .objectstore import ObjectStore, ObjectStoreUnavailable + +log = logging.getLogger("messenger.reconcile") + +OBJECT_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "final/" +SCANNED_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, OBJECT_PREFIX) +GC_LOCK_ID = 2026_0004 +BATCH = 500 + + +def _try_lock(conn) -> bool: + return conn.execute("SELECT pg_try_advisory_lock(%s)", (GC_LOCK_ID,)).fetchone()[0] + + +def _unlock(conn) -> None: + conn.execute("SELECT pg_advisory_unlock(%s)", (GC_LOCK_ID,)) + + +def collect_stale_pending(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict) -> None: + """Проход A. dry-run: только подсчёт; apply: строка -> затем объект.""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, COALESCE(staging_key, object_key) FROM attachments + WHERE state = 'pending' AND created_at < clock_timestamp() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + """, + (grace_seconds, BATCH), + ).fetchall() + report["stale_pending"] = len(rows) + report["stale_pending_ids"] = [str(r[0]) for r in rows[:20]] + if not apply: + return + deleted_rows = deleted_objects = 0 + for att_id, _key in rows: + with pool.connection() as conn: + claimed = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s AND state = 'pending' + AND created_at < clock_timestamp() - make_interval(secs => %s) + RETURNING COALESCE(staging_key, object_key) + """, + (att_id, grace_seconds), + ).fetchone() + if claimed is None: + continue # загрузка успела завершиться (ready) — не трогаем + deleted_rows += 1 + try: + store.delete(claimed[0]) + deleted_objects += 1 + except ObjectStoreUnavailable: + # Объект стал сиротой без строки; его уберёт проход B. + log.warning("gc: object delete deferred to orphan sweep") + report["stale_pending_rows_deleted"] = deleted_rows + report["stale_pending_object_deletes"] = deleted_objects + + +def known_keys(pool: ConnectionPool, keys: List[str]) -> set: + """Ключи, на которые ссылается БД: оригинал, staging или вариант.""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT object_key FROM attachments WHERE object_key = ANY(%s) + UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s) + UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s) + """, + (keys, keys, keys), + ).fetchall() + return {r[0] for r in rows} + + +def clean_staging(pool: ConnectionPool, store: ObjectStore, retention_seconds: int, + apply: bool, report: Dict) -> None: + """Проход D: staging, который больше не нужен. Строка обновляется ПОСЛЕ удаления + объекта; сбой между шагами безопасен (повторное удаление отсутствующего — no-op).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, staging_key, state FROM attachments + WHERE staging_key IS NOT NULL + AND (state = 'ready' + OR (state = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => %s))) + ORDER BY updated_at LIMIT %s + """, + (retention_seconds, BATCH), + ).fetchall() + report["staging_to_clean"] = len(rows) + if not apply: + return + cleaned = 0 + for att_id, key, state in rows: + try: + store.delete(key) + except ObjectStoreUnavailable: + log.warning("gc: staging delete deferred") + break + with pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND staging_key = %s " + "AND state = %s", (att_id, key, state)) + cleaned += 1 + report["staging_cleaned"] = cleaned + + +def sweep_orphans(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict, verify_ready: bool = True) -> None: + """Проходы B и C: полный обход префикса бакета страницами по 1000.""" + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + token: Optional[str] = None + listed = orphans = orphans_young = deleted = 0 + seen_keys: List[str] = [] + orphan_sample: List[str] = [] + started = time.monotonic() + for prefix in SCANNED_PREFIXES: + token = None + while True: + items, token = store.list_page(prefix, token) + listed += len(items) + if items: + keys = [k for k, _, _ in items] + known = known_keys(pool, keys) + for key, modified, _size in items: + if key in known: + if verify_ready: + seen_keys.append(key) + continue + orphans += 1 + if modified > cutoff: + orphans_young += 1 # возможно, живая загрузка/обработка или ещё не истёк grace + continue + if len(orphan_sample) < 20: + orphan_sample.append(key) + if apply and not known_keys(pool, [key]): + # Повторная проверка непосредственно перед удалением. + store.delete(key) + deleted += 1 + if not token: + break + report["objects_listed"] = listed + report["orphan_objects"] = orphans + report["orphan_objects_within_grace"] = orphans_young + report["orphan_objects_sample"] = orphan_sample + if apply: + report["orphan_objects_deleted"] = deleted + report["bucket_scan_seconds"] = round(time.monotonic() - started, 3) + + if verify_ready: + # C: ready-строки, для которых объекта нет в листинге (или не тот размер). + present = set(seen_keys) + with pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, size_bytes FROM attachments WHERE state = 'ready' " + "UNION ALL SELECT attachment_id, object_key, size_bytes FROM attachment_variants" + ).fetchall() + missing = [str(r[0]) for r in rows if r[1] not in present] + report["ready_rows"] = len(rows) + report["missing_objects"] = len(missing) + report["missing_object_ids"] = missing[:20] + + +def run_once(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, apply: bool, + orphan_sweep: bool = True, staging_retention_seconds: int = 86400) -> Optional[Dict]: + """Один проход под advisory lock. None — сейчас работает другая реплика.""" + report: Dict = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds} + with pool.connection() as lock_conn: + lock_conn.autocommit = True + if not _try_lock(lock_conn): + return None + try: + collect_stale_pending(pool, store, grace_seconds, apply, report) + clean_staging(pool, store, staging_retention_seconds, apply, report) + if orphan_sweep: + sweep_orphans(pool, store, grace_seconds, apply, report) + finally: + _unlock(lock_conn) + # Соединение вернётся в общий пул — режим транзакций как у остальных. + lock_conn.autocommit = False + return report + + +def gc_loop(pool: ConnectionPool, store: ObjectStore, settings, stop) -> None: + """Фон в каждой реплике: проход A каждые N с, проход B — реже. Lock — одна реплика.""" + last_sweep = time.monotonic() + while not stop.wait(settings.attachment_gc_interval_seconds): + try: + sweep = time.monotonic() - last_sweep >= settings.orphan_sweep_interval_seconds + report = run_once(pool, store, settings.attachment_gc_grace_seconds, apply=True, + orphan_sweep=sweep, staging_retention_seconds=settings.staging_retention_seconds) + if report is None: + continue + if sweep: + last_sweep = time.monotonic() + if report.get("stale_pending") or report.get("orphan_objects_deleted") or report.get("missing_objects") \ + or report.get("staging_cleaned"): + log.info("attachment gc: %s", json.dumps(report, ensure_ascii=False)) + except Exception as exc: # БД/S3 недоступны — следующий проход повторит + log.warning("attachment gc skipped: %s", type(exc).__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reconcile attachments metadata (PostgreSQL) with S3 objects") + parser.add_argument("--apply", action="store_true", help="delete stale pending rows and orphan objects") + parser.add_argument("--grace-seconds", type=int, default=None, + help="override ATTACHMENT_GC_GRACE_SECONDS (values < 300 only for lab tests)") + parser.add_argument("--no-orphan-sweep", action="store_true", help="skip bucket listing") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + + from .config import load_settings + from .db import create_pool + + settings = load_settings() + if not ObjectStore.configured(settings): + raise SystemExit("S3 is not configured") + grace = settings.attachment_gc_grace_seconds if args.grace_seconds is None else args.grace_seconds + pool = create_pool(settings) + pool.wait(timeout=15) + store = ObjectStore(settings) + try: + started = time.monotonic() + report = run_once(pool, store, grace, args.apply, orphan_sweep=not args.no_orphan_sweep, + staging_retention_seconds=settings.staging_retention_seconds) + if report is None: + raise SystemExit("another reconciler holds the lock; retry later") + report["total_seconds"] = round(time.monotonic() - started, 3) + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/src/runtime.py b/src/runtime.py new file mode 100644 index 0000000..57904cb --- /dev/null +++ b/src/runtime.py @@ -0,0 +1,39 @@ +"""Состояние процесса, общее для middleware, readiness и сервера. + +INSTANCE_ID — значение X-Instance-Id. По умолчанию hostname контейнера: +Docker задаёт его равным короткому ID контейнера, поэтому scripts/ops.py +однозначно сопоставляет заголовок конкретному контейнеру (stop-one). +Значение не меняется в течение жизни процесса. + +DRAIN — фаза «осушения» после SIGTERM (см. src/server.py): процесс ещё +обслуживает запросы, но /health/ready уже отвечает 503, и балансировщик +выводит реплику из ротации до того, как она перестанет принимать соединения. +""" +import os +import socket +import threading +import time +from typing import Optional + +INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + + +class Drain: + def __init__(self) -> None: + self._event = threading.Event() + self.started_at: Optional[float] = None + + def begin(self) -> bool: + """True, если осушение началось этим вызовом.""" + if self._event.is_set(): + return False + self.started_at = time.monotonic() + self._event.set() + return True + + @property + def active(self) -> bool: + return self._event.is_set() + + +DRAIN = Drain() diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 0000000..0f83209 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,226 @@ +"""Bootstrap и проверка приватного бакета: python -m src.s3_bootstrap + +Запускается one-shot сервисом `s3-init` до старта API (depends_on +service_completed_successfully). Идемпотентно: повторный запуск не меняет +данные — BucketAlreadyOwnedByYou считается успехом. + +Проверки (любое нарушение -> exit 1, API не стартует): + * API-ключи могут PUT/GET/DELETE в своём бакете (проба в префиксе _bootstrap/, + который сборщик мусора не обходит); + * API-ключи НЕ могут создать бакет (нет прав администратора); + * анонимный GET объекта и анонимный листинг бакета -> 403; + * административные HTTP-порты SeaweedFS (master 9333, filer 8888, + volume 8080) не слушают в общей сети — доступен только S3 API. + + python -m src.s3_bootstrap # s3-init: создать бакет + проверки + python -m src.s3_bootstrap --check-only # только проверки (без admin-ключей) + python -m src.s3_bootstrap --check-only --sample-from-db + # + анонимный GET реального объекта последнего ready-вложения -> 403, + # GET тем же ключом с API-credentials -> 200 и совпадает SHA-256. + python -m src.s3_bootstrap --wait-ready 90 + # прогрев после рестарта: ждать, пока запись/чтение пробного объекта и + # чтение СУЩЕСТВУЮЩЕГО объекта занимают < 2 с. SeaweedFS отвечает /healthz + # сразу, но первые чтение/запись после рестарта блокируются на 15–30 с + # (выборы Raft-лидера master, heartbeat volume-сервера; замер в REPORT.md). +Значения ключей не печатаются. +""" +import argparse +import hashlib +import json +import os +import socket +import sys +import time +import urllib.error +import urllib.request +import uuid +from urllib.parse import quote, urlparse + +from botocore.exceptions import BotoCoreError, ClientError + +from .config import _read_secret +from .objectstore import make_client + +ADMIN_PORTS = (9333, 8888, 8080, 19333, 18888) + + +def anonymous_status(url: str) -> int: + req = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def port_open(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=1.5): + return True + except OSError: + return False + + +def wait_ready(timeout: float) -> int: + """Ждать, пока S3 быстро (< 2 с на операцию) выполняет запись, чтение и удаление + пробного объекта и чтение реального (самого старого ready) объекта, если он есть.""" + import psycopg + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + client = make_client(endpoint, os.environ.get("S3_REGION", "us-east-1"), _read_secret("S3_ACCESS_KEY_ID"), + _read_secret("S3_SECRET_ACCESS_KEY"), 1, 2, 1) + started = time.monotonic() + key = None + looked_up = False + while time.monotonic() - started < timeout: + try: + if not looked_up and _read_secret("DATABASE_URL"): + with psycopg.connect(_read_secret("DATABASE_URL"), connect_timeout=3) as conn: + row = conn.execute("SELECT object_key FROM attachments WHERE state = 'ready' " + "ORDER BY created_at LIMIT 1").fetchone() + key = row[0] if row else None + looked_up = True + t = time.monotonic() + probe = "_bootstrap/warmup-%s" % uuid.uuid4().hex + client.put_object(Bucket=bucket, Key=probe, Body=b"warmup") + client.get_object(Bucket=bucket, Key=probe)["Body"].read() + client.delete_object(Bucket=bucket, Key=probe) + if key: + client.get_object(Bucket=bucket, Key=key)["Body"].read() + print(json.dumps({"s3_ready_after_s": round(time.monotonic() - started, 2), + "last_cycle_s": round(time.monotonic() - t, 3), + "existing_object_read": bool(key)})) + return 0 + except (ClientError, BotoCoreError, OSError, psycopg.Error): + time.sleep(0.5) + print("S3 did not become ready for reads/writes within %.0fs" % timeout, file=sys.stderr) + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--sample-from-db", action="store_true") + parser.add_argument("--wait-ready", type=float, default=0.0, metavar="SECONDS") + args = parser.parse_args() + if args.wait_ready: + return wait_ready(args.wait_ready) + + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + wait_seconds = int(os.environ.get("S3_BOOTSTRAP_WAIT_SECONDS", "90")) + report = {"endpoint_host": urlparse(endpoint).hostname, "bucket": bucket, "checks": []} + ok = True + + def check(name, condition, detail=""): + nonlocal ok + ok = ok and bool(condition) + report["checks"].append({"check": name, "ok": bool(condition), "detail": detail}) + + api_key = _read_secret("S3_ACCESS_KEY_ID") + api_secret = _read_secret("S3_SECRET_ACCESS_KEY") + if not (api_key and api_secret): + print("S3 API credentials are not configured", file=sys.stderr) + return 2 + api = make_client(endpoint, region, api_key, api_secret, 2, 10, 2) + + if not args.check_only: + admin_key = _read_secret("S3_ADMIN_ACCESS_KEY_ID") + admin_secret = _read_secret("S3_ADMIN_SECRET_ACCESS_KEY") + if not (admin_key and admin_secret): + print("S3 admin credentials are not configured", file=sys.stderr) + return 2 + admin = make_client(endpoint, region, admin_key, admin_secret, 2, 10, 1) + deadline = time.monotonic() + wait_seconds + while True: + try: + admin.list_buckets() + break + except (ClientError, BotoCoreError) as exc: + if time.monotonic() > deadline: + print("S3 is not reachable: %s" % type(exc).__name__, file=sys.stderr) + return 1 + time.sleep(1) + try: + admin.create_bucket(Bucket=bucket) + report["bucket_action"] = "created" + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + report["bucket_action"] = "already exists (idempotent no-op)" + else: + print("create_bucket failed: %s" % code, file=sys.stderr) + return 1 + + # Проба API-ключами. + probe_key = "_bootstrap/probe-%s" % uuid.uuid4().hex + body = b"probe " + uuid.uuid4().bytes + try: + api.put_object(Bucket=bucket, Key=probe_key, Body=body) + got = api.get_object(Bucket=bucket, Key=probe_key)["Body"].read() + check("api credentials: put/get own bucket", got == body) + except (ClientError, BotoCoreError) as exc: + check("api credentials: put/get own bucket", False, type(exc).__name__) + got = None + + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(probe_key))) + check("anonymous GET object is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status("%s/%s?list-type=2" % (endpoint, bucket)) + check("anonymous list bucket is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status(endpoint + "/") + check("anonymous list buckets is denied", status in (401, 403), "HTTP %d" % status) + + other = "forbidden-%s" % uuid.uuid4().hex[:12] + try: + api.create_bucket(Bucket=other) + check("api credentials cannot create buckets", False, "bucket was created") + try: + api.delete_bucket(Bucket=other) + except (ClientError, BotoCoreError): + pass + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot create buckets", code == "AccessDenied", code) + try: + api.delete_bucket(Bucket=bucket) + check("api credentials cannot delete the bucket", False, "bucket deleted") + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot delete the bucket", code in ("AccessDenied", "BucketNotEmpty"), code) + + try: + api.delete_object(Bucket=bucket, Key=probe_key) + except (ClientError, BotoCoreError): + pass + + host = urlparse(endpoint).hostname + exposed = [p for p in ADMIN_PORTS if port_open(host, p)] + check("SeaweedFS admin/UI ports are not reachable on the shared network", not exposed, + "open: %s" % exposed if exposed else "checked %s" % list(ADMIN_PORTS)) + + if args.sample_from_db: + import psycopg + dsn = _read_secret("DATABASE_URL") + with psycopg.connect(dsn, connect_timeout=5) as conn: + row = conn.execute( + "SELECT object_key, sha256 FROM attachments WHERE state = 'ready' " + "ORDER BY created_at DESC LIMIT 1" + ).fetchone() + if row is None: + check("sample ready attachment exists in DB", False, "upload an attachment first (make test)") + else: + key, sha = row + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(key))) + check("anonymous GET of a real attachment object is denied", status in (401, 403), "HTTP %d" % status) + data = api.get_object(Bucket=bucket, Key=key)["Body"].read() + check("same object is readable with API credentials and matches sha256", + hashlib.sha256(data).hexdigest() == sha, "%d bytes" % len(data)) + + report["ok"] = ok + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..6656463 --- /dev/null +++ b/src/security.py @@ -0,0 +1,97 @@ +"""Пароли, разбор Basic/Bearer и генерация токенов.""" +import base64 +import binascii +import hashlib +import re +import secrets +import threading +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +# Argon2id (библиотека argon2-cffi). Параметры — рекомендация OWASP для +# Argon2id: m=19 MiB, t=2, p=1. Пароль хешируется целиком в UTF-8, без усечения. +_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1) + +# Хеш-«пустышка»: проверка для несуществующего пользователя стоит столько же, +# сколько для существующего, — время ответа не выдаёт наличие username. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing-equalization") + +# 32 байта случайности = 256 бит энтропии; base64url без padding = 43 символа. +TOKEN_BYTES = 32 +MAX_TOKEN_LENGTH = 256 + + +# lab3: у каждой реплики лимит памяти (deploy.resources.limits). Один хеш Argon2id +# занимает 19 MiB, а sync-обработчики FastAPI выполняются в threadpool до 40 потоков. +# Без ограничения всплеск входов мог бы занять 40 x 19 MiB и привести к OOM-kill +# реплики. Семафор ограничивает число одновременных хешей (PASSWORD_HASH_CONCURRENCY). +_kdf_slots = threading.BoundedSemaphore(4) + + +def configure_kdf_concurrency(limit: int) -> None: + global _kdf_slots + _kdf_slots = threading.BoundedSemaphore(max(1, limit)) + + +def hash_password(password: str) -> str: + with _kdf_slots: + return _hasher.hash(password) + + +def verify_password(password_hash: Optional[str], password: str) -> bool: + try: + with _kdf_slots: + return _hasher.verify(password_hash or _DUMMY_HASH, password) and bool( + password_hash + ) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(TOKEN_BYTES) + + +def token_digest(token: str) -> bytes: + """Токен высокоэнтропийный, поэтому для поиска достаточно быстрого SHA-256. + + В отличие от пароля, перебор 256-битного токена невозможен, и KDF здесь + только замедлил бы каждый запрос. + """ + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]: + """Authorization: Basic base64(username:password) -> (username, password).""" + if not header: + return None + scheme, _, value = header.strip().partition(" ") + if scheme.lower() != "basic" or not value.strip(): + return None + try: + raw = base64.b64decode(value.strip(), validate=True) + decoded = raw.decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо + if not sep or not USERNAME_RE.match(username): + return None + if not 12 <= len(password) <= 128: + return None + return username, password + + +def parse_bearer(header: Optional[str]) -> Optional[str]: + if not header: + return None + scheme, _, value = header.strip().partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token or len(token) > MAX_TOKEN_LENGTH: + return None + if not token.isascii() or any(ch.isspace() for ch in token): + return None + return token diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..361d81a --- /dev/null +++ b/src/server.py @@ -0,0 +1,65 @@ +"""Точка входа контейнера: Uvicorn с фазой осушения (drain) перед остановкой. + +Последовательность при SIGTERM (docker stop / rolling update / stop-one): + + t=0 SIGTERM. Реплика ПРОДОЛЖАЕТ принимать и обслуживать запросы, + но /health/ready -> 503 и ответы идут с Connection: close. + t<=~1.5 с HAProxy (inter 1s, fastinter 500ms, fall 2) помечает реплику DOWN + и больше не отправляет ей новые запросы. + t=DRAIN_SECONDS (3 с) + Uvicorn закрывает listening-сокет, дожидается завершения запросов + в обработке (не дольше GRACEFUL_TIMEOUT_SECONDS = 10 с), закрывает + пул PostgreSQL и выходит. + stop_grace_period 20 с > 3 + 10 — Docker не успевает прислать SIGKILL. + +Повторный SIGTERM/SIGINT во время осушения — немедленный переход к остановке. +Запуск: python -m src.server (CMD в Dockerfile, процесс — PID 1). +""" +import logging +import os +import threading + +import uvicorn + +from .runtime import DRAIN, INSTANCE_ID + +log = logging.getLogger("messenger.server") + + +class DrainingServer(uvicorn.Server): + drain_seconds = 3.0 + + def handle_exit(self, sig, frame): # вызывается из обработчика сигнала + if self.should_exit or not DRAIN.begin(): + # Второй сигнал (или уже останавливаемся) — без ожидания. + return super().handle_exit(sig, frame) + log.info( + "signal %s: instance %s draining for %.1fs (readiness=503), then graceful stop", + sig, INSTANCE_ID, self.drain_seconds, + ) + timer = threading.Timer(self.drain_seconds, super().handle_exit, args=(sig, frame)) + timer.daemon = True + timer.start() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + config = uvicorn.Config( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + # Дольше, чем HAProxy держит простаивающие соединения к серверу + # (pool-purge-delay 5s): соединение закрывает LB, а не сервер под запросом. + timeout_keep_alive=int(os.environ.get("KEEP_ALIVE_SECONDS", "30")), + timeout_graceful_shutdown=int(os.environ.get("GRACEFUL_TIMEOUT_SECONDS", "10")), + access_log=False, + server_header=False, + log_config=None, + ) + server = DrainingServer(config) + server.drain_seconds = float(os.environ.get("DRAIN_SECONDS", "3")) + server.run() + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 0000000..d906fea --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,240 @@ +"""Сессии: PostgreSQL — источник истины, Redis — кэш проверок токена. + +Роль Redis +---------- +Каждый защищённый запрос сначала ищет `sess:` в Redis. +* попадание -> пользователь берётся из кэша, PostgreSQL не трогаем; +* промах -> читаем sessions JOIN users в PostgreSQL и кладём результат + в Redis через SET NX с TTL = min(SESSION_CACHE_TTL_SECONDS, + время до expires_at). Запись кэша никогда не переживает сессию. + +Инвалидация (logout) +-------------------- +1. UPDATE sessions SET revoked_at (COMMIT в PostgreSQL — момент отзыва); +2. SET sess: = "revoked" PX=SESSION_CACHE_TTL (обычный SET, перезаписывает). +Заполнение кэша идёт через SET NX, поэтому параллельный запрос, прочитавший +сессию до отзыва, не может перезаписать «надгробие» валидной записью. + +Надёжность отзыва при сбое Redis +-------------------------------- +Отзыв в PostgreSQL и строка-«долг» в session_cache_invalidations пишутся +ОДНОЙ транзакцией (миграция 002). Строка удаляется только после успешной +записи «надгробия» в Redis. Если записать его не удалось (таймаут, обрыв +связи, Redis заморожен), на кэш больше нельзя полагаться: в нём могла +остаться положительная запись отозванного токена. Поэтому: +* пока долг не погашен, положительные попадания кэша НЕ принимаются — + каждый токен проверяется в PostgreSQL (отозванный -> 401); +* не чаще раза в секунду API пытается погасить долг: пишет надгробия в + Redis и удаляет строки; строки старше SESSION_CACHE_TTL + 5 с удаляются + без записи (любая положительная запись к этому моменту уже истекла); +* наличие долга перечитывается из PostgreSQL не реже раза в секунду, так что + и другие реплики (lab3) перестают доверять кэшу не позже чем через 1 с; + реплика, выполнившая logout, — сразу, до ответа 204. + +Отказ Redis +----------- +Любая ошибка Redis (таймаут 250 мс) -> кэш пропускается, проверка идёт в +PostgreSQL; на CACHE_RETRY_AFTER_SECONDS Redis выключается (простой +circuit breaker), чтобы не платить таймаут на каждом запросе. +Redis работает без persistence: после рестарта кэш пуст, и это безопасно — +следующие запросы просто заполнят его из PostgreSQL. +""" +import json +import logging +import threading +import time +from datetime import datetime, timezone +from typing import Optional, Tuple + +import redis + +from .config import Settings +from .models import User +from .security import new_token, token_digest +from .storage import Storage + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionCache: + def __init__(self, settings: Settings) -> None: + self.ttl_ms = settings.session_cache_ttl_seconds * 1000 + self.retry_after = settings.cache_retry_after_seconds + self._down_until = 0.0 + self._lock = threading.Lock() + self.client: Optional[redis.Redis] = None + if settings.cache_url: + self.client = redis.Redis.from_url( + settings.cache_url, + socket_timeout=settings.cache_timeout_seconds, + socket_connect_timeout=settings.cache_timeout_seconds, + retry_on_timeout=False, + health_check_interval=30, + max_connections=20, + decode_responses=True, + ) + # Счётчики для наблюдения и тестов (GET /health/cache не нужен контракту, + # поэтому они только логируются). + self.hits = 0 + self.misses = 0 + self.errors = 0 + + @staticmethod + def key(digest: bytes) -> str: + return "sess:" + digest.hex() + + def _available(self) -> bool: + return self.client is not None and time.monotonic() >= self._down_until + + def _failed(self, exc: Exception) -> None: + with self._lock: + self.errors += 1 + if time.monotonic() >= self._down_until: + log.warning("redis unavailable (%s); bypassing cache for %.0fs", + type(exc).__name__, self.retry_after) + self._down_until = time.monotonic() + self.retry_after + + def get(self, digest: bytes): + """None — промах/кэш недоступен; REVOKED — отозвано; иначе (User, expires_at).""" + if not self._available(): + return None + try: + raw = self.client.get(self.key(digest)) + except redis.RedisError as exc: + self._failed(exc) + return None + if raw is None: + self.misses += 1 + return None + self.hits += 1 + if raw == REVOKED: + return REVOKED + try: + data = json.loads(raw) + expires_at = datetime.fromisoformat(data["expires_at"]) + return User(**data["user"]), expires_at + except (ValueError, KeyError, TypeError): + return None + + def put(self, digest: bytes, user: User, expires_at: datetime) -> None: + if not self._available(): + return + remaining_ms = int((expires_at - datetime.now(timezone.utc)).total_seconds() * 1000) + ttl = min(self.ttl_ms, remaining_ms) + if ttl <= 0: + return + value = json.dumps({"user": user.model_dump(mode="json"), "expires_at": expires_at.isoformat()}) + try: + self.client.set(self.key(digest), value, px=ttl, nx=True) + except redis.RedisError as exc: + self._failed(exc) + + def revoke(self, digest: bytes) -> bool: + """True — надгробие записано (или кэша нет вовсе); False — не удалось.""" + if self.client is None: + return True + if not self._available(): + return False + try: + self.client.set(self.key(digest), REVOKED, px=self.ttl_ms) + return True + except redis.RedisError as exc: + self._failed(exc) + return False + + def ping(self) -> bool: + if self.client is None: + return False + try: + return bool(self.client.ping()) + except redis.RedisError: + return False + + +class SessionService: + # Как часто перечитывать из PostgreSQL наличие непогашенных инвалидаций. + RECHECK_SECONDS = 1.0 + + def __init__(self, storage: Storage, cache: SessionCache, settings: Settings) -> None: + self.storage = storage + self.cache = cache + self.ttl = settings.session_ttl_seconds + # Строки старше этого возраста можно удалять без записи в Redis. + self.stale_after = settings.session_cache_ttl_seconds + 5 + self._lock = threading.Lock() + self._pending = True # до первой сверки с БД кэшу не доверяем + self._checked_at = float("-inf") + + def create(self, user: User) -> Tuple[str, datetime]: + token = new_token() + expires_at = self.storage.create_session(token_digest(token), user.id, self.ttl) + return token, expires_at + + def _reconcile(self) -> None: + """Погасить долг инвалидации; вызывается под self._lock.""" + try: + rows = self.storage.pending_invalidations() + done = [] + for digest, age in rows: + if age > self.stale_after or self.cache.revoke(digest): + done.append(digest) + else: + break # Redis всё ещё недоступен — попробуем позже + self.storage.delete_invalidations(done) + self._pending = len(done) < len(rows) or len(rows) >= 500 + if rows and not self._pending: + log.info("cache invalidation backlog cleared (%d)", len(done)) + except Exception as exc: # PostgreSQL недоступна: доверять кэшу нельзя + log.warning("cannot verify cache invalidations: %s", type(exc).__name__) + self._pending = True + # Пробрасываем сразу (-> 503): иначе запрос ждал бы пул второй раз + # в find_session и отвечал за 2 x DB_POOL_TIMEOUT_SECONDS. + raise + + def cache_trusted(self) -> bool: + now = time.monotonic() + if now - self._checked_at < self.RECHECK_SECONDS: + return not self._pending + with self._lock: + if now - self._checked_at >= self.RECHECK_SECONDS: + self._reconcile() + self._checked_at = time.monotonic() + return not self._pending + + def resolve(self, token: str) -> Optional[User]: + digest = token_digest(token) + # Сначала сверка (она может дописать надгробия в Redis), и только + # потом чтение кэша — иначе прочитанная до сверки запись устарела бы. + trusted = self.cache_trusted() + cached = self.cache.get(digest) + if cached == REVOKED: + return None + if cached is not None and trusted: + user, expires_at = cached + if expires_at > datetime.now(timezone.utc): + return user + found = self.storage.find_session(digest) # PostgreSQL недоступна -> исключение -> 503 + if found is None: + return None + user, expires_at = found + self.cache.put(digest, user, expires_at) + return user + + def revoke(self, token: str) -> bool: + digest = token_digest(token) + if not self.storage.revoke_session(digest): # COMMIT: отзыв + долг инвалидации + return False + if self.cache.revoke(digest): + try: + self.storage.delete_invalidations([digest]) + except Exception: + pass # строка останется и будет удалена при следующей сверке + else: + # Надгробие не записано: до погашения долга кэшу не доверяем. + with self._lock: + self._pending = True + self._checked_at = time.monotonic() + log.warning("logout: cache invalidation deferred; positive cache disabled") + return True diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..211e7d0 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,661 @@ +"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается +со значениями пользователя. Каждая публичная операция — одна транзакция +(`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении). +""" +from datetime import datetime +from typing import List, Optional, Sequence, Tuple +from uuid import UUID, uuid4 + +from psycopg import errors +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from .models import Attachment, Chat, Message, User, Variant + + +class UsernameTaken(Exception): + pass + + +class UnknownMember(Exception): + pass + + +class NotChatMember(Exception): + """Чата нет или пользователь не участник -> 404.""" + + +class IdempotencyConflict(Exception): + """Тот же ключ уже использован с другим телом -> 409.""" + + +class IdempotencyRace(Exception): + """Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом.""" + + +class AttachmentNotFound(Exception): + """Вложение не существует, ещё не принято (pending) или принадлежит другому чату -> 404.""" + + +class AttachmentFailed(Exception): + """lab5: вложение в состоянии failed нельзя прикрепить к сообщению -> 409.""" + + +# Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py). +ATTACHMENT_GC_LOCK_ID = 2026_0004 + +ATTACHMENT_COLUMNS = ( + "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.state, a.error_code, a.pipeline_version, a.created_at, a.updated_at" +) + +# Внутреннее состояние -> публичный status. pending (приём ещё не завершён) наружу +# не попадает: такие строки запросы API не выбирают. +PUBLIC_STATES = ("queued", "processing", "ready", "failed") + +# Подзапрос: attachment_ids сообщения в исходном порядке (position). +MESSAGE_ATTACHMENT_IDS = ( + "COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) " + "FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids" +) + + +def _attachment(row, variants=()) -> Attachment: + state = row["state"] + return Attachment( + id=row["id"], + chat_id=row["chat_id"], + owner_id=row["owner_id"], + filename=row["filename"], + content_type=row["content_type"], + size_bytes=row["size_bytes"], + sha256=row["sha256"], + status=state, + # Варианты публикуются вместе со статусом ready (одна транзакция worker). + variants=list(variants) if state == "ready" else [], + error_code=row.get("error_code") if state == "failed" else None, + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +# Одна реплика за раз чистит истёкшие ключи идемпотентности. +IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003 + + +Position = Optional[Tuple[datetime, UUID]] + + +def _user(row) -> User: + return User( + id=row["id"], + username=row["username"], + display_name=row["display_name"], + created_at=row["created_at"], + ) + + +class Storage: + def __init__(self, pool: ConnectionPool) -> None: + self.pool = pool + + def ping(self) -> None: + with self.pool.connection() as conn: + conn.execute("SELECT 1") + + # users + + def create_user(self, username: str, display_name: str, password_hash: str) -> User: + try: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ).fetchone() + except errors.UniqueViolation: + # Гонку двух одинаковых регистраций решает UNIQUE(username) в БД, + # а не проверка «SELECT, потом INSERT» в приложении. + raise UsernameTaken() + return _user(row) + + def get_credentials(self, username: str) -> Optional[Tuple[User, str]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT id, username, display_name, created_at, password_hash + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if row is None: + return None + return _user(row), row["password_hash"] + + # sessions (источник истины) + + def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime: + with self.pool.connection() as conn: + # Заодно убираем давно истёкшие сессии этого пользователя. + conn.execute( + "DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'", + (user_id,), + ) + row = conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING expires_at + """, + (token_hash, user_id, ttl_seconds), + ).fetchone() + return row[0] + + def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked_at IS NULL + AND s.expires_at > clock_timestamp() + """, + (token_hash,), + ).fetchone() + if row is None: + return None + return _user(row), row["expires_at"] + + def revoke_session(self, token_hash: bytes) -> bool: + """Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = %s + AND revoked_at IS NULL + AND expires_at > clock_timestamp() + RETURNING 1 + """, + (token_hash,), + ).fetchone() + if row is not None: + conn.execute( + """ + INSERT INTO session_cache_invalidations (token_hash) + VALUES (%s) ON CONFLICT (token_hash) DO NOTHING + """, + (token_hash,), + ) + return row is not None + + # долг инвалидации кэша + + def has_pending_invalidations(self) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)" + ).fetchone()[0] + + def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]: + """[(token_hash, возраст в секундах)] — самые старые первыми.""" + with self.pool.connection() as conn: + rows = conn.execute( + """ + SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at) + FROM session_cache_invalidations + ORDER BY revoked_at LIMIT %s + """, + (limit,), + ).fetchall() + return [(bytes(h), float(age)) for h, age in rows] + + def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None: + if not token_hashes: + return + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", + (list(token_hashes),), + ) + + # chats + + def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + [(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)], + ) + + def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat: + """Чат и все участники создаются в одной транзакции. + + Существование участников проверяет внешний ключ chat_members.user_id: + если хоть один id неизвестен, INSERT падает, и откатывается вся + транзакция, включая уже вставленную строку chats и других участников. + """ + chat_id = uuid4() + try: + with self.pool.connection() as conn: + with conn.transaction(): + created_at = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at", + (chat_id, title), + ).fetchone()[0] + self._insert_members(conn, chat_id, member_ids) + except errors.ForeignKeyViolation: + raise UnknownMember() + return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at) + + def _members_for(self, conn, chat_ids: List[UUID]) -> dict: + members = {chat_id: [] for chat_id in chat_ids} + if not chat_ids: + return members + rows = conn.execute( + """ + SELECT chat_id, user_id FROM chat_members + WHERE chat_id = ANY(%s) ORDER BY chat_id, position + """, + (chat_ids,), + ).fetchall() + for chat_id, user_id in rows: + members[chat_id].append(user_id) + return members + + def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s + WHERE c.id = %s + """, + (user_id, chat_id), + ).fetchone() + if row is None: + return None + members = self._members_for(conn, [row["id"]]) + return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"]) + + def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]: + params: list = [user_id] + keyset = "" + if after is not None: + keyset = "AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chat_members m JOIN chats c ON c.id = m.chat_id + WHERE m.user_id = %s {keyset} + ORDER BY c.created_at, c.id + LIMIT %s + """.format(keyset=keyset), + params, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + members = self._members_for(conn, [r["id"] for r in rows]) + chats = [ + Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"]) + for r in rows + ] + return chats, has_more + + # messages + + @staticmethod + def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]: + """Проверка членства и вставка — один оператор, без окна гонки.""" + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s + ) + RETURNING id, chat_id, sender_id, text, created_at + """, + (message_id, chat_id, sender_id, text, chat_id, sender_id), + ).fetchone() + return Message(**row) if row is not None else None + + @staticmethod + def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None: + """Привязка вложений в той же транзакции, что и сообщение. + + lab5: можно ссылаться на queued/processing/ready вложения ЭТОГО чата. + Неизвестное/чужое/ещё не принятое (pending) -> AttachmentNotFound (404); + failed -> AttachmentFailed (409). Любое исключение -> ROLLBACK сообщения и + ключа идемпотентности. FOR KEY SHARE не даёт удалить строку до COMMIT; + составной FK (attachment_id, chat_id) дублирует проверку чата в БД. + """ + if not attachment_ids: + return + ids = list(attachment_ids) + rows = conn.execute( + """ + SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'pending' + FOR KEY SHARE + """, + (ids, message.chat_id), + ).fetchall() + if len(rows) != len(ids): + raise AttachmentNotFound() + if any(state == "failed" for _id, state in rows): + raise AttachmentFailed() + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) + VALUES (%s, %s, %s, %s) + """, + [(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)], + ) + message.attachment_ids = ids + + def create_message( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + idem_key: str, + request_hash: bytes, + idem_ttl_seconds: int, + attachment_ids: Sequence[UUID] = (), + ) -> Tuple[Message, bool]: + """Идемпотентная отправка. Возвращает (сообщение, replayed). + + Всё — одна транзакция READ COMMITTED (см. migrations/003): + 1. INSERT строки ключа. Если такой ключ уже есть и он не истёк, + ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если + конкурентная транзакция с тем же ключом ещё не завершилась, INSERT + ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные + повторы через разные реплики создают ровно одно сообщение. + 2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник -> + исключение -> ROLLBACK: ключ не «сгорает» на 404. + 2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем + hash тела: совпал -> то же сообщение; нет -> 409. + Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение. + """ + new_id = uuid4() + with self.pool.connection() as conn: + with conn.transaction(): + owned = conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, chat_id, idem_key, request_hash, message_id, expires_at) + VALUES (%s, 'POST', %s, %s, %s, %s, + clock_timestamp() + make_interval(secs => %s)) + ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = EXCLUDED.message_id, + created_at = clock_timestamp(), + expires_at = EXCLUDED.expires_at + -- истёкший (ещё не удалённый очисткой) ключ можно занять заново + WHERE idempotency_keys.expires_at <= clock_timestamp() + RETURNING message_id + """, + (sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds), + ).fetchone() + if owned is not None: + message = self._insert_message(conn, new_id, chat_id, sender_id, text) + if message is None: + raise NotChatMember() # -> ROLLBACK вместе со строкой ключа + # lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего. + self._link_attachments(conn, message, attachment_ids) + return message, False + + # Ключ уже подтверждён другой (возможно, конкурентной) транзакцией. + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if member is None: + raise NotChatMember() + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + WHERE k.user_id = %s AND k.method = 'POST' + AND k.chat_id = %s AND k.idem_key = %s + """.format(attachments=MESSAGE_ATTACHMENT_IDS), + (sender_id, chat_id, idem_key), + ).fetchone() + if row is None: + # Теоретически: строка удалена между шагами (очистка/каскад). + # Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом. + raise IdempotencyRace() + if bytes(row.pop("request_hash")) != request_hash: + raise IdempotencyConflict() + return Message(**row), True + + def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]: + """Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика.""" + with self.pool.connection() as conn: + with conn.transaction(): + got = conn.execute( + "SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,) + ).fetchone()[0] + if not got: + return None + cur = conn.execute( + """ + DELETE FROM idempotency_keys + WHERE ctid IN ( + SELECT ctid FROM idempotency_keys + WHERE expires_at <= clock_timestamp() + LIMIT %s + ) + """, + (batch,), + ) + return cur.rowcount + + def list_messages( + self, chat_id: UUID, user_id: UUID, limit: int, after: Position + ) -> Optional[Tuple[List[Message], bool]]: + params: list = [chat_id] + keyset = "" + if after is not None: + keyset = "AND (created_at, id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if member is None: + return None + # Keyset-пагинация по индексу messages_chat_created_id_idx: + # Index Scan сразу с позиции курсора, читается limit+1 строк. + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM messages m + WHERE m.chat_id = %s {keyset} + ORDER BY m.created_at, m.id + LIMIT %s + """.format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"), + attachments=MESSAGE_ATTACHMENT_IDS), + params, + ).fetchall() + has_more = len(rows) > limit + return [Message(**r) for r in rows[:limit]], has_more + + # attachments (lab4) + + def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)", + (chat_id, user_id), + ).fetchone()[0] + + def create_pending_attachment( + self, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + object_key: str, + staging_key: Optional[str] = None, + pipeline_version: int = 1, + ) -> bool: + """Шаг 1 из 3: намерение (state=pending) COMMIT до S3 PUT в staging. + + У любого объекта, который API кладёт в бакет, уже есть строка с его ключом + (staging_key): объект без строки может появиться только после удаления + строки сборщиком и никогда не «потеряется» навсегда (src/reconcile.py). + object_key — будущий конечный ключ оригинала, его запишет worker. + Проверка членства и вставка — один оператор. False -> не участник. + """ + with self.pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, width, height, object_key, + staging_key, pipeline_version, state) + SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending' + WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) + RETURNING 1 + """, + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, staging_key, pipeline_version, chat_id, owner_id), + ).fetchone() + return row is not None + + def accept_attachment( + self, + attachment_id: UUID, + job_id: UUID, + payload: dict, + max_attempts: int, + outbox_delay_seconds: float, + ) -> Optional[Tuple[Attachment, int]]: + """Шаг 3 из 3 (lab5): после успешного PUT в staging — ОДНА транзакция: + pending -> queued, запись задания и строки outbox. COMMIT => можно 202. + + Возвращает (вложение, outbox_id) или None, если строку уже забрал сборщик. + """ + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.cursor(row_factory=dict_row).execute( + """ + UPDATE attachments a SET state = 'queued', updated_at = clock_timestamp() + WHERE a.id = %s AND a.state = 'pending' + RETURNING {cols} + """.format(cols=ATTACHMENT_COLUMNS), + (attachment_id,), + ).fetchone() + if row is None: + return None + conn.execute( + """ + INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, + attempt, max_attempts) + VALUES (%s, %s, %s, 'queued', 1, %s) + """, + (job_id, attachment_id, row["pipeline_version"], max_attempts), + ) + outbox_id = conn.execute( + """ + INSERT INTO job_outbox (job_id, payload, available_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING id + """, + (job_id, Jsonb(payload), outbox_delay_seconds), + ).fetchone()[0] + conn.execute( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, actor) " + "VALUES (%s, %s, 1, 'accepted', 'api')", + (job_id, attachment_id), + ) + return _attachment(row), outbox_id + + def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]: + """lab4-совместимость (синхронная загрузка без worker). В lab5 не используется.""" + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp() + WHERE a.id = %s AND a.state = 'pending' + RETURNING {cols} + """.format(cols=ATTACHMENT_COLUMNS), + (attachment_id,), + ).fetchone() + return _attachment(row) if row is not None else None + + def delete_pending_attachment(self, attachment_id: UUID) -> bool: + """Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение.""" + with self.pool.connection() as conn: + cur = conn.execute( + "DELETE FROM attachments WHERE id = %s AND state = 'pending'", + (attachment_id,), + ) + return cur.rowcount == 1 + + @staticmethod + def _variants(conn, attachment_id: UUID, pipeline_version: int) -> List[Variant]: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END + """, + (attachment_id, pipeline_version), + ).fetchall() + return [Variant(**r) for r in rows] + + def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str, int]]: + """(метаданные, object_key оригинала, pipeline_version) принятого вложения, если user — участник чата. + + pending (приём не завершён) — как несуществующее. object_key читаем только в ready. + """ + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT {cols}, a.object_key + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s AND a.state <> 'pending' + """.format(cols=ATTACHMENT_COLUMNS), + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + variants = self._variants(conn, attachment_id, row["pipeline_version"]) if row["state"] == "ready" else [] + return _attachment(row, variants), row["object_key"], row["pipeline_version"] + + def variant_object(self, attachment_id: UUID, pipeline_version: int, name: str) -> Optional[Tuple[str, str, int, str]]: + """(object_key, content_type, size_bytes, sha256) варианта или None.""" + with self.pool.connection() as conn: + row = conn.execute( + """ + SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (attachment_id, pipeline_version, name), + ).fetchone() + return tuple(row) if row else None diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..e52a78d --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,397 @@ +"""lab6: OpenTelemetry для API и worker — traces, metrics, logs по OTLP в Collector. + +Что настраивается в setup(): + * Resource: service.name (messenger-api / messenger-worker), service.version, + service.instance.id (= X-Instance-Id), deployment.environment.name. + * Traces: BatchSpanProcessor (очередь ограничена OTEL_BSP_MAX_QUEUE_SIZE, при + переполнении новые spans отбрасываются, процесс не растёт). Sampler отбрасывает + «сиротские» CLIENT-spans без родителя: это SQL/S3 фоновых циклов (outbox + sweeper, GC, readiness, callbacks метрик) — иначе в Tempo тысячи пустых трасс. + Всё, что происходит внутри HTTP server / producer / consumer span, сохраняется. + * Metrics: PeriodicExportingMetricReader (10 с). Корзины гистограмм заданы + явно в секундах (View), лишние метрики размера тела отброшены. + * Logs: JSON в stdout (для docker logs) + OTLP LoggingHandler. В каждой записи + timestamp, severity, service.name, event, trace_id/span_id (если есть span). + * Автоинструментация: psycopg (SQL без параметров), botocore (S3). + Redis НЕ инструментируется: команды AUTH/кэш сессий могли бы утечь в spans. + +Что никогда не экспортируется: тела сообщений, байты изображений, заголовки +Authorization/Cookie (FastAPI-инструментация заголовки не пишет без явной +настройки), параметры SQL, URL с подписью. Метрики не имеют меток user/chat/message. +Отключение: OTEL_SDK_DISABLED=true (остаются только JSON-логи в stdout). +""" +import json +import logging +import os +import resource +import sys +import time +from datetime import datetime, timezone +from typing import Callable, Dict, Iterable, Optional + +from opentelemetry import metrics, trace +from opentelemetry.context import Context +from opentelemetry.metrics import CallbackOptions, Observation +from opentelemetry.trace import SpanKind +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from .runtime import INSTANCE_ID + +log = logging.getLogger("messenger.telemetry") + +# Только W3C traceparent/tracestate: baggage в сообщение брокера не попадает +# (jobs.schema.json: additionalProperties=false). +PROPAGATOR = TraceContextTextMapPropagator() + +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0) +READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 900.0) + +_STATE = {"service": "messenger", "enabled": False, "configured": False} + + +def enabled() -> bool: + return _STATE["enabled"] + + +def tracer(): + return trace.get_tracer("messenger") + + +def meter(): + return metrics.get_meter("messenger") + + +# ------------------------------------------------------------------ logging + +_STD_ATTRS = set(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | {"message", "asctime", "event"} + + +class _EventFilter(logging.Filter): + """У каждой записи есть поле event: явное (extra={"event": ...}) или имя логгера.""" + + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "event", None): + record.event = record.name + return True + + +class JsonFormatter(logging.Formatter): + def __init__(self, compact: bool = False) -> None: + super().__init__() + self.compact = compact + + def format(self, record: logging.LogRecord) -> str: + out = { + "timestamp": datetime.fromtimestamp(record.created, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "severity": record.levelname, + "service.name": _STATE["service"], + "service.instance.id": INSTANCE_ID, + "event": getattr(record, "event", record.name), + "logger": record.name, + "message": record.getMessage(), + } + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + out["trace_id"] = format(ctx.trace_id, "032x") + out["span_id"] = format(ctx.span_id, "016x") + for key, value in record.__dict__.items(): + if key not in _STD_ATTRS and not key.startswith("_") and key not in out: + out[key] = value if isinstance(value, (str, int, float, bool)) or value is None else str(value) + if record.exc_info: + if self.compact: + out["error"] = type(record.exc_info[1]).__name__ + else: + out["exception"] = self.formatException(record.exc_info) + return json.dumps(out, ensure_ascii=False) + + +def _configure_logging(otel_handler: Optional[logging.Handler]) -> None: + root = logging.getLogger() + for h in list(root.handlers): + root.removeHandler(h) + stdout = logging.StreamHandler(sys.stdout) + stdout.setFormatter(JsonFormatter()) + stdout.addFilter(_EventFilter()) + root.addHandler(stdout) + if otel_handler is not None: + otel_handler.addFilter(_EventFilter()) + root.addHandler(otel_handler) + root.setLevel(logging.INFO) + # Шумные библиотеки: их DEBUG/INFO не нужны ни в stdout, ни в Loki. + for name in ("botocore", "boto3", "urllib3"): + logging.getLogger(name).setLevel(logging.WARNING) + # Ошибки самого SDK (Collector недоступен): только в stdout, одной строкой, + # без трейсбека и НЕ в OTLP — иначе петля «лог об ошибке экспорта логов». + otel = logging.getLogger("opentelemetry") + for h in list(otel.handlers): + otel.removeHandler(h) + sdk_handler = logging.StreamHandler(sys.stdout) + sdk_handler.setFormatter(JsonFormatter(compact=True)) + sdk_handler.addFilter(_EventFilter()) + otel.addHandler(sdk_handler) + otel.propagate = False + otel.setLevel(logging.WARNING) + + +# ------------------------------------------------------------------ setup + +def _sampler(): + from opentelemetry.sdk.trace.sampling import ALWAYS_ON, Decision, ParentBased, Sampler, SamplingResult + + class DropOrphanClientSpans(Sampler): + """CLIENT-span без родителя (фоновый SQL/S3) не сэмплируется; остальное — ParentBased(AlwaysOn).""" + + _base = ParentBased(ALWAYS_ON) + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + parent = trace.get_current_span(parent_context).get_span_context() + if not parent.is_valid and kind == SpanKind.CLIENT: + return SamplingResult(Decision.DROP) + return self._base.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + + def get_description(self) -> str: + return "DropOrphanClientSpans{ParentBased{AlwaysOn}}" + + return DropOrphanClientSpans() + + +def setup(service_name: str) -> None: + """Вызывать один раз в процессе ДО создания пула БД и клиента S3.""" + if _STATE["configured"]: + return + _STATE["configured"] = True + _STATE["service"] = service_name + if os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() == "true": + _configure_logging(None) + log.info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)", extra={"event": "telemetry.disabled"}) + return + # Стабильные HTTP semconv: http.server.request.duration в секундах, + # атрибуты http.request.method / http.route / http.response.status_code. + os.environ.setdefault("OTEL_SEMCONV_STABILITY_OPT_IN", "http") + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import DropAggregation, ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # Resource.create добавляет OTEL_RESOURCE_ATTRIBUTES из окружения. + res = Resource.create({ + "service.name": service_name, + "service.version": os.environ.get("APP_VERSION", "lab6"), + "service.instance.id": INSTANCE_ID, + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + }) + + # Размеры очередей/таймауты — OTEL_BSP_* / OTEL_BLRP_* / OTEL_EXPORTER_OTLP_TIMEOUT из окружения. + tp = TracerProvider(resource=res, sampler=_sampler()) + tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(tp) + + views = [ + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + View(instrument_name="http.server.request.body.size", aggregation=DropAggregation()), + View(instrument_name="http.server.response.body.size", aggregation=DropAggregation()), + ] + interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000")) + reader = PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval) + metrics.set_meter_provider(MeterProvider(resource=res, metric_readers=[reader], views=views)) + + lp = LoggerProvider(resource=res) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + set_logger_provider(lp) + _configure_logging(LoggingHandler(level=logging.INFO, logger_provider=lp)) + + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + + PsycopgInstrumentor().instrument(skip_dep_check=True) + BotocoreInstrumentor().instrument(skip_dep_check=True) + _register_process_metrics() + _STATE["enabled"] = True + log.info("OpenTelemetry enabled: endpoint=%s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"), + extra={"event": "telemetry.enabled"}) + + +def shutdown() -> None: + """Сбросить буферы при штатной остановке (ограничено таймаутами экспортёров).""" + if not _STATE["enabled"]: + return + from opentelemetry._logs import get_logger_provider + + for provider in (trace.get_tracer_provider(), metrics.get_meter_provider(), get_logger_provider()): + try: + provider.shutdown() + except Exception: + pass + + +def instrument_fastapi(app) -> None: + if not _STATE["enabled"]: + return + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + # Health-пробы (HAProxy каждую секунду) не входят ни в трассы, ни в метрики + # HTTP: доля ошибок считается по бизнес-запросам. + FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready", + exclude_spans=["receive", "send"]) + + +# ------------------------------------------------------------------ messaging context + +def inject(carrier: Dict) -> Dict: + """Записать traceparent/tracestate текущего span в словарь (сообщение задания).""" + PROPAGATOR.inject(carrier) + return carrier + + +def extract(carrier: Optional[Dict]) -> Context: + if not isinstance(carrier, dict): + return Context() + return PROPAGATOR.extract({k: v for k, v in carrier.items() if k in ("traceparent", "tracestate") + and isinstance(v, str)}) + + +# ------------------------------------------------------------------ app metrics + +class _Lazy: + """Инструменты создаются при первом обращении (после setup()).""" + + def __init__(self) -> None: + self._made = False + + def _make(self) -> None: + if self._made: + return + m = meter() + self.jobs_processed = m.create_counter( + "messenger.jobs.processed", unit="{job}", + description="Обработанные сообщения заданий по исходу (succeeded/retry/failed/noop/...)") + self.job_duration = m.create_histogram( + "messenger.job.duration", unit="s", description="Длительность обработки задания worker'ом") + self.time_to_ready = m.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="От durable acceptance (COMMIT задания, ответ 202) до публикации ready") + self.published = m.create_counter( + "messenger.jobs.published", unit="{job}", description="Сообщения, опубликованные outbox relay в брокер") + self._made = True + + def __getattr__(self, item): + if item.startswith("_"): + raise AttributeError(item) + self._make() + return self.__dict__[item] + + +INSTRUMENTS = _Lazy() + + +def _safe(callback: Callable[[], Iterable[Observation]]) -> Callable[[CallbackOptions], Iterable[Observation]]: + last = {"t": 0.0} + + def wrapped(_options: CallbackOptions) -> Iterable[Observation]: + try: + return list(callback()) + except Exception as exc: # БД недоступна: точку пропускаем, а не роняем экспорт + if time.monotonic() - last["t"] > 60: # не чаще раза в минуту + last["t"] = time.monotonic() + log.warning("metric callback skipped: %s", type(exc).__name__, + extra={"event": "telemetry.callback_failed"}) + return [] + return wrapped + + +def _register_process_metrics() -> None: + m = meter() + page = os.sysconf("SC_PAGE_SIZE") + + def rss(): + with open("/proc/self/statm") as fh: + yield Observation(int(fh.read().split()[1]) * page) + + def cpu(): + r = resource.getrusage(resource.RUSAGE_SELF) + c = resource.getrusage(resource.RUSAGE_CHILDREN) # подпроцессы обработки изображений + yield Observation(r.ru_utime + r.ru_stime + c.ru_utime + c.ru_stime) + + m.create_observable_gauge("process.memory.rss", [_safe(rss)], unit="By", description="RSS процесса") + m.create_observable_counter("process.cpu.time", [_safe(cpu)], unit="s", + description="CPU-время процесса и его подпроцессов") + + +def register_pool_metrics(pool) -> None: + """DB pool: занято/свободно/ожидающих/максимум (psycopg_pool.get_stats).""" + m = meter() + + def conns(): + st = pool.get_stats() + size, idle = st.get("pool_size", 0), st.get("pool_available", 0) + yield Observation(size - idle, {"state": "used"}) + yield Observation(idle, {"state": "idle"}) + + m.create_observable_gauge("messenger.db.pool.connections", [_safe(conns)], unit="{connection}") + m.create_observable_gauge("messenger.db.pool.waiting", + [_safe(lambda: [Observation(pool.get_stats().get("requests_waiting", 0))])], + unit="{request}", description="Запросы, ждущие свободное соединение") + m.create_observable_gauge("messenger.db.pool.max", [_safe(lambda: [Observation(pool.max_size)])], + unit="{connection}") + + +def register_queue_metrics(pool, queue: str) -> None: + """Глубина очереди и возраст старейшего задания — из PostgreSQL (источник истины). + + Публикуется API-репликами: метрика должна жить, когда worker остановлен + (именно в этот момент она важнее всего). Несколько реплик дают одинаковые + значения — в PromQL берётся max(). + """ + m = meter() + cache = {"t": 0.0, "row": None} + + def row(): + if time.monotonic() - cache["t"] > 5 or cache["row"] is None: + with pool.connection(timeout=1) as conn: + cache["row"] = conn.execute( + """ + SELECT count(*) FILTER (WHERE status = 'queued'), + count(*) FILTER (WHERE status = 'processing'), + COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0), + (SELECT count(*) FROM job_outbox) + FROM attachment_jobs WHERE status IN ('queued', 'processing') + """ + ).fetchone() + cache["t"] = time.monotonic() + return cache["row"] + + attrs = {"queue": queue} + m.create_observable_gauge( + "messenger.queue.depth", + [_safe(lambda: [Observation(row()[0], dict(attrs, state="queued")), + Observation(row()[1], dict(attrs, state="processing"))])], + unit="{job}", description="Незавершённые задания") + m.create_observable_gauge( + "messenger.queue.oldest_age", [_safe(lambda: [Observation(float(row()[2]), attrs)])], + unit="s", description="Возраст старейшего незавершённого задания") + m.create_observable_gauge( + "messenger.outbox.pending", [_safe(lambda: [Observation(row()[3], attrs)])], + unit="{job}", description="Строки outbox, ещё не опубликованные в брокер") + + +def register_worker_metrics(busy: Callable[[], int], capacity: int) -> None: + m = meter() + m.create_observable_gauge("messenger.worker.busy", [_safe(lambda: [Observation(busy())])], unit="{slot}", + description="Занятые consumer-потоки") + m.create_observable_gauge("messenger.worker.capacity", [_safe(lambda: [Observation(capacity)])], + unit="{slot}", description="WORKER_CONCURRENCY") diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..b8768ce --- /dev/null +++ b/src/worker.py @@ -0,0 +1,417 @@ +"""Worker обработки вложений (lab5): python -m src.worker + +Отдельный процесс/контейнер. WORKER_CONCURRENCY потоков; у каждого свой consumer +в группе Redis Streams и prefetch = 1 (XREADGROUP COUNT 1), поэтому одновременно +в работе не больше WORKER_CONCURRENCY сообщений на процесс. + +Обработка одного сообщения: + 1. Проверка схемы (contracts/jobs.schema.json). Мусор -> ack + лог (poison). + 2. claim(): аренда в PostgreSQL (lease_token, JOB_TIMEOUT_SECONDS). Нельзя + захватить (готово/упало/устарело/арендовано живым) -> no-op, ack. + 3. Источник: staging-объект (или уже опубликованный оригинал при повторе); + ключи берутся из БД, не из сообщения. Размер <= MAX_UPLOAD_BYTES, SHA-256 + должен совпасть с метаданными. + 4. Оригинал -> final//original (если там ещё нет того же SHA-256). + 5. Варианты в изолированном подпроцессе (src/imaging.py) -> детерминированные + ключи final//p<версия>/<вариант>.: повтор перезаписывает те же байты. + 6. complete(): ОДНА транзакция, fenced по lease_token: варианты + ready. + 7. XACK — только после COMMIT (consumer ack после устойчивого эффекта). + 8. Удаление staging (не удалось — уберёт сборщик, src/reconcile.py, проход D). + +Ошибки: временные (S3/БД недоступны, таймаут обработки) -> retry через outbox с +backoff+jitter, не больше JOB_MAX_ATTEMPTS; постоянные (битое изображение, лимит +памяти, нет источника) -> failed + dead_letters сразу. Если недоступна сама БД, +сообщение НЕ подтверждается и вернётся через PEL. + +SIGTERM: потоки перестают брать новые сообщения и дорабатывают текущее +(stop_grace_period в compose больше JOB_TIMEOUT_SECONDS). +""" +import hashlib +import logging +import os +import signal +import threading +import time +from pathlib import Path +from typing import Dict, List, Optional + +import psycopg +from opentelemetry.trace import SpanKind, Status, StatusCode +from psycopg_pool import PoolTimeout + +from . import telemetry +from .broker import Broker, BrokerUnavailable, Delivery +from .config import load_settings +from .db import create_pool +from .imaging import VARIANT_EXT, ImageFailure, ImageTimeout, Sandbox, sha256_hex +from .jobs import JOB_TYPE, Claim, JobStore, validate_payload +from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable +from .outbox import Relay +from .reconcile import gc_loop +from .runtime import INSTANCE_ID + +log = logging.getLogger("messenger.worker") + +HEARTBEAT = Path("/tmp/worker-heartbeat") + + +class Transient(Exception): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = "") -> None: + super().__init__(code) + self.code = code + self.detail = detail + + +class InjectedCrash(Exception): + pass + + +def final_original_key(attachment_id) -> str: + return "final/%s/original" % attachment_id + + +def variant_key(attachment_id, pipeline_version: int, name: str) -> str: + return "final/%s/p%d/%s.%s" % (attachment_id, pipeline_version, name, VARIANT_EXT[name]) + + +class Worker: + def __init__(self) -> None: + self.settings = load_settings() + s = self.settings + if not s.broker_url: + raise RuntimeError("BROKER_URL or BROKER_URL_FILE must be set") + if not ObjectStore.configured(s): + raise RuntimeError("S3 must be configured for the worker") + self.pool = create_pool(s) + self.jobs = JobStore(self.pool, s) + self.store = ObjectStore(s) + self.sandbox = Sandbox(s.worker_image_memory_mb, s.worker_image_timeout_seconds, s.max_image_pixels) + self.stop = threading.Event() + self.in_flight = 0 + self.lock = threading.Lock() + self.processed = 0 + self.beats: Dict[int, float] = {} + # lab6: DB pool, занятость worker'а (busy/capacity) + telemetry.register_pool_metrics(self.pool) + telemetry.register_worker_metrics(lambda: self.in_flight, s.worker_concurrency) + + # ---------------------------------------------------------------- loop + + def consumer_name(self, index: int) -> str: + # Стабильно при рестарте контейнера (hostname = короткий ID контейнера): + # новая инкарнация сразу видит свой неподтверждённый PEL. + return "%s-%d" % (INSTANCE_ID, index) + + def run_thread(self, index: int) -> None: + name = self.consumer_name(index) + broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group, + self.settings.broker_timeout_seconds, block_ms=2000) + recovered_own = False + last_autoclaim = 0.0 + while not self.stop.is_set(): + self.beats[index] = time.monotonic() + try: + deliveries: List[Delivery] = [] + if not recovered_own: + deliveries = broker.read(name, own_pending=True) + if not deliveries: + recovered_own = True + else: + log.info("%s: recovering own unacked message %s", name, deliveries[0].message_id) + if not deliveries and time.monotonic() - last_autoclaim > 2: + last_autoclaim = time.monotonic() + deliveries = broker.autoclaim(name, self.settings.job_claim_idle_seconds * 1000) + for d in deliveries: + log.info("%s: claimed idle message %s (deliveries=%d)", name, d.message_id, d.deliveries) + if not deliveries: + deliveries = broker.read(name) + for d in deliveries: + self.handle(broker, name, d) + except BrokerUnavailable: + log.warning("%s: broker unavailable, retrying", name) + self.stop.wait(1.0) + except (psycopg.Error, PoolTimeout) as exc: + # БД недоступна до/после обработки: сообщение не подтверждено, вернётся. + log.warning("%s: database unavailable (%s); message stays pending", name, type(exc).__name__) + self.stop.wait(1.0) + except Exception: + log.exception("%s: unexpected error in loop", name) + self.stop.wait(1.0) + broker.close() + log.info("%s: stopped", name) + + # ---------------------------------------------------------------- one message + + def handle(self, broker: Broker, consumer: str, d: Delivery) -> None: + """lab6: CONSUMER span — ребёнок PRODUCER span из traceparent сообщения. + Внутри него — SQL (psycopg) и S3 (botocore) spans этой попытки; retry + пишет в outbox traceparent этого span, следующая попытка — в той же трассе.""" + raw = d.payload() + started = time.monotonic() + with telemetry.tracer().start_as_current_span( + "%s process" % broker.stream, context=telemetry.extract(raw), kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "redis", + "messaging.operation.type": "process", + "messaging.destination.name": broker.stream, + "messaging.consumer.group.name": broker.group, + "messaging.message.id": d.message_id, + "messaging.message.delivery_count": d.deliveries, + "messenger.job.type": JOB_TYPE, + }, + ) as span: + outcome = "error" + try: + outcome = self._handle(broker, consumer, d, raw, span) + finally: + kind = outcome.split(":", 1)[0].split(" ", 1)[0] + span.set_attribute("messenger.job.outcome", kind) + if kind in ("failed", "poison", "error"): + span.set_status(Status(StatusCode.ERROR, outcome)) + attrs = {"job.type": JOB_TYPE, "outcome": kind} + telemetry.INSTRUMENTS.jobs_processed.add(1, attrs) + if kind not in ("noop", "poison"): + telemetry.INSTRUMENTS.job_duration.record(time.monotonic() - started, attrs) + log.info("%s: message %s -> %s in %.2fs", consumer, d.message_id, outcome, + time.monotonic() - started, + extra={"event": "job.processed", "outcome": kind, "job_type": JOB_TYPE, + "duration_s": round(time.monotonic() - started, 3)}) + + def _handle(self, broker: Broker, consumer: str, d: Delivery, raw, span) -> str: + msg = validate_payload(raw) + if msg is None: + log.error("%s: poison message %s (invalid schema), acking", consumer, d.message_id) + broker.ack(d.message_id) + return "poison" + span.set_attribute("messenger.job.id", str(msg["job_id"])) + span.set_attribute("messenger.attachment.id", str(msg["attachment_id"])) + span.set_attribute("messenger.job.attempt", msg["attempt"]) + if d.deliveries > self.settings.job_max_deliveries: + # Сообщение многократно «убивало» worker (или его не могли подтвердить). + self.jobs.fail_poison(msg["job_id"], "poison_message", consumer) + log.error("%s: job %s delivered %d times, marked failed", consumer, msg["job_id"], d.deliveries) + broker.ack(d.message_id) + return "failed:poison_message" + claim, reason = self.jobs.claim(msg, consumer) + if claim is None: + log.info("%s: job %s attempt %d: no-op (%s), ack", consumer, msg["job_id"], msg["attempt"], reason) + broker.ack(d.message_id) + return "noop:" + reason + with self.lock: + self.in_flight += 1 + started = time.monotonic() + try: + outcome = self.process(claim, consumer) + finally: + with self.lock: + self.in_flight -= 1 + # Эффект (ready/retry/failed) уже зафиксирован в БД -> теперь можно ack. + if outcome == "ready" and self.settings.worker_faults and self._marker(claim) == "crash-before-ack" \ + and not self.jobs.has_event(claim.job_id, "fault_injected"): + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected", + "crash after COMMIT before XACK", consumer) + log.error("FAULT_INJECTION: worker exits after COMMIT before XACK (job %s)", claim.job_id) + os._exit(70) + broker.ack(d.message_id) + self.processed += 1 + log.info("%s: job %s attempt %d -> %s in %.2fs", consumer, claim.job_id, claim.attempt, outcome, + time.monotonic() - started) + if outcome == "ready": + self.cleanup_staging(claim) + return "succeeded" + return outcome + + def process(self, claim: Claim, consumer: str) -> str: + try: + variants, original_key = self.produce(claim, consumer) + age = self.jobs.complete(claim, variants, original_key, consumer) + if age is not None: + telemetry.INSTRUMENTS.time_to_ready.record(age, {"job.type": JOB_TYPE}) + else: + log.warning("%s: lease lost for job %s; result not published (fenced)", consumer, claim.job_id) + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "lease_lost", + "result not published: lease token no longer valid", consumer) + return "lease_lost" + return "ready" + except Permanent as exc: + self.jobs.fail(claim, exc.code, consumer, exc.detail or None) + return "failed:" + exc.code + except Transient as exc: + result, delay = self.jobs.retry_or_fail(claim, exc.code, consumer) + return "retry in %.1fs (%s)" % (delay, exc.code) if result == "retry" else result + + def _marker(self, claim: Claim) -> str: + """Маркеры внедрения отказов в filename — ТОЛЬКО при WORKER_FAULTS=true + (compose.faults.yaml). В обычном стенде filename ни на что не влияет.""" + if not self.settings.worker_faults: + return "" + src = self.jobs.attachment_source(claim.attachment_id) or {} + name = src.get("filename") or "" + for marker in ("crash-after-effects", "crash-before-ack", "transient-until-3", + "transient-until-replay", "permanent", "slow"): + if name.startswith("fault-" + marker): + return marker + return "" + + def produce(self, claim: Claim, consumer: str): + """Шаги 3–5: прочитать источник, записать оригинал и варианты в final/.""" + src = self.jobs.attachment_source(claim.attachment_id) + if src is None: + raise Permanent("processing_failed", "attachment row disappeared") + marker = self._marker(claim) + if marker == "transient-until-3" and claim.attempt < 3: + raise Transient("injected_transient") + if marker == "transient-until-replay" and claim.replay_count == 0: + raise Transient("injected_transient") + if marker == "permanent": + raise Permanent("invalid_image", "injected permanent failure") + if marker == "slow" and claim.attempt == 1: + # «Зависание» заметно дольше аренды + grace + период сборщика (5 с). + time.sleep(self.settings.job_timeout_seconds + 20) + + original_key = final_original_key(claim.attachment_id) + data = self.read_source(src, original_key) + try: + # 4. Оригинал в конечный префикс (неизменен: те же байты, тот же SHA-256). + try: + head = self.store.head(original_key) + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + if head is None or (head.get("Metadata") or {}).get("sha256") != src["sha256"]: + self._put(original_key, data, src["content_type"]) + # 5. Варианты. + try: + rendered = self.sandbox.render(data) + log.info("job %s: variants rendered, subprocess peak RSS %.0f MiB", + claim.job_id, self.sandbox.last_peak_mb) + except ImageTimeout: + raise Transient("processing_timeout") + except ImageFailure as exc: + raise Permanent(exc.code if exc.code in ("invalid_image", "resource_limit") else "processing_failed", + str(exc)) + finally: + del data + variants: List[Dict] = [] + for name in ("thumbnail", "crop", "optimized"): + body, width, height, ctype = rendered[name] + key = variant_key(claim.attachment_id, claim.pipeline_version, name) + self._put(key, body, ctype) + variants.append({"name": name, "content_type": ctype, "size_bytes": len(body), "width": width, + "height": height, "sha256": sha256_hex(body), "object_key": key}) + if marker == "crash-after-effects" and not self.jobs.has_event(claim.job_id, "fault_injected"): + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected", + "crash after S3 writes before COMMIT", consumer) + log.error("FAULT_INJECTION: worker exits after S3 writes before COMMIT (job %s)", claim.job_id) + os._exit(70) + return variants, original_key + + def read_source(self, src: Dict, original_key: str) -> bytes: + limit = self.settings.max_upload_bytes + keys = [k for k in (src["staging_key"], original_key) if k] + for key in keys: + try: + data = self.store.read_all(key, limit) + except ObjectMissing: + continue + except ValueError: + raise Permanent("invalid_image", "source exceeds MAX_UPLOAD_BYTES") + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + if len(data) != src["size_bytes"] or sha256_hex(data) != src["sha256"]: + if key == original_key: + continue # частично/чужой объект в final — перезапишем из staging + raise Permanent("processing_failed", "staging checksum mismatch") + return data + raise Permanent("processing_failed", "source object is missing") + + def _put(self, key: str, data: bytes, content_type: str) -> None: + try: + self.store.put(key, data, content_type, hashlib.md5(data, usedforsecurity=False).digest(), + sha256_hex(data)) + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + + def cleanup_staging(self, claim: Claim) -> None: + """Шаг 8: staging больше не нужен — результат опубликован и подтверждён.""" + try: + src = self.jobs.attachment_source(claim.attachment_id) + if not src or src["state"] != "ready" or not src["staging_key"]: + return + self.store.delete(src["staging_key"]) + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND state = 'ready' " + "AND staging_key = %s", (claim.attachment_id, src["staging_key"])) + except Exception as exc: + log.warning("staging cleanup deferred to gc: %s", type(exc).__name__) + + # ---------------------------------------------------------------- housekeeping + + def housekeeping(self) -> None: + broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group, + self.settings.broker_timeout_seconds) + while not self.stop.wait(30): + try: + trimmed = broker.trim_acknowledged() + if trimmed: + log.info("broker: trimmed %d acknowledged entries", trimmed) + except BrokerUnavailable: + pass + broker.close() + + def run(self) -> None: + s = self.settings + log.info("worker %s: concurrency=%d max_attempts=%d lease=%ss claim_idle=%ss image_timeout=%ss " + "image_memory=%dMiB retry=%.1f..%.1fs faults=%s", + INSTANCE_ID, s.worker_concurrency, s.job_max_attempts, s.job_timeout_seconds, + s.job_claim_idle_seconds, s.worker_image_timeout_seconds, s.worker_image_memory_mb, + s.job_retry_base_seconds, s.job_retry_max_seconds, "on" if s.worker_faults else "off") + # Прогрев forkserver до приёма сообщений. + self.sandbox.ctx.Process(target=time.sleep, args=(0,)).start() + HEARTBEAT.touch() + + def on_signal(sig, _frame): + log.info("signal %s: finishing in-flight jobs (%d) and stopping", sig, self.in_flight) + self.stop.set() + + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + + threads = [threading.Thread(target=self.run_thread, args=(i,), name="consumer-%d" % i) + for i in range(s.worker_concurrency)] + aux_stop = threading.Event() + relay_broker = Broker(s.broker_url, s.broker_stream, s.broker_group, s.broker_timeout_seconds) + relay = Relay(self.pool, relay_broker, s, "worker:" + INSTANCE_ID) + aux = [threading.Thread(target=relay.loop, args=(aux_stop,), name="outbox-relay", daemon=True), + threading.Thread(target=self.housekeeping, name="housekeeping", daemon=True), + threading.Thread(target=gc_loop, args=(self.pool, self.store, s, aux_stop), name="attachment-gc", + daemon=True)] + for t in threads + aux: + t.start() + # Heartbeat для healthcheck: пишется, только пока КАЖДЫЙ consumer-поток + # проходит цикл (зависание дольше аренды -> контейнер unhealthy). + limit = s.job_timeout_seconds + 15 + while any(t.is_alive() for t in threads): + now = time.monotonic() + if self.beats and all(now - b < limit for b in self.beats.values()): + HEARTBEAT.touch() + for t in threads: + t.join(timeout=2) + aux_stop.set() + self.pool.close() + log.info("worker %s stopped; processed %d messages", INSTANCE_ID, self.processed) + telemetry.shutdown() + + +def main() -> None: + telemetry.setup("messenger-worker") + Worker().run() + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..3c1ac55 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,127 @@ +"""Общие HTTP-помощники собственных тестов (только стандартная библиотека).""" +import base64 +import json +import os +import uuid +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Resp: + def __init__(self, status, data, headers, elapsed=0.0): + self.status = status + self.json = data + self.headers = headers + self.elapsed = elapsed + + +def request(method, path, token=None, payload=None, raw=None, headers=None, + base_url=None, timeout=10): + import time + h = {"Accept": "application/json"} + if token is not None: + h["Authorization"] = "Bearer " + token + body = None + if payload is not None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + body = raw + h["Content-Type"] = "application/json" + h.update(headers or {}) + # Переход 2 -> 3: POST сообщения требует Idempotency-Key. Старые тесты lab2 + # его не передают — добавляем уникальный ключ, если тест не задал свой. + # Явное значение None в headers означает «не отправлять заголовок вовсе». + if method == "POST" and path.split("?")[0].endswith("/messages") \ + and not any(k.lower() == "idempotency-key" for k in h): + h["Idempotency-Key"] = uuid.uuid4().hex + h = {k: v for k, v in h.items() if v is not None} + req = Request((base_url or BASE_URL) + path, data=body, headers=h, method=method) + started = time.monotonic() + try: + response = urlopen(req, timeout=timeout) + except HTTPError as error: + response = error + with response: + content = response.read() + status = response.status if hasattr(response, "status") else response.code + rh = response.headers + try: + data = json.loads(content.decode("utf-8")) if content else None + except (UnicodeDecodeError, json.JSONDecodeError): + data = None + return Resp(status, data, rh, time.monotonic() - started) + + +def basic(username, password): + raw = (username + ":" + password).encode("utf-8") + return "Basic " + base64.b64encode(raw).decode("ascii") + + +def new_password(): + return "Pw-" + uuid.uuid4().hex + + +def register(prefix="t", password=None, base_url=None): + username = (prefix + "_" + uuid.uuid4().hex[:16])[:32] + password = password or new_password() + r = request("POST", "/api/v1/users", base_url=base_url, payload={ + "username": username, + "display_name": "Test " + prefix, + "password": password, + }) + assert r.status == 201, (r.status, r.json) + user = r.json + user["_password"] = password + return user + + +def login(user, base_url=None): + r = request("POST", "/api/v1/auth/sessions", base_url=base_url, + headers={"Authorization": basic(user["username"], user["_password"])}) + assert r.status == 201, (r.status, r.json) + return r.json["token"] + + +def actor(prefix="t", base_url=None): + user = register(prefix, base_url=base_url) + user["_token"] = login(user, base_url=base_url) + return user + + +def create_chat(owner, *members, base_url=None): + r = request("POST", "/api/v1/chats", token=owner["_token"], base_url=base_url, payload={ + "title": "Test chat " + uuid.uuid4().hex[:8], + "member_ids": [owner["id"]] + [m["id"] for m in members], + }) + assert r.status == 201, (r.status, r.json) + return r.json + + +def reachable(base_url): + try: + urlopen(base_url.rstrip("/") + "/health/live", timeout=2).read() + return True + except (URLError, OSError, HTTPError): + return False + + +def wait_ready(token, attachment_id, timeout=60.0, allow_failed=False): + """lab5: опрашивать метаданные до конечного состояния (ready/failed).""" + import time + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + r = request("GET", "/api/v1/attachments/" + attachment_id, token=token) + last = r + if r.status == 200 and r.json["status"] == "ready": + return r.json + if r.status == 200 and r.json["status"] == "failed": + if allow_failed: + return r.json + raise AssertionError("attachment failed: %s" % r.json.get("error_code")) + time.sleep(0.2) + raise AssertionError("attachment %s not ready in %.0fs (last: %s)" % ( + attachment_id, timeout, last.json if last else None)) diff --git a/student-tests/dependency_failure.py b/student-tests/dependency_failure.py new file mode 100644 index 0000000..b543fde --- /dev/null +++ b/student-tests/dependency_failure.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Автоматическая проверка недоступности Redis и PostgreSQL. + +Сценарий явно останавливает зависимости, поэтому не входит в discovery +unittest. По умолчанию использует docker compose из корня репозитория: + + python student-tests/dependency_failure.py --base-url http://127.0.0.1:8080 + +Команды остановки/запуска можно переопределить (--stop-redis и т.д.), например +для стенда без Docker. Результат — JSON-сводка (без токенов) в stdout и, +при --output, в файл. + +Ожидаемое поведение: +* Redis недоступен -> API обходит кэш, всё работает через PostgreSQL, + readiness = 200, задержка ограничена таймаутом Redis (250 мс) и + circuit breaker'ом; logout по-прежнему отзывает сессию. +* Разрыв сети API <-> Redis БЕЗ рестарта Redis (данные кэша живы) и logout + во время разрыва -> logout 204, и отозванный токен получает 401 и во время + разрыва, и сразу после восстановления связи, и после закрытия circuit + breaker (регрессия: раньше доверяли старой положительной записи ~60 с). +* PostgreSQL недоступна -> /health/live = 200, /health/ready = 503, + запросы к данным -> 503 unavailable за ограниченное время; анонимный + запрос -> 401; неизвестный токен не принимается (503, а не 200). +* После возврата зависимости тот же токен снова работает, данные на месте. +""" +import argparse +import json +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import actor, basic, create_chat, login, request # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +MAX_FAIL_SECONDS = 6.0 # верхняя граница ответа при отказе зависимости +MAX_DEGRADED_SECONDS = 1.5 # верхняя граница ответа при отказе Redis + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, condition, detail=""): + self.results.append({"check": name, "ok": bool(condition), "detail": detail}) + print(("PASS " if condition else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + return condition + + @property + def ok(self): + return all(r["ok"] for r in self.results) + + +def run(cmd): + print("$ " + cmd, flush=True) + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_ready(timeout=90): + deadline = time.monotonic() + timeout + started = time.monotonic() + while time.monotonic() < deadline: + try: + if request("GET", "/health/ready", timeout=3).status == 200: + return time.monotonic() - started + except OSError: + pass + time.sleep(0.5) + return None + + +def is_unavailable(r): + return (r.status == 503 and r.json and r.json["error"]["code"] == "unavailable" + and r.json["error"]["request_id"] == r.headers.get("X-Request-Id")) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--base-url", default="http://127.0.0.1:8080") + p.add_argument("--stop-redis", default="docker compose stop redis") + p.add_argument("--start-redis", default="docker compose up -d --wait redis") + p.add_argument("--partition-redis", + default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1", + help="cut API<->Redis network WITHOUT restarting Redis") + p.add_argument("--heal-redis", + default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") + p.add_argument("--stop-postgres", default="docker compose stop postgres") + p.add_argument("--start-postgres", default="docker compose up -d --wait postgres") + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + + check = Check() + summary = {"base_url": common.BASE_URL, "timings": {}} + + alice, bob = actor("dep"), actor("dep") + chat = create_chat(alice, bob) + path = "/api/v1/chats/%s/messages" % chat["id"] + first = request("POST", path, token=alice["_token"], payload={"text": "до отказов"}).json + request("GET", "/api/v1/users/me", token=alice["_token"]) # прогрев кэша + + # 1. Redis недоступен + run(args.stop_redis) + try: + r = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("redis down: existing token still valid via PostgreSQL", r.status == 200, "%.3fs" % r.elapsed) + summary["timings"]["redis_down_first_request_s"] = round(r.elapsed, 3) + timings = [] + for i in range(10): + r = request("POST", path, token=bob["_token"], payload={"text": "redis down %d" % i}) + timings.append(r.elapsed) + if r.status != 201: + break + check("redis down: writes succeed", r.status == 201) + check("redis down: bounded latency", max(timings) < MAX_DEGRADED_SECONDS, + "max %.3fs over %d requests" % (max(timings), len(timings))) + summary["timings"]["redis_down_max_write_s"] = round(max(timings), 3) + check("redis down: readiness stays 200 (cache is not critical)", + request("GET", "/health/ready").status == 200) + token = login(bob) + check("redis down: login works", bool(token)) + check("redis down: logout works", + request("DELETE", "/api/v1/auth/sessions/current", token=token).status == 204) + check("redis down: revoked token rejected", + request("GET", "/api/v1/users/me", token=token).status == 401) + check("redis down: anonymous rejected", request("GET", "/api/v1/users/me").status == 401) + check("redis down: unknown token rejected", + request("GET", "/api/v1/users/me", token="x" * 43).status == 401) + finally: + run(args.start_redis) + time.sleep(6) # больше CACHE_RETRY_AFTER_SECONDS: API снова пробует Redis + check("redis back: requests ok", + request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 2. Разрыв связи с живым Redis + logout (замечание ревью №1) + carol = actor("dep") + for _ in range(2): # промах -> запись в кэш, затем попадание + request("GET", "/api/v1/users/me", token=carol["_token"]) + run(args.partition_redis) + try: + r = request("DELETE", "/api/v1/auth/sessions/current", token=carol["_token"]) + check("redis partition: logout -> 204", r.status == 204, "status %d, %.3fs" % (r.status, r.elapsed)) + check("redis partition: revoked token -> 401", + request("GET", "/api/v1/users/me", token=carol["_token"]).status == 401) + finally: + run(args.heal_redis) + # Опрашиваем дольше CACHE_RETRY_AFTER_SECONDS (5 с): после закрытия circuit + # breaker API снова читает Redis, где жила положительная запись этого токена. + statuses = set() + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + statuses.add(request("GET", "/api/v1/users/me", token=carol["_token"]).status) + statuses.add(request("POST", path, token=carol["_token"], payload={"text": "revoked"}).status) + time.sleep(0.25) + check("redis healed (same data): revoked token stays 401 for 8 s", statuses == {401}, str(sorted(statuses))) + check("redis healed: valid token works", request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 3. PostgreSQL недоступна + run(args.stop_postgres) + try: + check("postgres down: liveness 200", request("GET", "/health/live").status == 200) + r = request("GET", "/health/ready") + check("postgres down: readiness 503 unavailable", is_unavailable(r), "%.3fs" % r.elapsed) + r = request("GET", "/api/v1/users/me") + check("postgres down: anonymous -> 401 (not trusted)", r.status == 401, "status %d" % r.status) + r = request("GET", "/api/v1/users/me", token="y" * 43) + check("postgres down: unknown token is not accepted (503)", is_unavailable(r), + "status %d, %.3fs" % (r.status, r.elapsed)) + r = request("POST", path, token=alice["_token"], payload={"text": "должно не записаться"}) + check("postgres down: write -> 503 within bound", is_unavailable(r) and r.elapsed < MAX_FAIL_SECONDS, + "status %d, %.3fs" % (r.status, r.elapsed)) + summary["timings"]["postgres_down_write_s"] = round(r.elapsed, 3) + r = request("GET", path, token=alice["_token"]) + check("postgres down: history read -> 503", is_unavailable(r), "status %d" % r.status) + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(alice["username"], alice["_password"])}) + check("postgres down: login -> 503 (not 401, not 201)", is_unavailable(r), "status %d" % r.status) + body = json.dumps(r.json, ensure_ascii=False) + check("postgres down: error hides internals", + not any(s in body.lower() for s in ("psycopg", "postgres", "select", "traceback", "5432"))) + finally: + run(args.start_postgres) + recovered = wait_ready() + check("postgres back: readiness recovered", recovered is not None, + "%.1fs" % recovered if recovered is not None else "timeout") + summary["timings"]["postgres_recovery_s"] = round(recovered or -1, 2) + r = request("GET", path + "?limit=100", token=alice["_token"]) + ok = r.status == 200 and r.json["items"][0] == first + texts = [m["text"] for m in r.json["items"]] if r.status == 200 else [] + check("postgres back: same token reads intact history", ok) + check("postgres back: rejected write was not stored", "должно не записаться" not in texts) + + summary["checks"] = check.results + summary["passed"] = check.ok + if args.output: + Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(("PASS" if check.ok else "FAIL") + ": dependency failure scenario") + sys.exit(0 if check.ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 0000000..18e85e0 --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Уровень 4: последовательная остановка КАЖДОЙ API-реплики под нагрузкой. + +Для каждой реплики (по X-Instance-Id): нагрузка -> stop-one -> 15 с работы без неё +-> restore -> возврат реплики в ротацию. Собираются RPS, ошибки, задержки по +окнам, время обнаружения отказа балансировщиком (по странице HAProxy) и время +восстановления. Перед следующей репликой предыдущая уже восстановлена. + + python student-tests/failover_series.py --output evidence/failover_series.json + python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json + +Остановка/восстановление по умолчанию — scripts/ops.py (docker compose); +для другого стенда их можно переопределить --stop-cmd / --restore-cmd. +""" +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import request # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def observe_instances(token, n=40): + seen = set() + for _ in range(n): + r = request("GET", "/api/v1/users/me", token=token, timeout=3) + seen.add(r.headers.get("X-Instance-Id")) + return sorted(x for x in seen if x) + + +def run_hook(cmd, env_extra, timeout): + env = dict(os.environ, **env_extra) + started = time.monotonic() + proc = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=timeout) + return proc.returncode, time.monotonic() - started, proc.stdout[-2000:] + + +def one_round(args, target): + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + up_before = load.lb[-1][1] if load.lb else () + t_stop = load.now() + rc, stop_s, out = run_hook(args.stop_cmd, {"TARGET_INSTANCE_ID": target, "STOP_MODE": args.mode}, 60) + if rc != 0: + load.stop() + raise SystemExit("stop hook failed for %s:\n%s" % (target, out)) + time.sleep(max(0.0, args.post - stop_s)) + t_restore = load.now() + rc, restore_s, out = run_hook(args.restore_cmd, {}, 180) + if rc != 0: + load.stop() + raise SystemExit("restore hook failed:\n%s" % out) + # ждём, пока восстановленная реплика снова обслужит запрос + back = None + deadline = time.monotonic() + 30 + while back is None and time.monotonic() < deadline: + for e in reversed(load.events): + if e[0] >= t_restore and e[2] and e[4] == target: + back = e[0] + break + time.sleep(0.2) + time.sleep(args.tail) + load.stop() + t_end = load.now() + + detect = next((t - t_stop for t, up in load.lb if t >= t_stop and len(up) < len(up_before)), None) + lb_back = next((t - t_restore for t, up in load.lb if t >= t_restore and len(up) >= len(up_before)), None) + errs = load.errors_between(t_stop, t_restore) + # Сколько ещё после SIGTERM реплика отвечала (фаза осушения), до restore. + served_after = [e[0] - t_stop for e in load.events if t_stop <= e[0] < t_restore and e[2] and e[4] == target] + # После возврата hook (реплика остановлена) и до restore она не должна отвечать. + after_stop_ok = [e for e in load.events if t_stop + stop_s <= e[0] < t_restore and e[2]] + return { + "target_instance": target, + "mode": args.mode, + "lb_up_before": list(up_before), + "stop_hook_seconds": round(stop_s, 2), + "lb_detect_down_seconds": round(detect, 2) if detect is not None else None, + "target_last_served_after_stop_seconds": round(max(served_after), 2) if served_after else None, + "target_served_after_hook_returned": any(e[4] == target for e in after_stop_ok), + "errors_during_outage": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "error_window_seconds": round(max(e[0] for e in errs) - t_stop, 2) if errs else 0.0, + "restore_hook_seconds": round(restore_s, 2), + "lb_back_up_seconds": round(lb_back, 2) if lb_back is not None else None, + "target_back_in_rotation_seconds": round(back - t_restore, 2) if back is not None else None, + "windows": { + "before": load.window(0, t_stop), + "outage_first_10s": load.window(t_stop, t_stop + 10), + "outage_rest": load.window(t_stop + 10, t_restore), + "after_restore": load.window(t_restore, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + } + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--stop-cmd", default=None, help="default: python scripts/ops.py stop-one") + p.add_argument("--restore-cmd", default=None, help="default: python scripts/ops.py restore") + p.add_argument("--mode", choices=["stop", "kill"], default="stop", help="stop = SIGTERM (graceful), kill = SIGKILL") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=15) + p.add_argument("--tail", type=float, default=3) + p.add_argument("--output") + args = p.parse_args() + args.stop_cmd = shlex.split(args.stop_cmd) if args.stop_cmd else [py, "scripts/ops.py", "stop-one"] + args.restore_cmd = shlex.split(args.restore_cmd) if args.restore_cmd else [py, "scripts/ops.py", "restore"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("probe") + instances = observe_instances(probe["_token"]) + if len(instances) < 2: + raise SystemExit("need >= 2 replicas behind LB, observed %s" % instances) + print("replicas observed through LB: %s" % instances, flush=True) + rounds = [] + for target in instances: + print("-- stopping %s (mode=%s) under load" % (target, args.mode), flush=True) + r = one_round(args, target) + rounds.append(r) + w = r["windows"] + print(" hook %.2fs | LB DOWN after %s s | errors %d (%s) | RPS before %.1f, outage %.1f, after restore %.1f" + " | p95 outage %s ms | back in rotation %s s | data ok=%s" + % (r["stop_hook_seconds"], r["lb_detect_down_seconds"], r["errors_during_outage"], + ",".join(r["error_kinds"]) or "-", w["before"]["ok_rps"], w["outage_first_10s"]["ok_rps"], + w["after_restore"]["ok_rps"], w["outage_first_10s"]["latency_ms_ok"]["p95"], + r["target_back_in_rotation_seconds"], r["data"]["ok"]), flush=True) + report = { + "scenario": "stop each API replica in turn under load, restore before the next", + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201"}, + "replicas": instances, + "rounds": rounds, + } + ok = all(r["data"]["ok"] and r["target_back_in_rotation_seconds"] is not None + and not r["target_served_after_hook_returned"] for r in rounds) + report["pass"] = ok + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if ok else "FAIL") + ": every replica stopped and restored; confirmed data intact, no duplicates") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/orient6.jpg b/student-tests/fixtures/orient6.jpg new file mode 100644 index 0000000..c842b15 Binary files /dev/null and b/student-tests/fixtures/orient6.jpg differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000..f0a2c95 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/fixtures/sample.png b/student-tests/fixtures/sample.png new file mode 100644 index 0000000..e7c9207 Binary files /dev/null and b/student-tests/fixtures/sample.png differ diff --git a/student-tests/fixtures/sample_progressive.jpg b/student-tests/fixtures/sample_progressive.jpg new file mode 100644 index 0000000..aa39b16 Binary files /dev/null and b/student-tests/fixtures/sample_progressive.jpg differ diff --git a/student-tests/fixtures/small.png b/student-tests/fixtures/small.png new file mode 100644 index 0000000..ff552a5 Binary files /dev/null and b/student-tests/fixtures/small.png differ diff --git a/student-tests/fixtures/transparent.png b/student-tests/fixtures/transparent.png new file mode 100644 index 0000000..1b909b0 Binary files /dev/null and b/student-tests/fixtures/transparent.png differ diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 0000000..bf6951e --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,93 @@ +"""Синтетические изображения для тестов вложений (только стандартная библиотека). + +PNG собирается вручную (zlib + CRC32), поэтому можно получить и корректные +файлы заданного размера, и намеренно испорченные варианты. JPEG — небольшие +fixtures из student-tests/fixtures (синтетические, сгенерированы Pillow). +""" +import os +import struct +import zlib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +PNG_SIG = b"\x89PNG\r\n\x1a\n" +MAX_UPLOAD = 10 * 1024 * 1024 + + +def chunk(ctype: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF) + + +def ihdr(width, height, depth=8, color=2, interlace=0) -> bytes: + return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)) + + +def png(width=16, height=16, color=2, depth=8, fill=b"\x40", level=6, extra=b"") -> bytes: + channels = {0: 1, 2: 3, 4: 2, 6: 4}[color] + row = b"\x00" + (fill * (width * channels * depth // 8 + 1))[: (width * channels * depth + 7) // 8] + raw = row * height + return PNG_SIG + ihdr(width, height, depth, color) + extra + chunk(b"IDAT", zlib.compress(raw, level)) + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Несжимаемый RGB PNG размером около target_bytes (шум, zlib level 0).""" + width = 1024 + row_bytes = 1 + width * 3 + height = max(1, (target_bytes - 200) // row_bytes) + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + return PNG_SIG + ihdr(width, height) + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"") + + +def png_of_exact_size(size: int) -> bytes: + """Корректный PNG ровно size байт: маленькая картинка + вспомогательный чанк-заполнитель.""" + base = png(8, 8) + pad = size - len(base) - 12 + assert pad >= 0 + filler = chunk(b"zzPd", b"\x00" * pad) # строчная первая буква: вспомогательный, игнорируется + data = base[:8 + 25] + filler + base[8 + 25:] + assert len(data) == size + return data + + +def huge_dimensions_png(width=8000, height=6000) -> bytes: + """48 Мп 1-битная картинка: файл ~50 КБ, распаковка была бы огромной (pixel bomb).""" + row = b"\x00" + b"\x00" * ((width + 7) // 8) + return PNG_SIG + ihdr(width, height, depth=1, color=0) + chunk(b"IDAT", zlib.compress(row * height, 9)) + chunk(b"IEND", b"") + + +def bad_crc_png() -> bytes: + data = bytearray(png(16, 16)) + data[8 + 8 + 13] ^= 0xFF # CRC чанка IHDR + return bytes(data) + + +def truncated_png() -> bytes: + data = png(64, 64, fill=b"\x13\x37\x99") + return data[: len(data) // 2] + + +def wrong_length_png() -> bytes: + """Все CRC верны, но распакованных данных меньше, чем требует IHDR.""" + raw = b"\x00" + b"\x10" * 3 * 4 + return PNG_SIG + ihdr(4, 10) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def sample_png() -> bytes: + # В compose-сети (ops.py in-network) монтируется только student-tests/, + # поэтому копия публичного fixture лежит и в student-tests/fixtures. + public = ROOT / "tests" / "fixtures" / "sample.png" + return (public if public.is_file() else HERE / "fixtures" / "sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (HERE / "fixtures" / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (HERE / "fixtures" / "sample_progressive.jpg").read_bytes() + + +def truncated_jpeg() -> bytes: + data = sample_jpeg() + return data[: len(data) - 400] diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..b2772f9 --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,169 @@ +"""Фоновая нагрузка для сценариев отказа/обновления (только стандартная библиотека). + +Писатели: POST сообщения с Idempotency-Key. Неопределённый результат (обрыв, +timeout, 5xx) повторяется с ТЕМ ЖЕ ключом и телом, как требует контракт, пока +не будет 201. Читатели: GET истории и /users/me. Параллельно опрашивается +страница состояния HAProxy (UP/DOWN реплик), чтобы измерить время обнаружения. + +Каждая попытка записывается как событие (время от старта, тип, успех, статус, +X-Instance-Id, задержка). По событиям считаются RPS/ошибки/перцентили в окнах. +""" +import csv +import io +import math +import threading +import time +import uuid +from urllib.parse import quote +from urllib.request import urlopen + +import common +from common import actor, create_chat, request + + +def percentile(values, p): + if not values: + return None + values = sorted(values) + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) + + +class Load: + def __init__(self, base_url, writers=4, readers=2, timeout=2.0, stats_url=None): + common.BASE_URL = base_url.rstrip("/") + self.writers, self.readers, self.timeout = writers, readers, timeout + self.stats_url = stats_url + self.events = [] # (t, kind, ok, status, instance, latency) + self.lb = [] # (t, tuple(sorted UP servers)) + self.confirmed = [] + self.same_key_retries = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._threads = [] + self.alice = actor("lda") + self.bob = actor("ldb") + self.chat = create_chat(self.alice, self.bob) + self.t0 = None + + def now(self): + return time.monotonic() - self.t0 + + def _record(self, kind, ok, status, instance, latency): + with self._lock: + self.events.append((self.now(), kind, ok, status, instance, latency)) + + def _call(self, kind, method, path, token, payload=None, headers=None): + started = time.monotonic() + try: + r = request(method, path, token=token, payload=payload, headers=headers, timeout=self.timeout) + ok = r.status < 500 + self._record(kind, ok, r.status, r.headers.get("X-Instance-Id"), time.monotonic() - started) + return r + except OSError as exc: # обрыв соединения / timeout — результат неизвестен + self._record(kind, False, type(exc).__name__, None, time.monotonic() - started) + return None + + def _writer(self, wid): + n = 0 + path = "/api/v1/chats/%s/messages" % self.chat["id"] + while not self._stop.is_set(): + key = "lg-%d-%d-%s" % (wid, n, uuid.uuid4().hex[:8]) + body = {"text": "load %d/%d %s" % (wid, n, key)} + for attempt in range(200): + r = self._call("w", "POST", path, self.alice["_token"], body, {"Idempotency-Key": key}) + if r is not None and r.status == 201: + with self._lock: + self.confirmed.append(r.json) + break + if r is not None and r.status < 500: + raise AssertionError("unexpected %s on write: %s" % (r.status, r.json)) + with self._lock: + self.same_key_retries += 1 + time.sleep(0.05) # повтор с тем же ключом и телом + n += 1 + + def _reader(self, rid): + i = 0 + while not self._stop.is_set(): + if i % 2: + self._call("r", "GET", "/api/v1/users/me", self.bob["_token"]) + else: + self._call("r", "GET", "/api/v1/chats/%s/messages?limit=20" % self.chat["id"], self.bob["_token"]) + i += 1 + + def _lb_poller(self): + last = None + while not self._stop.is_set(): + try: + data = urlopen(self.stats_url, timeout=1).read().decode("utf-8", "replace") + rows = csv.DictReader(io.StringIO(data.lstrip("# "))) + up = tuple(sorted(r["svname"] for r in rows if r.get("pxname") == "api" + and r.get("svname") not in ("BACKEND", "FRONTEND") + and r.get("status", "").startswith("UP"))) + if up != last: + with self._lock: + self.lb.append((self.now(), up)) + last = up + except Exception: + pass + time.sleep(0.1) + + def start(self): + self.t0 = time.monotonic() + targets = [(self._writer, i) for i in range(self.writers)] + [(self._reader, i) for i in range(self.readers)] + if self.stats_url: + targets.append((self._lb_poller, None)) + for fn, arg in targets: + t = threading.Thread(target=fn, args=(() if arg is None else (arg,)), daemon=True) + t.start() + self._threads.append(t) + return self + + def stop(self): + self._stop.set() + for t in self._threads: + t.join(timeout=30) + + def window(self, a, b): + ev = [e for e in self.events if a <= e[0] < b] + ok = [e for e in ev if e[2]] + dur = max(b - a, 1e-9) + lat = [e[5] for e in ok] + return { + "from_s": round(a, 2), "to_s": round(b, 2), + "requests": len(ev), "ok": len(ok), "errors": len(ev) - len(ok), + "error_ratio": round((len(ev) - len(ok)) / len(ev), 4) if ev else None, + "ok_rps": round(len(ok) / dur, 1), + "write_ok_rps": round(sum(1 for e in ok if e[1] == "w") / dur, 1), + "latency_ms_ok": {"p50": percentile(lat, .5), "p95": percentile(lat, .95), "p99": percentile(lat, .99)}, + "instances": sorted({e[4] for e in ok if e[4]}), + } + + def errors_between(self, a, b): + return [e for e in self.events if a <= e[0] < b and not e[2]] + + def verify(self): + """Все подтверждённые сообщения на месте, дублей (в т.ч. от повторов) нет.""" + items, cursor = [], None + while True: + path = "/api/v1/chats/%s/messages?limit=100" % self.chat["id"] + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=self.bob["_token"], timeout=10).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + by_id = {m["id"]: m for m in items} + missing = [m for m in self.confirmed if by_id.get(m["id"]) != m] + return { + "confirmed_writes": len(self.confirmed), + "stored_messages": len(items), + "missing_or_altered": len(missing), + "duplicate_ids": len(ids) - len(set(ids)), + "duplicate_texts": len(texts) - len(set(texts)), + "same_key_retries": self.same_key_retries, + "ok": not missing and len(ids) == len(set(ids)) and len(texts) == len(set(texts)), + } diff --git a/student-tests/race.py b/student-tests/race.py new file mode 100644 index 0000000..655b54a --- /dev/null +++ b/student-tests/race.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Конкурентная запись: 10 клиентов x 10 сообщений (перенесено из lab1, Bearer).""" +import argparse +import base64 +import json +import threading +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import urlencode + + +def request(base_url, method, path, token=None, body=None, headers=None): + headers = dict(headers or {}) + headers["Accept"] = "application/json" + + if token: + headers["Authorization"] = "Bearer " + token + + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + base_url + path, + data=data, + headers=headers, + method=method, + ) + + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + +def create_user(base_url, name): + username = "race_%s_%s" % (name, uuid.uuid4().hex[:8]) + password = "Race-password-" + uuid.uuid4().hex + + status, user = request( + base_url, + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "Race " + name, + "password": password, + }, + ) + assert status == 201 + + credential = base64.b64encode((username + ":" + password).encode()).decode() + status, session = request( + base_url, + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert status == 201 + user["token"] = session["token"] + return user + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-url", + default="http://127.0.0.1:8080", + ) + args = parser.parse_args() + + base_url = args.base_url.rstrip("/") + + users = [ + create_user(base_url, str(i)) + for i in range(10) + ] + + owner = users[0] + + status, chat = request( + base_url, + "POST", + "/api/v1/chats", + token=owner["token"], + body={ + "title": "Concurrent race test", + "member_ids": [user["id"] for user in users], + }, + ) + + assert status == 201 + + chat_id = chat["id"] + + barrier = threading.Barrier(10) + + def worker(worker_id): + user = users[worker_id] + created = [] + + barrier.wait() + + for message_id in range(10): + text = "race-%d-%d" % (worker_id, message_id) + + status, message = request( + base_url, + "POST", + "/api/v1/chats/%s/messages" % chat_id, + token=user["token"], + body={"text": text}, + headers={"Idempotency-Key": "race-%d-%d-%s" % (worker_id, message_id, uuid.uuid4().hex[:8])}, + ) + + assert status == 201 + created.append(message) + + return created + + with ThreadPoolExecutor(max_workers=10) as executor: + results = list(executor.map(worker, range(10))) + + created = [ + message + for worker_messages in results + for message in worker_messages + ] + + assert len(created) == 100 + + created_ids = {message["id"] for message in created} + created_texts = {message["text"] for message in created} + + assert len(created_ids) == 100 + assert len(created_texts) == 100 + + messages = [] + seen_cursors = set() + cursor = None + page_limit = 13 + + while True: + path = ( + "/api/v1/chats/%s/messages?limit=%d" + % (chat_id, page_limit) + ) + + if cursor is not None: + path += "&" + urlencode({"cursor": cursor}) + + status, page = request( + base_url, + "GET", + path, + token=owner["token"], + ) + + assert status == 200 + + page_messages = page["items"] + messages.extend(page_messages) + + next_cursor = page["next_cursor"] + + if next_cursor is None: + break + + assert next_cursor not in seen_cursors, ( + "Cursor повторился: возможен бесконечный цикл пагинации" + ) + + seen_cursors.add(next_cursor) + cursor = next_cursor + + assert len(messages) == 100 + + history_ids = {message["id"] for message in messages} + history_texts = {message["text"] for message in messages} + + assert len(history_ids) == 100 + assert len(history_texts) == 100 + + assert history_ids == created_ids + assert history_texts == created_texts + + user_ids = {user["id"] for user in users} + + assert all( + message["sender_id"] in user_ids + for message in messages + ) + + print( + "PASS: concurrent write test; " + "10 clients x 10 messages = 100 messages; " + "no lost or duplicate messages" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/revocation_partition.py b/student-tests/revocation_partition.py new file mode 100644 index 0000000..1193a24 --- /dev/null +++ b/student-tests/revocation_partition.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Минимальное воспроизведение замечания ревью №1 (отзыв токена при разрыве с Redis). + +Прогрев кэша -> разрыв сети API<->Redis без рестарта Redis -> logout -> +восстановление связи -> 8 с опроса старого токена (дольше circuit breaker). +Ожидается только 401. По умолчанию — команды docker compose-стенда: + python student-tests/revocation_partition.py --base-url http://127.0.0.1:8080 +""" +import argparse, shlex, subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common +from common import actor, request + +p = argparse.ArgumentParser() +p.add_argument("--base-url", default="http://127.0.0.1:8080") +p.add_argument("--partition", default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1") +p.add_argument("--heal", default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") +args = p.parse_args() +common.BASE_URL = args.base_url.rstrip("/") +ROOT = Path(__file__).resolve().parents[1] +def run(cmd): + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True) +a = actor("repro"); t = a["_token"] +for _ in range(2): request("GET","/api/v1/users/me",token=t) +run(args.partition) +try: + print("logout during partition:", request("DELETE","/api/v1/auth/sessions/current",token=t).status) +finally: + run(args.heal) +seen=set(); end=time.monotonic()+8 +while time.monotonic() healthy/UP -> graceful stop старой, и так N раз). +Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые +реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS, +ошибки и задержки до/во время/после. + + python student-tests/rolling_update_load.py --output evidence/rolling_update.json +""" +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from failover_series import observe_instances # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=8) + p.add_argument("--output") + args = p.parse_args() + cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("rprobe") + before = observe_instances(probe["_token"]) + print("replicas before: %s" % before, flush=True) + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + t_upd = load.now() + proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200) + t_done = load.now() + print(proc.stdout.strip()[-3000:], flush=True) + time.sleep(args.post) + load.stop() + t_end = load.now() + after = observe_instances(probe["_token"]) + tail_instances = load.window(t_end - 3, t_end)["instances"] + errs = load.errors_between(t_upd, t_done) + report = { + "scenario": "rolling update of all API replicas under continuous load", + "update_command": " ".join(cmd[1:] if cmd[0] == py else cmd), + "update_exit_code": proc.returncode, + "update_seconds": round(t_done - t_upd, 2), + "replicas_before": before, + "replicas_after": after, + "all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)), + "errors_during_update": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "windows": { + "before": load.window(0, t_upd), + "during_update": load.window(t_upd, t_done), + "after": load.window(t_done, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0}, + } + report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"] + w = report["windows"] + print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f" + " | p95 during %s ms | data ok=%s" + % (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-", + w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"], + w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"])) + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_all.py b/student-tests/run_all.py new file mode 100644 index 0000000..349d113 --- /dev/null +++ b/student-tests/run_all.py @@ -0,0 +1,51 @@ +"""Запуск собственных тестов, которые идут по HTTP через балансировщик. + + python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +Входит: API/auth/конкурентность (lab2), идемпотентность через реплики (lab3), +race.py, вложения lab4 (test_attachments.py, lab5: 202 + ожидание ready), варианты lab5 +(test_pipeline.py). Отказы pipeline (test_pipeline_faults.py) здесь пропускаются — нужен доступ +к БД/брокеру и стенд отказов: python scripts/ops.py pipeline-faults. Тест согласованности БД<->S3 +(test_attachment_consistency.py) здесь пропускается: нужен доступ к БД и S3 — +python scripts/ops.py in-network consistency. Отдельно (меняют состояние стенда или нужен доступ к БД): + python scripts/ops.py in-network storage # тесты хранилища внутри compose-сети + python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl) + python student-tests/failover_series.py # остановка каждой реплики под нагрузкой + python student-tests/rolling_update_load.py # rolling update под нагрузкой + python student-tests/dependency_failure.py # отказ Redis/PostgreSQL (останавливает контейнеры) +""" +import argparse +import os +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--base-url", default="http://127.0.0.1:8080") +parser.add_argument("--expiry-url", default=os.environ.get("EXPIRY_BASE_URL", "")) +parser.add_argument("--with-failures", action="store_true", + help="run student-tests/dependency_failure.py (stops redis/postgres)") +parser.add_argument("--failure-args", default="", + help="extra arguments for dependency_failure.py (custom stop/start commands)") +args = parser.parse_args() + +env = os.environ.copy() +env["BASE_URL"] = args.base_url.rstrip("/") +if args.expiry_url: + env["EXPIRY_BASE_URL"] = args.expiry_url.rstrip("/") + + +def step(title, cmd): + print("\n== " + title, flush=True) + subprocess.run(cmd, env=env, check=True) + + +step("API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env)", + [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_*.py", "-v"]) +step("Concurrent write test", [sys.executable, "student-tests/race.py", "--base-url", args.base_url]) +if args.with_failures: + import shlex + step("Dependency failure scenario", + [sys.executable, "student-tests/dependency_failure.py", "--base-url", args.base_url] + + shlex.split(args.failure_args)) + +print("\nALL STUDENT TESTS PASSED", flush=True) diff --git a/student-tests/s3_outage.py b/student-tests/s3_outage.py new file mode 100644 index 0000000..a0ea48f --- /dev/null +++ b/student-tests/s3_outage.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Недоступность S3: деградация только операций с файлами, восстановление без рестарта API. + + python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json + +По умолчанию останавливает/запускает сервис s3 через docker compose и ждёт +прогрева чтения/записи (python scripts/ops.py wait-s3). Для другого стенда — +--stop-cmd / --start-cmd. Тома не удаляются. + +Ожидаемое поведение (S3 НЕ входит в readiness, см. REPORT.md): + * /health/ready = 200, обе реплики остаются в LB; + * регистрация, вход, сообщения и история — работают; + * GET метаданных вложения — 200 (это PostgreSQL); + * POST вложения и GET /content -> 503 unavailable (+ Retry-After) за ограниченное + время (таймауты клиента S3); pending-строку удаляет компенсация + (проверяется в test_attachment_consistency.py через api-fault-s3); + * после возврата S3 тот же токен загружает и скачивает файлы, старый файл цел. +""" +import argparse +import hashlib +import http.client +import json +import os +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import urlparse + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 +from common import actor, create_chat, request # noqa: E402 + +ROOT = HERE.parent +MAX_FAIL_SECONDS = 15.0 # connect 2 с x 2 попытки + backoff; ответ клиенту не позже + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, ok, detail=""): + self.results.append({"check": name, "ok": bool(ok), "detail": detail}) + print(("PASS " if ok else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + + +def run(cmd): + print("$ " + " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=180) + + +def upload(token, chat_id, data): + return request("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat_id, token=token, raw=data, + headers={"Content-Type": "image/png"}, timeout=40) + + +def download(token, att_id): + u = urlparse(common.BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=40) + t = time.monotonic() + conn.request("GET", "/api/v1/attachments/%s/content" % att_id, headers={"Authorization": "Bearer " + token}) + r = conn.getresponse() + body = r.read() + conn.close() + return r.status, body, time.monotonic() - t, r.headers + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + py = sys.executable or "python3" + p.add_argument("--stop-cmd", default="docker compose --env-file .env stop s3") + p.add_argument("--start-cmd", default="docker compose --env-file .env start s3") + p.add_argument("--wait-cmd", default="%s scripts/ops.py wait-s3" % shlex.quote(py)) + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + check = Check() + + alice, bob = actor("s3o_a"), actor("s3o_b") + chat = create_chat(alice, bob) + original = imagegen.sample_png() + r = upload(alice["_token"], chat["id"], original) + check("baseline upload 202 (lab5)", r.status == 202, str(r.status)) + att = common.wait_ready(alice["_token"], r.json["id"]) # lab5: обработку делает worker + + run(shlex.split(args.stop_cmd)) + try: + ready = request("GET", "/health/ready") + check("readiness stays 200 while S3 is down (S3 is not a readiness dependency)", ready.status == 200, + str(ready.status)) + m = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "S3 лежит, а чат работает"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("text message 201 during S3 outage", m.status == 201, str(m.status)) + h = request("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], token=bob["_token"]) + check("history 200 during S3 outage", h.status == 200, str(h.status)) + meta = request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]) + check("attachment metadata 200 (PostgreSQL only)", meta.status == 200, str(meta.status)) + ref = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("message referencing existing attachment 201", ref.status == 201, str(ref.status)) + + for i in range(2): + u = upload(alice["_token"], chat["id"], imagegen.png(10 + i, 10)) + code = (u.json or {}).get("error", {}).get("code") + check("upload #%d -> 503 unavailable" % (i + 1), u.status == 503 and code == "unavailable", + "%s %s in %.2fs" % (u.status, code, u.elapsed)) + check("upload #%d failed fast (<= %.0fs)" % (i + 1, MAX_FAIL_SECONDS), u.elapsed <= MAX_FAIL_SECONDS, + "%.2fs" % u.elapsed) + check("upload #%d has Retry-After" % (i + 1), bool(u.headers.get("Retry-After"))) + status, _, elapsed, _ = download(bob["_token"], att["id"]) + check("download -> 503 during outage", status == 503, "%s in %.2fs" % (status, elapsed)) + check("download failed fast", elapsed <= MAX_FAIL_SECONDS, "%.2fs" % elapsed) + me = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("session still valid", me.status == 200, str(me.status)) + finally: + run(shlex.split(args.start_cmd)) + t0 = time.monotonic() + run(shlex.split(args.wait_cmd)) + recovered = round(time.monotonic() - t0, 2) + + check("S3 read/write warm-up after start", True, "%.2fs" % recovered) + status, body, elapsed, headers = download(bob["_token"], att["id"]) + check("old file downloads byte-for-byte after recovery", + status == 200 and hashlib.sha256(body).hexdigest() == att["sha256"], "%s %.2fs" % (status, elapsed)) + u = upload(alice["_token"], chat["id"], imagegen.png(12, 12)) + check("new upload 202 after recovery (same token, no API restart)", u.status == 202, str(u.status)) + + report = {"scenario": "S3 outage and recovery", "max_fail_seconds": MAX_FAIL_SECONDS, + "s3_warmup_seconds": recovered, "checks": check.results, + "pass": all(c["ok"] for c in check.results)} + if args.output: + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS" if report["pass"] else "FAIL") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..5943901 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,138 @@ +"""Поведенческие тесты API, перенесённые из lab1 и переведённые на Bearer.""" +import base64 +import json +import unittest +import uuid +from urllib.parse import urlencode + +from common import actor, create_chat, request + + +class ApiBehaviorTests(unittest.TestCase): + + def test_malformed_json_returns_400(self): + r = request("POST", "/api/v1/users", raw=b'{"username": ') + self.assertEqual(r.status, 400, r.json) + self.assertIn("request_id", r.json["error"]) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def test_registration_without_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "nopass_" + uuid.uuid4().hex[:10], + "display_name": "No password", + }) + self.assertEqual(r.status, 400, r.json) + + def test_short_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "short_" + uuid.uuid4().hex[:10], + "display_name": "Short", + "password": "elevenchars", + }) + self.assertEqual(r.status, 400, r.json) + + def test_password_never_returned(self): + r = request("POST", "/api/v1/users", payload={ + "username": "secret_" + uuid.uuid4().hex[:10], + "display_name": "Secret", + "password": "Very-secret-password-42", + }) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"id", "username", "display_name", "created_at"}) + + def test_duplicate_username_returns_409(self): + payload = { + "username": "duplicate_" + uuid.uuid4().hex[:10], + "display_name": "First User", + "password": "Duplicate-pass-123", + } + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 201) + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 409) + + def test_empty_message_returns_400(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], + token=owner["_token"], payload={"text": " "}) + self.assertEqual(r.status, 400, r.json) + + def test_sender_id_in_body_rejected(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=owner["_token"], + payload={"text": "hi", "sender_id": owner["id"]}) + self.assertEqual(r.status, 400, r.json) + + def test_outsider_cannot_access_chat(self): + owner, member, outsider = actor(), actor(), actor() + chat = create_chat(owner, member) + chat_path = "/api/v1/chats/%s" % chat["id"] + self.assertEqual(request("GET", chat_path, token=outsider["_token"]).status, 404) + self.assertEqual(request("GET", chat_path + "/messages", token=outsider["_token"]).status, 404) + r = request("POST", chat_path + "/messages", token=outsider["_token"], + payload={"text": "Unauthorized message"}) + self.assertEqual(r.status, 404) + self.assertEqual(request("GET", chat_path, token=member["_token"]).status, 200) + + def test_naive_timestamp_cursor_returns_400(self): + owner = actor() + chat = create_chat(owner) + payload = { + "resource_id": chat["id"], + "created_at": "2026-10-02T12:00:00", + "item_id": str(uuid.uuid4()), + } + encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat["id"], urlencode({"cursor": encoded})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_cursor_from_other_chat_returns_400(self): + owner = actor() + chat_a, chat_b = create_chat(owner), create_chat(owner) + for i in range(3): + request("POST", "/api/v1/chats/%s/messages" % chat_a["id"], token=owner["_token"], + payload={"text": "m%d" % i}) + page = request("GET", "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], token=owner["_token"]).json + self.assertTrue(page["next_cursor"]) + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat_b["id"], urlencode({"cursor": page["next_cursor"]})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_pagination_returns_all_messages_without_duplicates(self): + owner = actor() + chat = create_chat(owner) + path = "/api/v1/chats/%s/messages" % chat["id"] + expected = [] + for index in range(8): + r = request("POST", path, token=owner["_token"], payload={"text": "page-%d" % index}) + self.assertEqual(r.status, 201, r.json) + expected.append(r.json["id"]) + received, cursor = [], None + while True: + query = {"limit": 3} + if cursor: + query["cursor"] = cursor + page = request("GET", path + "?" + urlencode(query), token=owner["_token"]).json + received.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(received, expected) + + def test_chat_list_pagination(self): + owner = actor() + created = [create_chat(owner)["id"] for _ in range(5)] + got, cursor = [], None + while True: + q = "?limit=2" + ("&" + urlencode({"cursor": cursor}) if cursor else "") + page = request("GET", "/api/v1/chats" + q, token=owner["_token"]).json + got.extend(c["id"] for c in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(got, created) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_attachment_consistency.py b/student-tests/test_attachment_consistency.py new file mode 100644 index 0000000..d4c36aa --- /dev/null +++ b/student-tests/test_attachment_consistency.py @@ -0,0 +1,310 @@ +"""Согласованность PostgreSQL <-> S3 для вложений (нужен прямой доступ к БД и S3). + +lab5: API пишет байты в staging/ (а не originals/), строка pending ссылается на +staging_key; успешная загрузка отвечает 202. Логика сборщика та же. + +Запуск внутри compose-сети (БД и S3 наружу не публикуются): + python scripts/ops.py in-network consistency +Без DATABASE_URL(_FILE) и S3-настроек тесты пропускаются. + +Что доказывается: + * сбой между S3 PUT и COMMIT (pending + объект) обнаруживается dry-run'ом, + исправляется --apply, повторный запуск ничего не меняет (идемпотентность); + * свежие pending (моложе grace) и ready/используемые вложения не удаляются; + * объект-сирота (нет строки) удаляется только после grace; + * ready без объекта — только отчёт (missing_objects), без авто-удаления; + * обрыв загрузки клиентом не оставляет ни строки, ни объекта; + * внедрённые отказы в API (если заданы FAULT_*_URL — экземпляры api-fault-*): + ошибка S3 PUT -> 503 и компенсация; ошибка БД после PUT -> 503, объект удалён; + падение процесса после PUT -> остаётся pending+объект, их убирает сборщик. +""" +import hashlib +import json +import os +import socket +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +sys.path.insert(0, str(HERE)) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +HAS_S3 = bool(os.environ.get("S3_ENDPOINT")) +GRACE = 900 + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class AttachmentConsistencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool + from src.objectstore import ObjectStore + from src.security import hash_password + from src.storage import Storage + from src import reconcile + + cls.settings = load_settings() + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.store = ObjectStore(cls.settings) + cls.rec = reconcile + user = cls.storage.create_user("cons_" + uuid.uuid4().hex[:16], "Consistency", hash_password("Consistency-pw-1")) + cls.user = user + cls.chat = cls.storage.create_chat("consistency", [user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + # helpers + + def pending_with_object(self, age_seconds=0, put=True): + att_id = uuid.uuid4() + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + data = b"\x89PNG fake bytes " + uuid.uuid4().bytes + ok = self.storage.create_pending_attachment(att_id, self.chat.id, self.user.id, "p.png", "image/png", + len(data), hashlib.sha256(data).hexdigest(), 1, 1, key) + self.assertTrue(ok) + if put: + self.store.put(key, data, "image/png", hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest()) + if age_seconds: + self.backdate(att_id, age_seconds) + return att_id, key + + def backdate(self, att_id, seconds): + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - make_interval(secs => %s) WHERE id = %s", + (seconds, att_id)) + + def row(self, att_id): + with self.pool.connection() as conn: + return conn.execute("SELECT state FROM attachments WHERE id = %s", (att_id,)).fetchone() + + def run_gc(self, apply, grace=GRACE, sweep=True): + report = None + for _ in range(50): # фоновый GC реплики мог держать lock + report = self.rec.run_once(self.pool, self.store, grace, apply, orphan_sweep=sweep) + if report is not None: + return report + time.sleep(0.2) + self.fail("reconciler lock is busy") + + # tests + + def test_crash_between_put_and_commit_dry_run_then_apply_then_noop(self): + att_id, key = self.pending_with_object(age_seconds=GRACE + 60) + dry = self.run_gc(apply=False, sweep=False) + self.assertIn(str(att_id), dry["stale_pending_ids"]) + self.assertIsNotNone(self.row(att_id), "dry-run must not change anything") + self.assertIsNotNone(self.store.head(key)) + applied = self.run_gc(apply=True, sweep=False) + self.assertGreaterEqual(applied["stale_pending_rows_deleted"], 1) + self.assertIsNone(self.row(att_id)) + self.assertIsNone(self.store.head(key)) + again = self.run_gc(apply=True, sweep=False) + self.assertNotIn(str(att_id), again["stale_pending_ids"]) + + def test_fresh_pending_is_kept(self): + att_id, key = self.pending_with_object(age_seconds=0) + self.run_gc(apply=True) + self.assertEqual(self.row(att_id), ("pending",)) + self.assertIsNotNone(self.store.head(key)) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + def test_ready_and_referenced_are_never_collected(self): + att_id, key = self.pending_with_object() + self.assertIsNotNone(self.storage.mark_attachment_ready(att_id)) + msg, _ = self.storage.create_message(self.chat.id, self.user.id, "", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.assertEqual(msg.attachment_ids, [att_id]) + self.backdate(att_id, 10 * GRACE) + self.run_gc(apply=True, grace=GRACE) + self.assertEqual(self.row(att_id), ("ready",)) + self.assertIsNotNone(self.store.head(key)) + # Даже прямое удаление строки запрещено БД, пока на неё ссылается сообщение. + from psycopg import errors + with self.assertRaises(errors.ForeignKeyViolation): + with self.pool.connection() as conn: + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_orphan_object_deleted_only_after_grace(self): + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + self.store.put(key, b"orphan", "image/png", hashlib.md5(b"orphan").digest(), "0" * 64) + report = self.run_gc(apply=True, grace=GRACE) + self.assertGreaterEqual(report["orphan_objects_within_grace"], 1) + self.assertIsNotNone(self.store.head(key), "young orphan must survive (grace period)") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, False, report, verify_ready=False) + self.assertIn(key, report["orphan_objects_sample"]) + self.assertIsNotNone(self.store.head(key), "dry-run must not delete") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertIsNone(self.store.head(key)) + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertNotIn(key, report["orphan_objects_sample"]) + + def test_missing_object_is_reported_not_deleted(self): + att_id, key = self.pending_with_object() + self.storage.mark_attachment_ready(att_id) + self.store.delete(key) # внедрённая потеря байтов + report = self.run_gc(apply=True) + self.assertIn(str(att_id), report["missing_object_ids"]) + self.assertEqual(self.row(att_id), ("ready",)) + with self.pool.connection() as conn: # убрать искусственную запись, на неё нет ссылок + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_pending_cannot_be_attached(self): + att_id, key = self.pending_with_object() + from src.storage import AttachmentNotFound + with self.assertRaises(AttachmentNotFound): + self.storage.create_message(self.chat.id, self.user.id, "x", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + +# ------------------------------------------------------------------ HTTP + DB + +def _http(base, method, path, token=None, body=None, headers=None, timeout=20): + h = dict(headers or {}) + if token: + h["Authorization"] = "Bearer " + token + data = body + if isinstance(body, dict): + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = Request(base.rstrip("/") + path, data=data, headers=h, method=method) + try: + with urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def _actor(base): + import base64 + name = "flt_" + uuid.uuid4().hex[:16] + pw = "Fault-" + uuid.uuid4().hex + s, user = _http(base, "POST", "/api/v1/users", body={"username": name, "display_name": "F", "password": pw}) + assert s == 201, (s, user) + cred = base64.b64encode((name + ":" + pw).encode()).decode() + s, sess = _http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, (s, sess) + s, chat = _http(base, "POST", "/api/v1/chats", sess["token"], {"title": "faults", "member_ids": [user["id"]]}) + assert s == 201, (s, chat) + return user, sess["token"], chat + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class UploadFailureTests(unittest.TestCase): + """Отказы на пути загрузки, наблюдаемые со стороны БД и бакета.""" + + @classmethod + def setUpClass(cls): + AttachmentConsistencyTests.setUpClass.__func__(cls) + import imagegen + cls.png = imagegen.sample_png() + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def rows_of(self, owner_id): + with self.pool.connection() as conn: + return conn.execute("SELECT id, state, COALESCE(staging_key, object_key) FROM attachments " + "WHERE owner_id = %s", + (owner_id,)).fetchall() + + def object_count(self): + total, token = 0, None + while True: + items, token = self.store.list_page("staging/", token) + total += len(items) + if not token: + return total + + def upload(self, base, token, chat): + return _http(base, "POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], token, + self.png, {"Content-Type": "image/png"}) + + @unittest.skipUnless(os.environ.get("BASE_URL"), "BASE_URL is not set") + def test_aborted_upload_leaves_no_row_and_no_object(self): + base = os.environ["BASE_URL"] + user, token, chat = _actor(base) + before = self.object_count() + u = urlparse(base) + sock = socket.create_connection((u.hostname, u.port or 80), timeout=5) + head = ("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: %s\r\n" + "Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n" + % (chat["id"], u.hostname, token, 5 * 1024 * 1024)).encode() + sock.sendall(head + self.png + b"\x00" * 200000) # ~0.2 MiB из заявленных 5 MiB + time.sleep(0.5) + sock.close() # обрыв + time.sleep(2) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before) + s, att = self.upload(base, token, chat) # сервис продолжает работать + self.assertEqual(s, 202, att) + + @unittest.skipUnless(os.environ.get("FAULT_S3_URL"), "FAULT_S3_URL (api-fault-s3) is not set") + def test_s3_put_failure_returns_503_and_compensates(self): + base = os.environ["FAULT_S3_URL"] + user, token, chat = _actor(base) + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), [], "pending row must be removed by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_DB_URL"), "FAULT_DB_URL (api-fault-db) is not set") + def test_db_failure_after_put_returns_503_and_deletes_object(self): + base = os.environ["FAULT_DB_URL"] + user, token, chat = _actor(base) + before = self.object_count() + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before, "object must be deleted by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_CRASH_URL"), "FAULT_CRASH_URL (api-fault-crash) is not set") + def test_process_crash_after_put_is_collected_by_gc(self): + base = os.environ["FAULT_CRASH_URL"] + user, token, chat = _actor(base) + try: + self.upload(base, token, chat) + self.fail("instance was expected to crash") + except (URLError, OSError, ConnectionError): + pass # процесс умер между S3 PUT и COMMIT + rows = self.rows_of(user["id"]) + self.assertEqual(len(rows), 1) + att_id, state, key = rows[0] + self.assertEqual(state, "pending") + self.assertIsNotNone(self.store.head(key), "S3 PUT happened before the crash") + # Через grace сборщик убирает и строку, и объект (ускоряем, сдвигая created_at). + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = %s", + (att_id,)) + from src import reconcile + report = None + for _ in range(50): + report = reconcile.run_once(self.pool, self.store, GRACE, True, orphan_sweep=False) + if report: + break + time.sleep(0.2) + self.assertIn(str(att_id), report["stale_pending_ids"]) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertIsNone(self.store.head(key)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..fcaf779 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,309 @@ +"""Вложения lab4 через балансировщик (HTTP, только стандартная библиотека). + +lab5: загрузка возвращает 202 queued; там, где нужны байты, тест ждёт ready +(common.wait_ready). Варианты проверяет student-tests/test_pipeline.py. + + BASE_URL=http://127.0.0.1:8080 python -m unittest discover -s student-tests -p test_attachments.py -v + +Покрывает уровень 4: слишком большой файл (с Content-Length и chunked без него), +повреждённые изображения, подмену MIME, pixel bomb, опасные filename, чужое +attachment_id, привязку файла другого чата, неизвестный вариант. Во всех +негативных случаях дополнительно проверяется: ответ < 500, общий формат ошибки. +""" +import hashlib +import http.client +import sys +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote, urlparse + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +import imagegen as img # noqa: E402 +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 + +MAX = 10 * 1024 * 1024 + + +def upload(token, chat_id, data, content_type="image/png", filename="photo.png", headers=None): + path = "/api/v1/chats/%s/attachments" % chat_id + if filename is not None: + path += "?filename=" + quote(filename, safe="") + h = {"Content-Type": content_type} + h.update(headers or {}) + return request("POST", path, token=token, raw=data, headers=h, timeout=60) + + +def message(token, chat_id, text, attachment_ids=None, key=None): + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, + headers={"Idempotency-Key": key or uuid.uuid4().hex}) + + +def raw_get(path, token): + """GET, возвращающий байты (common.request декодирует только JSON).""" + u = urlparse(BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=30) + conn.request("GET", path, headers={"Authorization": "Bearer " + token}) + resp = conn.getresponse() + data = resp.read() + conn.close() + return resp.status, resp.headers, data + + +def chunked_upload(token, chat_id, total, piece=256 * 1024): + """Transfer-Encoding: chunked без Content-Length: лимит должен сработать по факту чтения.""" + u = urlparse(BASE_URL) + conn = http.client.HTTPConnection(u.hostname, u.port or 80, timeout=60) + path = "/api/v1/chats/%s/attachments?filename=big.png" % chat_id + conn.putrequest("POST", path) + conn.putheader("Authorization", "Bearer " + token) + conn.putheader("Content-Type", "image/png") + conn.putheader("Transfer-Encoding", "chunked") + conn.endheaders() + first = img.PNG_SIG + b"\x00" * (piece - 8) + sent = 0 + try: + while sent < total: + part = first if sent == 0 else b"\x00" * min(piece, total - sent) + conn.send(b"%x\r\n" % len(part) + part + b"\r\n") + sent += len(part) + conn.send(b"0\r\n\r\n") + except (BrokenPipeError, ConnectionResetError): + pass # сервер мог закрыть приём раньше; ответ всё равно читаем + resp = conn.getresponse() + status = resp.status + resp.read() + conn.close() + return status + + +@unittest.skipUnless(reachable(BASE_URL), "API is not reachable at %s" % BASE_URL) +class AttachmentTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("att_a") + cls.bob = actor("att_b") + cls.eve = actor("att_e") + cls.chat = create_chat(cls.alice, cls.bob) + cls.eve_chat = create_chat(cls.eve) + cls.other_chat = create_chat(cls.alice) # чат Алисы без Боба + + def assertError(self, r, status, code=None): + self.assertEqual(r.status, status, (r.status, r.json)) + self.assertLess(r.status, 500) + self.assertIsInstance(r.json, dict) + self.assertIn("error", r.json) + if code: + self.assertEqual(r.json["error"]["code"], code) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def upload_ok(self, data, content_type="image/png", filename="photo.png", who=None, chat=None, ready=False): + who = who or self.alice + chat = chat or self.chat + r = upload(who["_token"], chat["id"], data, content_type, filename) + self.assertEqual(r.status, 202, (r.status, r.json)) + att = r.json + self.assertEqual(att["status"], "queued") + self.assertEqual(att["variants"], []) + self.assertIsNone(att["error_code"]) + self.assertEqual(att["size_bytes"], len(data)) + self.assertEqual(att["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(att["content_type"], content_type) + self.assertEqual(att["chat_id"], chat["id"]) + self.assertEqual(att["owner_id"], who["id"]) + self.assertNotIn("object_key", att) + if ready: + done = wait_ready(who["_token"], att["id"]) + for k in ("id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "created_at"): + self.assertEqual(done[k], att[k], k) + return done + return att + + # ------------------------------------------------------------- happy path + + def test_png_roundtrip_member_reads_outsider_404(self): + data = img.sample_png() + att = self.upload_ok(data, filename="Отпуск 2026 (1).png", ready=True) + self.assertEqual(att["filename"], "Отпуск 2026 (1).png") + meta = request("GET", "/api/v1/attachments/" + att["id"], token=self.bob["_token"]) + self.assertEqual(meta.status, 200) + self.assertEqual(meta.json, att) + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(body, data) + self.assertEqual(headers.get_content_type(), "image/png") + self.assertEqual(int(headers["Content-Length"]), len(data)) + self.assertEqual(headers.get("X-Content-Type-Options"), "nosniff") + self.assertIn("no-store", headers.get("Cache-Control", "")) + for suffix in ("", "/content"): + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix, token=self.eve["_token"]), + 404, "not_found") + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix), 401, "unauthorized") + + def test_jpeg_baseline_and_progressive(self): + for data in (img.sample_jpeg(), img.progressive_jpeg()): + att = self.upload_ok(data, "image/jpeg", "photo.jpg", ready=True) + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.alice["_token"]) + self.assertEqual((status, body, headers.get_content_type()), (200, data, "image/jpeg")) + + def test_content_type_with_parameters_is_accepted(self): + self.upload_ok(img.png(8, 8), "image/png", "x.png") + r = upload(self.alice["_token"], self.chat["id"], img.png(8, 8), "image/png; charset=binary", "x.png") + self.assertEqual(r.status, 202, r.json) + + def test_exactly_max_size_is_accepted(self): + data = img.png_of_exact_size(MAX) + att = self.upload_ok(data, filename="max.png", ready=True) + status, _, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(hashlib.sha256(body).hexdigest(), att["sha256"]) + + def test_original_variant_and_unknown_variants(self): + att = self.upload_ok(img.png(4, 4), ready=True) + base = "/api/v1/attachments/%s/content" % att["id"] + status, _, _ = raw_get(base + "?variant=original", self.bob["_token"]) + self.assertEqual(status, 200) + # lab5: у ready есть thumbnail (вариант создаёт worker). + status, headers, _ = raw_get(base + "?variant=thumbnail", self.bob["_token"]) + self.assertEqual((status, headers.get_content_type()), (200, "image/jpeg")) + self.assertError(request("GET", base + "?variant=../../etc", token=self.bob["_token"]), 400, "invalid_request") + + # ------------------------------------------------------------- limits + + def test_too_large_with_content_length(self): + r = upload(self.alice["_token"], self.chat["id"], img.png_of_exact_size(MAX + 1), filename="big.png") + self.assertError(r, 413, "payload_too_large") + + def test_too_large_chunked_without_content_length(self): + status = chunked_upload(self.alice["_token"], self.chat["id"], MAX + 512 * 1024) + self.assertEqual(status, 413) + + def test_too_many_pixels(self): + data = img.huge_dimensions_png(8000, 6000) # 48 Мп при размере файла ~50 КБ + self.assertLess(len(data), MAX) + self.assertError(upload(self.alice["_token"], self.chat["id"], data), 413, "payload_too_large") + + # ------------------------------------------------------------- content checks + + def test_corrupted_images_are_415(self): + cases = { + "truncated png": img.truncated_png(), + "bad crc png": img.bad_crc_png(), + "png header + random": img.PNG_SIG + uuid.uuid4().bytes * 64, + "png with wrong data length": img.wrong_length_png(), + "truncated jpeg": img.truncated_jpeg(), + "empty body": b"", + "plain text": "не картинка".encode(), + "gif": b"GIF89a" + b"\x00" * 64, + } + for name, data in cases.items(): + ctype = "image/jpeg" if "jpeg" in name else "image/png" + with self.subTest(name): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_mime_spoofing_is_415(self): + cases = [ + (img.sample_png(), "image/jpeg"), + (img.sample_jpeg(), "image/png"), + (img.sample_png(), "image/gif"), + (img.sample_png(), "application/octet-stream"), + (img.sample_png(), "multipart/form-data; boundary=x"), + (img.sample_png(), "text/html"), + ] + for data, ctype in cases: + with self.subTest(ctype): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_dangerous_filenames_are_400(self): + for name in ("../etc/passwd", "a/b.png", "a\\b.png", "nul\x00.png", "bell\x07.png", "new\nline.png", + "", " ", ".", "..", "x" * 256, "rtl‮gnp.exe"): + with self.subTest(repr(name)): + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=name), + 400, "invalid_request") + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=None), + 400, "invalid_request") + self.upload_ok(img.png(2, 2), filename="x" * 255) + + # ------------------------------------------------------------- access + + def test_outsider_and_unknown_chat_upload_404(self): + self.assertError(upload(self.eve["_token"], self.chat["id"], img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], str(uuid.uuid4()), img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], "not-a-uuid", img.png(2, 2)), 400, "invalid_request") + self.assertError(upload(None, self.chat["id"], img.png(2, 2)), 401, "unauthorized") + + def test_invalid_and_unknown_attachment_ids(self): + self.assertError(request("GET", "/api/v1/attachments/not-a-uuid", token=self.bob["_token"]), 400) + self.assertError(request("GET", "/api/v1/attachments/%s" % uuid.uuid4(), token=self.bob["_token"]), 404) + self.assertError(request("GET", "/api/v1/attachments/%s/content" % uuid.uuid4(), token=self.bob["_token"]), 404) + + # ------------------------------------------------------------- messages + + def test_message_with_attachments_visible_to_members(self): + a1 = self.upload_ok(img.png(3, 3)) + a2 = self.upload_ok(img.sample_jpeg(), "image/jpeg", "b.jpg") + r = message(self.alice["_token"], self.chat["id"], "", [a2["id"], a1["id"]]) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(r.json["text"], "") + self.assertEqual(r.json["attachment_ids"], [a2["id"], a1["id"]]) # порядок сохранён + # Любой участник может сослаться на вложение чата повторно. + r2 = message(self.bob["_token"], self.chat["id"], "и снова", [a1["id"]]) + self.assertEqual(r2.status, 201, r2.json) + page = request("GET", "/api/v1/chats/%s/messages?limit=100" % self.chat["id"], token=self.bob["_token"]) + by_id = {m["id"]: m for m in page.json["items"]} + self.assertEqual(by_id[r.json["id"]]["attachment_ids"], [a2["id"], a1["id"]]) + self.assertEqual(by_id[r2.json["id"]]["attachment_ids"], [a1["id"]]) + text_only = message(self.alice["_token"], self.chat["id"], "без вложений") + self.assertEqual(text_only.json["attachment_ids"], []) + + def test_attachment_from_other_chat_is_404_and_nothing_saved(self): + foreign = self.upload_ok(img.png(2, 2), chat=self.other_chat) # чат Алисы без Боба + key = uuid.uuid4().hex + r = message(self.alice["_token"], self.chat["id"], "чужой файл", [foreign["id"]], key=key) + self.assertError(r, 404, "not_found") + # Отказ не «сжёг» ключ: тот же ключ с корректным телом создаёт сообщение. + ok = message(self.alice["_token"], self.chat["id"], "чужой файл", [], key=key) + self.assertEqual(ok.status, 201, ok.json) + # Посторонняя Ева не может прикрепить файл Алисы в своём чате. + a = self.upload_ok(img.png(2, 2)) + self.assertError(message(self.eve["_token"], self.eve_chat["id"], "x", [a["id"]]), 404, "not_found") + self.assertError(message(self.alice["_token"], self.chat["id"], "x", [str(uuid.uuid4())]), 404) + + def test_message_attachment_validation(self): + a = self.upload_ok(img.png(2, 2)) + cid, tok = self.chat["id"], self.alice["_token"] + self.assertError(message(tok, cid, ""), 400) + self.assertError(message(tok, cid, " "), 400) + self.assertError(message(tok, cid, "dup", [a["id"], a["id"]]), 400) + self.assertError(message(tok, cid, "many", [str(uuid.uuid4()) for _ in range(11)]), 400) + self.assertError(message(tok, cid, "bad", ["not-a-uuid"]), 400) + self.assertError(message(tok, cid, "bad", "not-a-list"), 400) + r = message(tok, cid, " ", [a["id"]]) # пробельный текст допустим с вложением + self.assertEqual(r.status, 201, r.json) + + def test_idempotency_includes_attachment_ids(self): + a1 = self.upload_ok(img.png(2, 2)) + a2 = self.upload_ok(img.png(3, 3)) + key = uuid.uuid4().hex + first = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual(first.status, 201) + replay = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a2["id"], a1["id"]], key=key), 409) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a1["id"]], key=key), 409) + # Отсутствующее поле == [] для идемпотентности. + k2 = uuid.uuid4().hex + m1 = message(self.alice["_token"], self.chat["id"], "без поля", None, key=k2) + m2 = message(self.alice["_token"], self.chat["id"], "без поля", [], key=k2) + self.assertEqual((m2.status, m2.json), (201, m1.json)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 0000000..2a7540a --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,109 @@ +"""Аутентификация: Basic только для входа, Bearer для остального, logout.""" +import base64 +import unittest +import uuid + +from common import actor, basic, login, register, request + + +def assert_challenge(test, r, scheme): + test.assertEqual(r.status, 401, r.json) + test.assertTrue(r.headers.get("WWW-Authenticate", "").startswith(scheme), r.headers.get("WWW-Authenticate")) + test.assertEqual(r.json["error"]["code"], "unauthorized") + + +class AuthTests(unittest.TestCase): + + def test_wrong_password_and_unknown_user_look_the_same(self): + user = register("wp") + wrong = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], "Wrong-password-000")}) + unknown = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic("nouser_" + uuid.uuid4().hex[:10], "Wrong-password-000")}) + for r in (wrong, unknown): + assert_challenge(self, r, 'Basic realm="messenger"') + self.assertIn('charset="UTF-8"', r.headers.get("WWW-Authenticate")) + self.assertEqual(wrong.json["error"]["message"], unknown.json["error"]["message"]) + + def test_malformed_basic_headers(self): + for header in (None, "Basic", "Basic !!!notbase64", "Bearer abc", + "Basic " + base64.b64encode(b"no-colon-here").decode()): + headers = {"Authorization": header} if header else {} + r = request("POST", "/api/v1/auth/sessions", headers=headers) + assert_challenge(self, r, "Basic") + + def test_session_shape_and_token_entropy(self): + user = register("shape") + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"token", "token_type", "expires_at", "user"}) + self.assertEqual(r.json["token_type"], "Bearer") + token = r.json["token"] + # 32 случайных байта в base64url = 43 символа (256 бит) + self.assertGreaterEqual(len(token), 43) + self.assertNotIn(user["username"], token) + self.assertNotIn(user["id"], token) + self.assertTrue(r.json["expires_at"].endswith("Z")) + self.assertNotEqual(token, login(user), "each login must issue a new token") + + def test_password_with_colon_and_unicode(self): + password = "пароль:с:двоеточием-🙂" + user = register("colon", password=password) + token = login(user) + self.assertEqual(request("GET", "/api/v1/users/me", token=token).json["id"], user["id"]) + + def test_login_with_body_rejected(self): + """Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет.""" + user = register("body") + auth = {"Authorization": basic(user["username"], user["_password"])} + cases = [ + dict(payload={"unexpected": True}), + dict(payload={}), + dict(raw=b"not json at all"), + dict(raw=b"username=x&password=y", headers={"Content-Type": "application/x-www-form-urlencoded"}), + ] + for case in cases: + headers = dict(auth) + headers.update(case.pop("headers", {})) + r = request("POST", "/api/v1/auth/sessions", headers=headers, **case) + self.assertEqual(r.status, 400, (case, r.json)) + self.assertEqual(r.json["error"]["code"], "invalid_request") + self.assertNotIn("token", r.json) + # Без тела тот же заголовок работает. + self.assertEqual(request("POST", "/api/v1/auth/sessions", headers=auth).status, 201) + + def test_basic_on_business_endpoint_rejected(self): + user = register("basic") + r = request("GET", "/api/v1/users/me", + headers={"Authorization": basic(user["username"], user["_password"])}) + assert_challenge(self, r, 'Bearer realm="messenger"') + + def test_x_user_id_gives_no_access(self): + user = actor("xuid") + other = actor("xuid") + assert_challenge(self, request("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}), "Bearer") + r = request("GET", "/api/v1/users/me", token=user["_token"], headers={"X-User-Id": other["id"]}) + self.assertEqual(r.json["id"], user["id"]) + + def test_unknown_and_garbage_tokens(self): + for token in ("x" * 43, "", "a b", "z" * 1000): + r = request("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert_challenge(self, r, "Bearer") + + def test_logout_revokes_only_current_session(self): + user = register("logout") + t1, t2 = login(user), login(user) + # Прогреваем кэш Redis для t1, чтобы проверить инвалидацию кэша. + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + r = request("DELETE", "/api/v1/auth/sessions/current", token=t1) + self.assertEqual(r.status, 204) + self.assertIsNone(r.json) + assert_challenge(self, request("GET", "/api/v1/users/me", token=t1), "Bearer") + assert_challenge(self, request("DELETE", "/api/v1/auth/sessions/current", token=t1), "Bearer") + self.assertEqual(request("GET", "/api/v1/users/me", token=t2).status, 200) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 0000000..0411490 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,80 @@ +"""Гонка регистрации одинакового username и rollback составной операции.""" +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import actor, basic, request + +PARALLEL = 16 + + +class RegistrationRaceTests(unittest.TestCase): + + def test_same_username_parallel_registration(self): + username = "race_" + uuid.uuid4().hex[:12] + barrier = threading.Barrier(PARALLEL) + + def attempt(i): + password = "Race-password-%02d-%s" % (i, uuid.uuid4().hex[:6]) + barrier.wait() + r = request("POST", "/api/v1/users", timeout=30, payload={ + "username": username, "display_name": "Racer %d" % i, "password": password, + }) + return r.status, r.json, password + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(attempt, range(PARALLEL))) + + statuses = sorted(s for s, _, _ in results) + self.assertEqual(statuses.count(201), 1, statuses) + self.assertEqual(statuses.count(409), PARALLEL - 1, statuses) + for status, body, _ in results: + if status == 409: + self.assertEqual(body["error"]["code"], "conflict") + + # Победитель входит своим паролем; пароли проигравших не подходят. + winner = next(r for r in results if r[0] == 201) + loser = next(r for r in results if r[0] == 409) + ok = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, winner[2])}) + self.assertEqual(ok.status, 201) + self.assertEqual(ok.json["user"]["id"], winner[1]["id"]) + bad = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, loser[2])}) + self.assertEqual(bad.status, 401) + + +class ChatRollbackTests(unittest.TestCase): + + def _chat_ids(self, owner): + r = request("GET", "/api/v1/chats?limit=100", token=owner["_token"]) + self.assertEqual(r.status, 200) + return [c["id"] for c in r.json["items"]] + + def test_unknown_member_rolls_back_whole_chat(self): + """Чат вставляется первым, затем участники по порядку (сначала сам владелец). + + Неизвестный участник вызывает нарушение внешнего ключа на 2-й строке + chat_members. Без транзакции у владельца остался бы чат с одним + участником; с транзакцией не остаётся ничего. + """ + owner, member = actor("rb"), actor("rb") + before = self._chat_ids(owner) + r = request("POST", "/api/v1/chats", token=owner["_token"], payload={ + "title": "Must roll back", + "member_ids": [owner["id"], member["id"], str(uuid.uuid4())], + }) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), before, "partial chat is visible after failed creation") + self.assertEqual(self._chat_ids(member), [], "partial membership is visible to other member") + + def test_duplicate_and_missing_caller_rejected(self): + owner, member = actor("rb"), actor("rb") + for ids in ([owner["id"], owner["id"]], [member["id"]]): + r = request("POST", "/api/v1/chats", token=owner["_token"], + payload={"title": "bad", "member_ids": ids}) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), []) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..c560140 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,49 @@ +"""Истечение сессии при коротком SESSION_TTL_SECONDS. + +Нужен отдельный экземпляр API с коротким TTL (compose-профиль ttl, порт 8081): + docker compose --profile ttl up -d api-short-ttl + EXPIRY_BASE_URL=http://127.0.0.1:8081 python -m unittest discover -s student-tests -p test_expiry.py +Без EXPIRY_BASE_URL тест пропускается (основной стенд работает с TTL >= 300). +""" +import datetime as dt +import os +import time +import unittest + +from common import basic, register, request + +EXPIRY_BASE_URL = os.environ.get("EXPIRY_BASE_URL", "").rstrip("/") + + +@unittest.skipUnless(EXPIRY_BASE_URL, "set EXPIRY_BASE_URL to the short-TTL API instance") +class ExpiryTests(unittest.TestCase): + + def test_session_expires_even_when_cached(self): + user = register("exp", base_url=EXPIRY_BASE_URL) + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + token = r.json["token"] + expires_at = dt.datetime.fromisoformat(r.json["expires_at"].replace("Z", "+00:00")) + ttl = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds() + self.assertLess(ttl, 30, "EXPIRY_BASE_URL must point to a short-TTL instance") + + # Два запроса подряд: первый кладёт сессию в Redis, второй читает из кэша. + for _ in range(2): + self.assertEqual(request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL).status, 200) + + time.sleep(max(ttl, 0) + 1.5) + r = request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, r.json) + self.assertTrue(r.headers.get("WWW-Authenticate", "").startswith("Bearer")) + r = request("DELETE", "/api/v1/auth/sessions/current", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, "logout with expired token must be 401") + + # Новый вход после истечения работает. + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..0242026 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,172 @@ +"""Idempotency-Key через балансировщик (lab3). Только стандартная библиотека. + + BASE_URL=http://127.0.0.1:8080 python -m unittest discover -s student-tests -p test_idempotency.py -v + +Главная проверка — одновременные повторы с одним ключом, которые LB +раскладывает по РАЗНЫМ репликам (это проверяется по X-Instance-Id), +создают ровно одно сообщение. Остальное — семантика из contracts/README.md. +""" +import json +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +from common import actor, create_chat, login, request + +PARALLEL = 24 + + +def history(token, chat_id): + items, cursor = [], None + for _ in range(100): + path = "/api/v1/chats/%s/messages?limit=100" % chat_id + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=token).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + raise AssertionError("pagination did not finish") + + +def post(token, chat_id, body, key, raw=None): + # None -> заголовок не отправляется (см. common.request). + headers = {"Idempotency-Key": key} + if raw is not None: + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, raw=raw, headers=headers) + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, headers=headers) + + +def parallel(n, fn): + barrier = threading.Barrier(n) + + def run(i): + barrier.wait() + return fn(i) + + with ThreadPoolExecutor(max_workers=n) as pool: + return list(pool.map(run, range(n))) + + +class IdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("ida") + cls.bob = actor("idb") + cls.eve = actor("ide") + cls.chat = create_chat(cls.alice, cls.bob) + cls.chat2 = create_chat(cls.alice, cls.bob) + + def test_concurrent_same_key_across_replicas_creates_one_message(self): + key = "par-" + uuid.uuid4().hex + text = "одновременно " + key + results = parallel(PARALLEL, lambda i: post(self.alice["_token"], self.chat["id"], {"text": text}, key)) + statuses = [r.status for r in results] + self.assertEqual(statuses, [201] * PARALLEL, statuses) + ids = {r.json["id"] for r in results} + self.assertEqual(len(ids), 1, "same key produced different messages") + bodies = {json.dumps(r.json, sort_keys=True) for r in results} + self.assertEqual(len(bodies), 1, "replays must return the identical Message") + instances = {r.headers.get("X-Instance-Id") for r in results} + self.assertGreaterEqual(len(instances), 2, "requests did not reach >= 2 replicas: %s" % instances) + stored = [m for m in history(self.bob["_token"], self.chat["id"]) if m["text"] == text] + self.assertEqual(len(stored), 1, "history must contain exactly one message") + replayed = sum(1 for r in results if r.headers.get("Idempotency-Replayed") == "true") + self.assertEqual(replayed, PARALLEL - 1) + print("\n %d parallel POSTs, one key -> 1 message; replicas: %s" % (PARALLEL, sorted(instances))) + + def test_concurrent_same_key_different_bodies_one_winner(self): + key = "race-" + uuid.uuid4().hex + results = parallel(12, lambda i: post(self.alice["_token"], self.chat["id"], {"text": "вариант %d" % i}, key)) + winners = [r for r in results if r.status == 201] + losers = [r for r in results if r.status == 409] + self.assertEqual(len(winners) + len(losers), 12, [r.status for r in results]) + self.assertGreaterEqual(len(winners), 1) + self.assertEqual(len({r.json["id"] for r in winners}), 1, "two bodies won with one key") + for r in losers: + self.assertEqual(r.json["error"]["code"], "conflict") + texts = [m["text"] for m in history(self.alice["_token"], self.chat["id"]) if m["text"].startswith("вариант ")] + self.assertEqual(len([t for t in texts if t == winners[0].json["text"]]), 1) + + def test_sequential_replay_and_conflict(self): + key = "seq-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual(first.status, 201) + again = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual((again.status, again.json), (201, first.json)) + other = post(self.alice["_token"], self.chat["id"], {"text": "два"}, key) + self.assertEqual(other.status, 409) + self.assertEqual(other.json["error"]["code"], "conflict") + + def test_json_formatting_and_key_order_do_not_matter(self): + key = "fmt-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{"text":"format"}') + spaced = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{ \n "text" : "format"\n}') + self.assertEqual(first.status, 201) + self.assertEqual((spaced.status, spaced.json), (201, first.json)) + + def test_missing_or_invalid_key_is_400(self): + token, chat_id = self.alice["_token"], self.chat["id"] + # Не-ASCII ключ передаём сырыми UTF-8 байтами (urllib пропускает только latin-1). + non_ascii = "ключ".encode("utf-8").decode("latin-1") + bad_keys = [None, "", " ", "a" * 129, non_ascii, "has space", "semi;colon", "slash/key"] + for key in bad_keys: + with self.subTest(key=key): + r = post(token, chat_id, {"text": "x"}, key) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(r.json["error"]["code"], "invalid_request") + ok = post(token, chat_id, {"text": "x"}, "A" * 128) + self.assertEqual(ok.status, 201) + ok2 = post(token, chat_id, {"text": "y"}, "a.b_c:d-1") + self.assertEqual(ok2.status, 201) + + def test_auth_checked_before_key(self): + r = post(None, self.chat["id"], {"text": "x"}, None) + self.assertEqual(r.status, 401) + + def test_scope_is_user_and_chat(self): + key = "scope-" + uuid.uuid4().hex + a1 = post(self.alice["_token"], self.chat["id"], {"text": "scope"}, key) + b1 = post(self.bob["_token"], self.chat["id"], {"text": "scope"}, key) + a2 = post(self.alice["_token"], self.chat2["id"], {"text": "scope"}, key) + self.assertEqual([a1.status, b1.status, a2.status], [201, 201, 201]) + self.assertEqual(len({a1.json["id"], b1.json["id"], a2.json["id"]}), 3) + # Другое тело другим пользователем с тем же ключом — не конфликт. + b2 = post(self.bob["_token"], self.chat2["id"], {"text": "другое"}, key) + self.assertEqual(b2.status, 201) + + def test_key_survives_new_session_of_same_user(self): + key = "sess-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "до перелогина"}, key) + token2 = login(self.alice) + replay = post(token2, self.chat["id"], {"text": "до перелогина"}, key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + + def test_failed_access_does_not_burn_key(self): + key = "burn-" + uuid.uuid4().hex + denied = post(self.eve["_token"], self.chat["id"], {"text": "чужое"}, key) + self.assertEqual(denied.status, 404) + # Тот же ключ у участника и даже у того же постороннего позже — не занят. + mine = post(self.alice["_token"], self.chat["id"], {"text": "своё"}, key) + self.assertEqual(mine.status, 201) + invalid = post(self.alice["_token"], self.chat2["id"], {"text": ""}, key) + self.assertEqual(invalid.status, 400) + valid = post(self.alice["_token"], self.chat2["id"], {"text": "после 400"}, key) + self.assertEqual(valid.status, 201) + + def test_replay_rechecks_session(self): + key = "revoked-" + uuid.uuid4().hex + token = login(self.bob) + first = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(first.status, 201) + self.assertEqual(request("DELETE", "/api/v1/auth/sessions/current", token=token).status, 204) + replay = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(replay.status, 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 0000000..fa57b8a --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,178 @@ +"""lab5: асинхронный pipeline вложений через балансировщик (HTTP, только stdlib). + + BASE_URL=http://127.0.0.1:8080 python -m unittest discover -s student-tests -p test_pipeline.py -v + +Проверяет контракт вариантов по ФАКТИЧЕСКИМ байтам (а не только по метаданным): + * 202 queued -> ready; варианты thumbnail (JPEG, вписан в 256), crop (JPEG 128x128), + optimized (WebP, вписан в 1280, без увеличения); size/sha256 совпадают с байтами; + * EXIF orientation применяется (400x100 с orientation=6 -> 64x256); + * маленький исходник не увеличивается в thumbnail/optimized, crop увеличивается; + * прозрачность компонуется на белый (JPEG thumbnail прозрачного PNG — белый); + * в вариантах нет EXIF (APP1) — метаданные удалены; + * original после обработки неизменен (SHA-256), variant=original совпадает; + * повторный GET стабилен (повтор не создаёт новых вариантов); + * сообщение может ссылаться на queued-вложение; посторонний не видит вариантов. +""" +import hashlib +import struct +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent / "tests")) +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 +from image_probe import jpeg_size # noqa: E402 +from test_attachments import raw_get, upload # noqa: E402 + +FIX = HERE / "fixtures" + + +def webp_size(data): + """Размеры WebP по заголовку RIFF (VP8 / VP8L / VP8X).""" + if data[:4] != b"RIFF" or data[8:12] != b"WEBP": + raise AssertionError("not a WebP file") + chunk = data[12:16] + if chunk == b"VP8 ": + w, h = struct.unpack("> 14) & 0x3FFF) + 1 + if chunk == b"VP8X": + w = int.from_bytes(data[24:27], "little") + 1 + h = int.from_bytes(data[27:30], "little") + 1 + return w, h + raise AssertionError("unknown WebP chunk %r" % chunk) + + +def jpeg_has_exif(data): + i = 2 + while i + 4 <= len(data) and data[i] == 0xFF: + marker = data[i + 1] + if marker == 0xDA: + return False + length = int.from_bytes(data[i + 2:i + 4], "big") + if marker == 0xE1 and data[i + 4:i + 10] == b"Exif\x00\x00": + return True + i += 2 + length + return False + + +def jpeg_center_is_white(data): + """Грубая проверка без декодера: JPEG целиком белого кадра очень мал и его + размеры совпадают; точную проверку цвета делает test_pipeline_faults (Pillow).""" + return len(data) < 2000 + + +@unittest.skipUnless(reachable(BASE_URL), "API is not reachable at %s" % BASE_URL) +class PipelineTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("pipe_a") + cls.bob = actor("pipe_b") + cls.eve = actor("pipe_e") + cls.chat = create_chat(cls.alice, cls.bob) + + def accept(self, data, ctype, name): + r = upload(self.alice["_token"], self.chat["id"], data, ctype, name) + self.assertEqual(r.status, 202, r.json) + self.assertEqual(r.json["status"], "queued") + self.assertEqual(r.json["variants"], []) + self.assertIsNone(r.json["error_code"]) + return r.json + + def variants_of(self, att): + return {v["name"]: v for v in att["variants"]} + + def fetch_variant(self, att, name): + status, headers, body = raw_get("/api/v1/attachments/%s/content?variant=%s" % (att["id"], name), + self.bob["_token"]) + self.assertEqual(status, 200, name) + v = self.variants_of(att)[name] + self.assertEqual(headers.get_content_type(), v["content_type"]) + self.assertEqual(len(body), v["size_bytes"]) + self.assertEqual(hashlib.sha256(body).hexdigest(), v["sha256"]) + return body + + def check_all(self, data, ctype, name, thumb, opt): + att = self.accept(data, ctype, name) + ready = wait_ready(self.bob["_token"], att["id"]) + self.assertIsNone(ready["error_code"]) + v = self.variants_of(ready) + self.assertEqual(sorted(v), ["crop", "optimized", "thumbnail"]) + self.assertEqual((v["thumbnail"]["content_type"], v["crop"]["content_type"], v["optimized"]["content_type"]), + ("image/jpeg", "image/jpeg", "image/webp")) + self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), thumb) + self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128)) + self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), opt) + t = self.fetch_variant(ready, "thumbnail") + c = self.fetch_variant(ready, "crop") + o = self.fetch_variant(ready, "optimized") + self.assertEqual(jpeg_size(t), thumb) + self.assertEqual(jpeg_size(c), (128, 128)) + self.assertEqual(webp_size(o), opt) + self.assertFalse(jpeg_has_exif(t) or jpeg_has_exif(c), "EXIF must be stripped from variants") + self.assertNotIn(b"Exif\x00\x00", o) + # Оригинал неизменен. + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual((status, body, headers.get_content_type()), (200, data, ctype)) + status, _, body2 = raw_get("/api/v1/attachments/%s/content?variant=original" % att["id"], self.bob["_token"]) + self.assertEqual((status, body2), (200, data)) + return ready, t + + def test_fixture_320x200(self): + data = (HERE.parent / "tests/fixtures/sample.png").read_bytes() + self.check_all(data, "image/png", "sample.png", (256, 160), (320, 200)) + + def test_exif_orientation_applied_and_stripped(self): + data = (FIX / "orient6.jpg").read_bytes() + self.assertIn(b"Exif", data) + self.check_all(data, "image/jpeg", "orient6.jpg", (64, 256), (100, 400)) + + def test_small_source_not_upscaled_except_crop(self): + data = (FIX / "small.png").read_bytes() + self.check_all(data, "image/png", "small.png", (40, 20), (40, 20)) + + def test_transparency_on_white(self): + data = (FIX / "transparent.png").read_bytes() + _, thumb = self.check_all(data, "image/png", "transparent.png", (256, 128), (600, 300)) + self.assertTrue(jpeg_center_is_white(thumb)) + + def test_progressive_jpeg(self): + data = (FIX / "sample_progressive.jpg").read_bytes() + self.check_all(data, "image/jpeg", "p.jpg", (256, 160), (320, 200)) + + def test_result_is_stable_and_private(self): + data = (FIX / "sample.jpg").read_bytes() + att = self.accept(data, "image/jpeg", "stable.jpg") + # queued/processing-вложение уже можно прикрепить к сообщению. + m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"], + payload={"text": "", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + self.assertEqual(m.status, 201, m.json) + ready = wait_ready(self.alice["_token"], att["id"]) + time.sleep(1.0) + again = request("GET", "/api/v1/attachments/" + att["id"], token=self.bob["_token"]).json + self.assertEqual(again, ready) # повторные чтения не меняют результат + for suffix in ("", "/content?variant=thumbnail", "/content?variant=crop"): + r = request("GET", "/api/v1/attachments/" + att["id"] + suffix, token=self.eve["_token"]) + self.assertEqual(r.status, 404) + + def test_unknown_variant_is_400(self): + data = (FIX / "small.png").read_bytes() + att = self.accept(data, "image/png", "v.png") + wait_ready(self.bob["_token"], att["id"]) + r = request("GET", "/api/v1/attachments/%s/content?variant=%s" % (att["id"], quote("big")), + token=self.bob["_token"]) + self.assertEqual(r.status, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..9004911 --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,310 @@ +"""lab5, уровень 4: внедрённые отказы pipeline. Запуск — внутри compose-сети: + + python scripts/ops.py pipeline-faults + +(пересоздаёт worker с compose.faults.yaml, поднимает api-fault-publish/api-fault-dup, +запускает этот файл в one-off контейнере образа API, затем возвращает обычный worker). + +Нужен прямой доступ к PostgreSQL (журнал job_events — доказательство того, ЧТО +произошло, а не только итогового статуса), к S3 и брокеру — берутся из тех же +secrets, что у API. Без окружения тесты пропускаются. + +Сценарии и ожидаемые инварианты: + A. crash между COMMIT и publish (api-fault-publish) -> задачу публикует sweeper + другого процесса, ready без ручного создания нового задания; + B. crash после XADD до подтверждения в outbox (api-fault-dup) -> дубль сообщения, + worker делает no-op, ровно один succeeded и один набор вариантов; + C. worker умер после записи объектов в S3 до COMMIT -> после рестарта тот же + consumer забирает свой PEL, перезаписывает ТЕ ЖЕ байты, ready; + D. worker умер после COMMIT до XACK -> повторная доставка -> no-op already_succeeded; + E. временная ошибка -> retry с backoff+jitter (интервалы в заданных границах), ready на 3-й попытке; + F. постоянная ошибка -> failed сразу (без повторов), DLQ, content 409, прикрепить нельзя (409); + G. попытки исчерпаны -> failed retry_exhausted -> явный replay -> ready; повторный + replay успешного задания -> no-op (не исполняется дубль); + H. зависший worker (обработка дольше аренды) -> lease_expired, новая попытка у + другого consumer, ready; проснувшийся старый результат НЕ публикует (fencing, lease_lost); + I. staging удалён после безопасной фиксации; + J. мусорное сообщение в брокере (не по схеме) -> ack без падения worker. +""" +import hashlib +import os +import re +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent)) +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 + +FAULT_PUBLISH_URL = os.environ.get("FAULT_PUBLISH_URL", "").rstrip("/") +FAULT_DUP_URL = os.environ.get("FAULT_DUP_URL", "").rstrip("/") +FIX = HERE / "fixtures" + + +def _db_available(): + try: + from src.config import load_settings + load_settings() + return True + except Exception: + return False + + +DB_OK = _db_available() + + +def upload(token, chat_id, data, filename, base_url=None, ctype="image/png"): + return request("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, quote(filename)), + token=token, raw=data, headers={"Content-Type": ctype}, base_url=base_url, timeout=60) + + +@unittest.skipUnless(reachable(BASE_URL) and DB_OK, "needs API at %s and DB/S3/broker env" % BASE_URL) +class PipelineFaultTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool + from src.jobs import JobStore + from src.objectstore import ObjectStore + cls.settings = load_settings() + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=15) + cls.jobs = JobStore(cls.pool, cls.settings) + cls.store = ObjectStore(cls.settings) + cls.alice = actor("flt_a") + cls.bob = actor("flt_b") + cls.chat = create_chat(cls.alice, cls.bob) + cls.png = (HERE.parent / "tests/fixtures/sample.png").read_bytes() + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + # ------------------------------------------------------------ helpers + + def accept(self, filename, base_url=None, data=None): + r = upload(self.alice["_token"], self.chat["id"], data or self.png, filename, base_url) + self.assertEqual(r.status, 202, (r.status, r.json)) + self.assertEqual(r.json["status"], "queued") + return r.json + + def job(self, att_id): + with self.pool.connection() as conn: + row = conn.execute("SELECT job_id, status, attempt, max_attempts, replay_count, last_error_code " + "FROM attachment_jobs WHERE attachment_id = %s", (att_id,)).fetchone() + return dict(zip(("job_id", "status", "attempt", "max_attempts", "replay_count", "last_error_code"), row)) + + def events(self, att_id): + with self.pool.connection() as conn: + return conn.execute( + "SELECT event, attempt, COALESCE(detail, ''), COALESCE(actor, ''), " + "EXTRACT(EPOCH FROM at)::float FROM job_events WHERE job_id IN " + "(SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", (att_id,)).fetchall() + + def names(self, att_id): + return [e[0] for e in self.events(att_id)] + + def wait_event(self, att_id, event, timeout=60.0, detail=None): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for e in self.events(att_id): + if e[0] == event and (detail is None or detail in e[2]): + return e + time.sleep(0.3) + self.fail("event %s (%s) not seen for %s; events: %s" % (event, detail, att_id, self.names(att_id))) + + def assert_single_result(self, att_id): + ready = request("GET", "/api/v1/attachments/" + att_id, token=self.bob["_token"]).json + self.assertEqual(ready["status"], "ready") + self.assertEqual(sorted(v["name"] for v in ready["variants"]), ["crop", "optimized", "thumbnail"]) + with self.pool.connection() as conn: + n = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id = %s", + (att_id,)).fetchone()[0] + self.assertEqual(n, 3, "exactly one set of variants") + self.assertEqual(self.names(att_id).count("succeeded"), 1, "exactly one published result") + return ready + + def reference_variants(self): + """Эталон: те же байты, загруженные без отказов. Детерминизм вывода.""" + if not hasattr(self.__class__, "_ref"): + att = self.accept("reference.png") + self.__class__._ref = {v["name"]: v["sha256"] + for v in wait_ready(self.bob["_token"], att["id"])["variants"]} + return self.__class__._ref + + # ------------------------------------------------------------ A, B: outbox + + @unittest.skipUnless(FAULT_PUBLISH_URL and reachable(FAULT_PUBLISH_URL), "api-fault-publish is not running") + def test_a_crash_between_commit_and_publish(self): + att = self.accept("a.png", base_url=FAULT_PUBLISH_URL) # 202 получен ДО падения + ready = wait_ready(self.bob["_token"], att["id"], timeout=60) + ev = self.events(att["id"]) + fault = [e for e in ev if e[0] == "fault_injected"] + self.assertTrue(fault, "fault instance did not crash before publish") + published = [e for e in ev if e[0] == "published"] + self.assertEqual(len(published), 1) + self.assertNotEqual(published[0][3], fault[0][3], "row must be published by ANOTHER process") + self.assertFalse(reachable(FAULT_PUBLISH_URL), "fault instance should be dead") + self.assertEqual(self.job(att["id"])["attempt"], 1, "same job, no new job created manually") + self.assert_single_result(att["id"]) + self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants()) + + @unittest.skipUnless(FAULT_DUP_URL and reachable(FAULT_DUP_URL), "api-fault-dup is not running") + def test_b_crash_after_publish_duplicate_is_noop(self): + att = self.accept("b.png", base_url=FAULT_DUP_URL) + wait_ready(self.bob["_token"], att["id"], timeout=60) + self.wait_event(att["id"], "redelivery_noop", timeout=30) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ C, D: worker crash + + def test_c_worker_crash_after_side_effects_before_commit(self): + att = self.accept("fault-crash-after-effects.png") + ready = wait_ready(self.bob["_token"], att["id"], timeout=120) + names = self.names(att["id"]) + self.assertIn("fault_injected", names) + self.assertIn("lease_takeover", names, "restarted consumer must take over its own stale lease") + self.assert_single_result(att["id"]) + self.assertEqual(self.job(att["id"])["attempt"], 1, "redelivery does not create a new attempt") + self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants(), + "repeated side effect writes identical bytes") + + def test_d_worker_crash_after_commit_before_ack(self): + att = self.accept("fault-crash-before-ack.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + self.wait_event(att["id"], "fault_injected", timeout=30) + self.wait_event(att["id"], "redelivery_noop", timeout=120, detail="already_succeeded") + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ E, F, G: retry / DLQ / replay + + def test_e_transient_errors_retry_with_backoff(self): + att = self.accept("fault-transient-until-3.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + ev = self.events(att["id"]) + retries = [e for e in ev if e[0] == "retry_scheduled"] + self.assertEqual([e[1] for e in retries], [1, 2]) + claims = [e for e in ev if e[0] == "claimed"] + self.assertEqual([e[1] for e in claims], [1, 2, 3]) + # Значения worker'а из compose.faults.yaml (у one-off контейнера свои, обычные). + base, cap = self.settings_value("FAULTS_RETRY_BASE_SECONDS", 1.0), self.settings_value("FAULTS_RETRY_MAX_SECONDS", 4.0) + for (r, nxt) in zip(retries, claims[1:]): + delay = float(re.search(r"in ([0-9.]+)s", r[2]).group(1)) + d = min(cap, base * 2 ** (r[1] - 1)) + self.assertGreaterEqual(delay, d / 2 - 1e-6) + self.assertLessEqual(delay, d + 1e-6) + self.assertGreaterEqual(nxt[4] - r[4], delay - 0.05, "next attempt must not start before backoff") + self.assertEqual(self.job(att["id"])["attempt"], 3) + self.assert_single_result(att["id"]) + + def settings_value(self, name, default): + return float(os.environ.get(name) or default) + + def test_f_permanent_error_fails_without_retry(self): + att = self.accept("fault-permanent.png") + done = wait_ready(self.bob["_token"], att["id"], timeout=60, allow_failed=True) + self.assertEqual((done["status"], done["error_code"], done["variants"]), ("failed", "invalid_image", [])) + self.assertNotIn("retry_scheduled", self.names(att["id"])) + self.assertEqual(self.job(att["id"])["attempt"], 1) + with self.pool.connection() as conn: + dl = conn.execute("SELECT error_code FROM dead_letters WHERE attachment_id = %s", (att["id"],)).fetchone() + self.assertEqual(dl[0], "invalid_image") + r = request("GET", "/api/v1/attachments/%s/content" % att["id"], token=self.bob["_token"]) + self.assertEqual((r.status, r.json["error"]["code"]), (409, "conflict")) + m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"], + payload={"text": "x", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + self.assertEqual(m.status, 409) + + def test_g_retry_exhausted_dlq_then_explicit_replay(self): + att = self.accept("fault-transient-until-replay.png") + done = wait_ready(self.bob["_token"], att["id"], timeout=90, allow_failed=True) + self.assertEqual((done["status"], done["error_code"]), ("failed", "retry_exhausted")) + job = self.job(att["id"]) + self.assertEqual((job["status"], job["attempt"]), ("failed", job["max_attempts"])) + self.assertEqual(self.names(att["id"]).count("retry_scheduled"), job["max_attempts"] - 1) + outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay") + self.assertEqual(outcome, "replayed") + wait_ready(self.bob["_token"], att["id"], timeout=60) + job2 = self.job(att["id"]) + self.assertEqual((job2["status"], job2["attempt"], job2["replay_count"]), + ("succeeded", job["attempt"] + 1, 1)) + # Повторный replay успешного задания — no-op: дубль не исполняется. + outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay") + self.assertEqual(outcome, "noop") + self.assertEqual(self.job(att["id"])["attempt"], job2["attempt"]) + with self.pool.connection() as conn: + dl = conn.execute("SELECT replayed_at IS NOT NULL, replay_count FROM dead_letters WHERE attachment_id = %s", + (att["id"],)).fetchone() + self.assertEqual(dl, (True, 1)) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ H: hung worker + + def test_h_hung_worker_lease_expires_and_is_fenced(self): + att = self.accept("fault-slow.png") + wait_ready(self.bob["_token"], att["id"], timeout=90) + self.wait_event(att["id"], "lease_expired", timeout=5) + self.wait_event(att["id"], "lease_lost", timeout=80) + job = self.job(att["id"]) + self.assertEqual(job["attempt"], 2) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ I, J + + def test_i_staging_removed_after_safe_commit(self): + att = self.accept("plain.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + with self.pool.connection() as conn: + key = conn.execute("SELECT staging_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0] + if key is None: + break + time.sleep(0.3) + self.assertIsNone(key) + with self.pool.connection() as conn: + final = conn.execute("SELECT object_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0] + self.assertTrue(final.startswith("final/")) + data = self.store.read_all(final, 20 * 1024 * 1024) + self.assertEqual(hashlib.sha256(data).hexdigest(), att["sha256"]) + with self.pool.connection() as conn: + leftovers = conn.execute( + "SELECT count(*) FROM job_events WHERE event = 'accepted' AND attachment_id = %s", (att["id"],) + ).fetchone()[0] + self.assertEqual(leftovers, 1) + + def test_j_garbage_broker_message_is_acked(self): + from src.broker import Broker + b = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group) + mid = b.client.xadd(self.settings.broker_stream, {"job": '{"not":"a job"}'}).decode() + mid2 = b.publish({"schema_version": 1, "job_id": str(uuid.uuid4()), "type": "attachment.process.v1", + "attachment_id": str(uuid.uuid4()), "pipeline_version": 1, "attempt": 1, + "created_at": "2026-01-01T00:00:00Z"}) + deadline = time.monotonic() + 20 + pending = None + while time.monotonic() < deadline: + pending = [p for p in b.client.xpending_range(self.settings.broker_stream, self.settings.broker_group, + "-", "+", 100) + if p["message_id"].decode() in (mid, mid2)] + info = b.client.xinfo_groups(self.settings.broker_stream)[0] + delivered = info["last-delivered-id"].decode() + if not pending and _id(delivered) >= _id(mid2): + break + time.sleep(0.3) + self.assertEqual(pending, [], "poison/unknown messages must be acknowledged") + b.close() + + +def _id(x): + a, _, b = x.partition("-") + return int(a), int(b or 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_session_cache.py b/student-tests/test_session_cache.py new file mode 100644 index 0000000..e097270 --- /dev/null +++ b/student-tests/test_session_cache.py @@ -0,0 +1,149 @@ +"""Отзыв сессии при сбое инвалидации кэша (регрессия замечания ревью №1). + +Сценарий ревью: прогретый кэш -> связь API с Redis разорвана -> logout (204) +-> связь восстановлена с ТЕМ ЖЕ Redis, без его рестарта, положительная запись +отозванного токена ещё жива. Ожидание: токен отклоняется сразу, а не через +SESSION_CACHE_TTL_SECONDS. + +Redis здесь заменён управляемой заглушкой с тем же интерфейсом, что и +SessionCache (get/put/revoke): «разрыв» = операции падают, но данные заглушки +сохраняются — ровно как у живого Redis за сетевым разрывом. PostgreSQL — +настоящая (нужен DATABASE_URL/DATABASE_URL_FILE, иначе тест пропускается): + make test-storage +""" +import os +import sys +import time +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class PartitionableCache: + """Кэш, который «помнит» данные во время разрыва, но недоступен для API.""" + + def __init__(self): + self.data = {} + self.partitioned = False + + def get(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return None + value = self.data.get(digest) + if value is None: + return None + if value == REVOKED: + return REVOKED + user, expires_at = value + return (user, expires_at) if expires_at > datetime.now(timezone.utc) else None + + def put(self, digest, user, expires_at): + if not self.partitioned: + self.data.setdefault(digest, (user, expires_at)) # SET NX + + def revoke(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return False + self.data[digest] = REVOKED + return True + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class SessionRevocationCacheTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + cls.settings = load_settings() + wait_and_migrate(cls.settings) + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.user = cls.storage.create_user("rvk_" + uuid.uuid4().hex[:12], "Revoke", + hash_password("Revoke-test-password")) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def service(self, cache): + from src.sessions import SessionService + return SessionService(self.storage, cache, self.settings) + + def pending(self, digest): + with self.pool.connection() as conn: + return conn.execute("SELECT count(*) FROM session_cache_invalidations WHERE token_hash = %s", + (digest,)).fetchone()[0] + + def warm(self, svc, cache, token): + from src.security import token_digest + self.assertEqual(svc.resolve(token).id, self.user.id) + self.assertIsInstance(cache.data.get(token_digest(token)), tuple, "cache was not warmed") + + def test_revoked_token_rejected_after_partition_heals(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + digest = token_digest(token) + self.warm(svc, cache, token) + + cache.partitioned = True + self.assertTrue(svc.revoke(token), "logout must succeed (PostgreSQL is the source of truth)") + self.assertIsInstance(cache.data[digest], tuple, "precondition: stale positive entry survived") + self.assertEqual(self.pending(digest), 1, "invalidation debt must be committed with revoked_at") + self.assertIsNone(svc.resolve(token), "during partition") + + cache.partitioned = False # тот же «Redis», данные не потеряны + self.assertIsNone(svc.resolve(token), "immediately after partition heals") + time.sleep(svc.RECHECK_SECONDS + 0.1) + self.assertIsNone(svc.resolve(token), "after reconciliation") + self.assertEqual(cache.data[digest], "revoked", "reconciliation must write the tombstone") + self.assertEqual(self.pending(digest), 0, "debt must be cleared after the tombstone") + + # После погашения долга кэшу снова доверяют: другой токен обслуживается из кэша. + other, _ = svc.create(self.user) + self.warm(svc, cache, other) + + def test_debt_survives_api_restart(self): + """Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу.""" + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + cache.partitioned = True + self.assertTrue(svc.revoke(token)) + del svc # «рестарт API»: состояние процесса потеряно + + cache.partitioned = False + restarted = self.service(cache) + self.assertIsNone(restarted.resolve(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + + def test_normal_logout_leaves_no_debt(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + self.assertTrue(svc.revoke(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + self.assertEqual(self.pending(token_digest(token)), 0) + self.assertIsNone(svc.resolve(token)) + self.assertFalse(svc.revoke(token), "second logout must fail") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_storage_idempotency.py b/student-tests/test_storage_idempotency.py new file mode 100644 index 0000000..b433420 --- /dev/null +++ b/student-tests/test_storage_idempotency.py @@ -0,0 +1,144 @@ +"""Идемпотентность на уровне хранилища (нужен прямой доступ к PostgreSQL). + +* сбой ПОСЛЕ вставки ключа и сообщения внутри транзакции -> не остаётся ни ключа, + ни сообщения, и повтор с тем же ключом создаёт сообщение (5xx не фиксируется); +* 16 потоков с одним ключом -> ровно одна вставка, остальные — повтор; +* истёкший ключ можно занять заново; очистка удаляет только истёкшие строки и + выполняется одной репликой за раз (advisory lock). + +Запуск внутри compose-сети (БД наружу не публикуется): + python scripts/ops.py in-network storage +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import hashlib +import os +import sys +import threading +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +DAY = 86400 + + +class InjectedFailure(RuntimeError): + pass + + +def fp(text): + return hashlib.sha256(text.encode()).digest() + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageIdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Idempotency-test-password") + cls.user = cls.storage.create_user("idm_" + uuid.uuid4().hex[:12], "Idem", h) + cls.chat = cls.storage.create_chat("idem-" + uuid.uuid4().hex[:8], [cls.user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def keys(self, key): + return self.count("SELECT count(*) FROM idempotency_keys WHERE user_id = %s AND idem_key = %s", + (self.user.id, key)) + + def test_failure_inside_transaction_stores_nothing(self): + key = "fail-" + uuid.uuid4().hex + text = "сбой " + key + original = self.storage._insert_message + seen = {} + + def insert_then_fail(conn, message_id, chat_id, sender_id, body): + original(conn, message_id, chat_id, sender_id, body) + seen["keys"] = conn.execute("SELECT count(*) FROM idempotency_keys WHERE idem_key = %s", (key,)).fetchone()[0] + seen["messages"] = conn.execute("SELECT count(*) FROM messages WHERE id = %s", (message_id,)).fetchone()[0] + raise InjectedFailure("crash after both inserts, before COMMIT") + + self.storage._insert_message = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + finally: + self.storage._insert_message = original + self.assertEqual((seen["keys"], seen["messages"]), (1, 1), "inserts were visible inside the transaction") + self.assertEqual(self.keys(key), 0) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 0) + # Клиент повторяет неподтверждённый запрос с тем же ключом — теперь успешно. + msg, replayed = self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + self.assertFalse(replayed) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_threads_same_key_single_insert(self): + key = "thr-" + uuid.uuid4().hex + text = "потоки " + key + barrier = threading.Barrier(16) + + def call(_): + barrier.wait() + return self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + + with ThreadPoolExecutor(max_workers=16) as pool: + results = list(pool.map(call, range(16))) + self.assertEqual(sum(1 for _, replayed in results if not replayed), 1) + self.assertEqual(len({m.id for m, _ in results}), 1) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_conflict_on_different_body(self): + from src.storage import IdempotencyConflict + key = "cnf-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "a", key, fp("a"), DAY) + with self.assertRaises(IdempotencyConflict): + self.storage.create_message(self.chat.id, self.user.id, "b", key, fp("b"), DAY) + + def test_expired_key_can_be_reused_and_is_cleaned(self): + key = "exp-" + uuid.uuid4().hex + first, _ = self.storage.create_message(self.chat.id, self.user.id, "старый", key, fp("старый"), 1) + time.sleep(1.2) + # Истёк: другое тело уже не конфликт, создаётся новое сообщение. + second, replayed = self.storage.create_message(self.chat.id, self.user.id, "новый", key, fp("новый"), DAY) + self.assertFalse(replayed) + self.assertNotEqual(first.id, second.id) + self.assertEqual(self.keys(key), 1) + # Очистка удаляет только истёкшие ключи. + k2 = "exp2-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "x", k2, fp("x"), 1) + time.sleep(1.2) + while self.storage.delete_expired_idempotency_keys(): + pass + self.assertEqual(self.keys(k2), 0) + self.assertEqual(self.keys(key), 1) + + def test_cleanup_runs_on_one_replica_at_a_time(self): + from src.storage import IDEMPOTENCY_CLEANUP_LOCK_ID + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,)) + self.assertIsNone(self.storage.delete_expired_idempotency_keys()) + self.assertIsNotNone(self.storage.delete_expired_idempotency_keys()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_storage_rollback.py b/student-tests/test_storage_rollback.py new file mode 100644 index 0000000..305f0ec --- /dev/null +++ b/student-tests/test_storage_rollback.py @@ -0,0 +1,101 @@ +"""Rollback на уровне хранилища с внедрённым сбоем (нужен прямой доступ к БД). + +Сбой вызывается ПОСЛЕ того, как строки chats и chat_members уже вставлены +внутри транзакции. Проверяем, что в БД не осталось ни чата, ни участников. + +Запуск внутри compose-сети (БД наружу не публикуется): + docker compose run --rm --no-deps -v "$PWD/student-tests:/app/student-tests:ro" \ + api python -m unittest discover -s student-tests -p test_storage_rollback.py -v +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import os +import sys +import unittest +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class InjectedFailure(RuntimeError): + pass + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageRollbackTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Rollback-test-password") + cls.users = [ + cls.storage.create_user("rbk_" + uuid.uuid4().hex[:12], "Rollback", h) + for _ in range(3) + ] + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def test_failure_after_inserts_leaves_nothing(self): + title = "rollback-" + uuid.uuid4().hex + original = self.storage._insert_members + seen = {} + + def insert_then_fail(conn, chat_id, member_ids): + original(conn, chat_id, member_ids) + # Внутри транзакции строки видны — значит вставка действительно была. + seen["chats"] = conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone()[0] + seen["members"] = conn.execute("SELECT count(*) FROM chat_members WHERE chat_id = %s", (chat_id,)).fetchone()[0] + seen["chat_id"] = chat_id + raise InjectedFailure("simulated crash between statements") + + self.storage._insert_members = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_chat(title, [u.id for u in self.users]) + finally: + self.storage._insert_members = original + + self.assertEqual(seen["chats"], 1) + self.assertEqual(seen["members"], 3) + self.assertEqual(self.count("SELECT count(*) FROM chats WHERE title = %s", (title,)), 0) + self.assertEqual(self.count("SELECT count(*) FROM chat_members WHERE chat_id = %s", (seen["chat_id"],)), 0) + + def test_no_orphan_chats_in_database(self): + orphans = self.count( + "SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)", ()) + self.assertEqual(orphans, 0) + + def test_migrations_are_idempotent(self): + from src.config import load_settings + from src.db import apply_migrations + self.assertEqual(apply_migrations(load_settings()), []) + self.assertEqual(self.count("SELECT count(*) FROM schema_migrations WHERE version = %s", ("001_init",)), 1) + + def test_passwords_stored_as_argon2id(self): + with self.pool.connection() as conn: + stored = conn.execute("SELECT password_hash FROM users WHERE id = %s", + (self.users[0].id,)).fetchone()[0] + self.assertTrue(stored.startswith("$argon2id$"), stored[:10]) + self.assertNotIn("Rollback-test-password", stored) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..092ea4a --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Эксперимент: N одновременных загрузок ~9.5 MiB и пик памяти API-реплик. + +Гипотеза: память реплики ограничена сверху + baseline + MAX_CONCURRENT_UPLOADS x (~2 x размер файла) + разбор изображения, +и не растёт с числом одновременных клиентов: лишние ждут слот +(UPLOAD_QUEUE_TIMEOUT_SECONDS), а не читают тело в память. + +Источники памяти: + всегда VmHWM процесса API (пик RSS, считает ядро), обнулённый перед + экспериментом через /proc/PID/clear_refs — главный показатель; + --docker + docker stats (опрос ~1 с) по контейнерам сервиса api + + cgroup memory.peak (ядро >= 5.19) до и после; + --pids 123,456 + /proc//status: VmRSS каждые 20 мс (запуск без Docker). + + python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 \ + --concurrency 10 --docker --output evidence/upload_memory.json + +Изображения синтетические (шум, PNG без сжатия). Токены не сохраняются. +""" +import argparse +import hashlib +import json +import os +import platform +import re +import statistics +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 + +MiB = 1024 * 1024 +UNITS = {"b": 1, "kib": 1024, "mib": MiB, "gib": 1024 * MiB, "kb": 1000, "mb": 1000 ** 2, "gb": 1000 ** 3} + + +def parse_mem(text): + m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", text) + return float(m.group(1)) * UNITS.get(m.group(2).lower(), 1) if m else 0.0 + + +class Sampler(threading.Thread): + def __init__(self, args): + super().__init__(daemon=True) + self.args = args + self.stop = threading.Event() + self.samples = {} # name -> [bytes] + + def add(self, name, value): + self.samples.setdefault(name, []).append(value) + + def run(self): + while not self.stop.is_set(): + if self.args.pids: + for pid in self.args.pids.split(","): + try: + status = Path("/proc/%s/status" % pid).read_text() + rss = int(re.search(r"VmRSS:\s+(\d+) kB", status).group(1)) * 1024 + self.add("pid-" + pid, rss) + except (OSError, AttributeError): + pass + time.sleep(0.02) + else: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"] + docker_api_ids(self.args), + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, usage = line.split("\t", 1) + self.add(name, parse_mem(usage.split("/")[0])) + + +def docker_api_ids(args): + out = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + args.project, + "--filter", "label=com.docker.compose.service=api"], capture_output=True, text=True).stdout + return out.split() + + +def proc_hwm(args, reset=False): + """Пик RSS процесса (VmHWM, считает ядро — без пропусков между опросами). + + --docker: PID 1 контейнера api (python, exec-форма CMD); --pids: заданные PID. + reset=True обнуляет пик (echo 5 > /proc/PID/clear_refs) перед экспериментом. + """ + result = {} + if args.docker: + for cid in docker_api_ids(args): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + if reset: + subprocess.run(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"], + capture_output=True) + out = subprocess.run(["docker", "exec", cid, "cat", "/proc/1/status"], capture_output=True, + text=True).stdout + m = re.search(r"VmHWM:\s+(\d+) kB", out) + if m: + result[name] = int(m.group(1)) * 1024 + else: + for pid in args.pids.split(","): + try: + if reset: + Path("/proc/%s/clear_refs" % pid).write_text("5") + m = re.search(r"VmHWM:\s+(\d+) kB", Path("/proc/%s/status" % pid).read_text()) + result["pid-" + pid] = int(m.group(1)) * 1024 + except (OSError, AttributeError): + pass + return result + + +def cgroup_peaks(args): + """cgroup v2 memory.peak (или v1 max_usage_in_bytes) каждого контейнера api.""" + peaks = {} + for cid in docker_api_ids(args): + for path in ("/sys/fs/cgroup/memory.peak", "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"): + r = subprocess.run(["docker", "exec", cid, "cat", path], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip().isdigit(): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + peaks[name] = int(r.stdout.strip()) + break + return peaks + + +def one_upload(base, token, chat_id, data): + req = Request(base + "/api/v1/chats/%s/attachments?filename=mem.png" % chat_id, data=data, method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) + t0 = time.monotonic() + try: + with urlopen(req, timeout=120) as r: + body = json.loads(r.read()) + ok = body["sha256"] == hashlib.sha256(data).hexdigest() + return r.status, time.monotonic() - t0, r.headers.get("X-Instance-Id"), ok + except HTTPError as e: + return e.code, time.monotonic() - t0, e.headers.get("X-Instance-Id"), False + except (URLError, OSError): + return 0, time.monotonic() - t0, None, False + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + p.add_argument("--concurrency", type=int, default=10) + p.add_argument("--rounds", type=int, default=2, help="повторы волны одновременных загрузок") + p.add_argument("--size-mib", type=float, default=9.5) + p.add_argument("--docker", action="store_true") + p.add_argument("--project", default=os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab4")) + p.add_argument("--pids", default="") + p.add_argument("--label", default="") + p.add_argument("--output", default="") + args = p.parse_args() + if not args.docker and not args.pids: + p.error("choose --docker or --pids") + base = args.base_url.rstrip("/") + common.BASE_URL = base + + owner = common.actor("mem") + chat = common.create_chat(owner) + files = [imagegen.noise_png(int(args.size_mib * MiB)) for _ in range(args.concurrency)] + assert all(len(f) <= 10 * MiB for f in files) + + before = cgroup_peaks(args) if args.docker else {} + proc_hwm(args, reset=True) + sampler = Sampler(args) + sampler.start() + time.sleep(1.5) # baseline + baseline = {k: max(v) for k, v in sampler.samples.items()} + results = [] + started = time.monotonic() + for _ in range(args.rounds): + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results += list(pool.map(lambda d: one_upload(base, owner["_token"], chat["id"], d), files)) + duration = time.monotonic() - started + time.sleep(1.0) + sampler.stop.set() + sampler.join(timeout=5) + after = cgroup_peaks(args) if args.docker else {} + hwm = proc_hwm(args) + + statuses = {} + for status, _, _, _ in results: + statuses[str(status)] = statuses.get(str(status), 0) + 1 + lat = sorted(r[1] for r in results if r[0] in (201, 202)) + + def pct(q): + return round(lat[min(len(lat) - 1, int(q * len(lat)))], 3) if lat else None + + memory = {} + for name, values in sampler.samples.items(): + memory[name] = { + "baseline_mib": round(baseline.get(name, values[0]) / MiB, 1), + "peak_mib": round(max(values) / MiB, 1), + "growth_mib": round((max(values) - baseline.get(name, values[0])) / MiB, 1), + "samples": len(values), + } + report = { + "label": args.label, + "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "host": "%s %s, %d CPU" % (platform.system(), platform.machine(), os.cpu_count() or 0), + "source": "docker stats + cgroup peak" if args.docker else "/proc VmRSS (20 ms)", + "concurrency": args.concurrency, + "rounds": args.rounds, + "file_mib": round(len(files[0]) / MiB, 2), + "uploads": len(results), + "statuses": statuses, + "sha256_ok": sum(1 for r in results if r[3]), + "instances": sorted({r[2] for r in results if r[2]}), + "duration_s": round(duration, 2), + "latency_201_s": {"p50": pct(0.5), "p95": pct(0.95), "max": lat[-1] if lat else None, + "mean": round(statistics.mean(lat), 3) if lat else None}, + "memory": memory, + } + report["process_peak_rss_mib"] = {k: round(v / MiB, 1) for k, v in hwm.items()} + if args.docker: + report["cgroup_peak_mib_before"] = {k: round(v / MiB, 1) for k, v in before.items()} + report["cgroup_peak_mib_after"] = {k: round(v / MiB, 1) for k, v in after.items()} + text = json.dumps(report, ensure_ascii=False, indent=2) + print(text) + if args.output: + Path(args.output).write_text(text + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/wait_url.py b/student-tests/wait_url.py new file mode 100644 index 0000000..ddf59a4 --- /dev/null +++ b/student-tests/wait_url.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Ждать HTTP 200 по URL (до 90 с). Используется внутри compose-сети: ops.py in-network.""" +import sys +import time +from urllib.request import urlopen + +url = sys.argv[1] +deadline = time.monotonic() + float(sys.argv[2] if len(sys.argv) > 2 else 90) +while time.monotonic() < deadline: + try: + if urlopen(url, timeout=2).status == 200: + print("ready: " + url) + sys.exit(0) + except OSError: + pass + time.sleep(0.5) +print("not ready: " + url) +sys.exit(1) diff --git a/student-tests/worker_outage.py b/student-tests/worker_outage.py new file mode 100644 index 0000000..95db816 --- /dev/null +++ b/student-tests/worker_outage.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""lab5, уровень 3: сценарий защиты «worker остановлен -> upload 202 -> рестарт +API и broker -> worker запущен -> ready». Запускается с хоста (управляет Docker +через scripts/ops.py), печатает только ID и статусы — без токенов. + + python student-tests/worker_outage.py --output evidence/worker_outage.json + +Шаги: + 1. остановить worker (docker stop); + 2. загрузить файл -> 202 queued; GET -> queued; content -> 409; сообщение с этим + вложением и обычное сообщение -> 201 (остальной API работает); + 3. брокер: сообщение лежит в потоке (lag >= 1) — `ops.py jobs stats`; + 4. перезапустить реплики API, затем broker (volumes сохраняются); + 5. GET -> всё ещё queued (никто синхронно не обработал), сообщение в брокере + пережило рестарт (lag >= 1); + 6. запустить worker -> ready, thumbnail 256x160 — настоящий JPEG, SHA-256 совпадает. + +Команды управления переопределяются (--stop-worker и т.д.) для другого стенда. +""" +import argparse +import hashlib +import json +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(ROOT / "tests")) +import common # noqa: E402 +from image_probe import jpeg_size # noqa: E402 +from test_attachments import raw_get, upload # noqa: E402 + +PY = sys.executable + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="http://127.0.0.1:8080") + p.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY)) + p.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY)) + p.add_argument("--restart-api", default="%s scripts/ops.py api-restart" % shlex.quote(PY)) + p.add_argument("--restart-broker", default="%s scripts/ops.py broker-restart" % shlex.quote(PY)) + p.add_argument("--broker-stats", default="%s scripts/ops.py jobs stats" % shlex.quote(PY)) + p.add_argument("--output", default="") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + timeline = [] + t0 = time.monotonic() + + def note(step, **kw): + kw.update(step=step, t=round(time.monotonic() - t0, 2)) + timeline.append(kw) + print(json.dumps(kw, ensure_ascii=False), flush=True) + + def sh(cmd): + started = time.monotonic() + out = subprocess.run(shlex.split(cmd), cwd=ROOT, capture_output=True, text=True, timeout=600) + if out.returncode != 0: + raise SystemExit("command failed: %s\n%s" % (cmd, out.stderr[-800:])) + return out.stdout, round(time.monotonic() - started, 2) + + def broker_lag(): + out, _ = sh(args.broker_stats) + try: + data = json.loads(out[out.index("{"):]) + g = data["broker"]["groups"][0] + return {"length": data["broker"]["length"], "lag": g.get("lag"), "pending": g["pending"], + "jobs": data["jobs"], "outbox_rows": data["outbox"]["rows"]} + except (ValueError, KeyError, IndexError, TypeError): + return {"raw": out[-300:]} + + alice = common.actor("out_a") + bob = common.actor("out_b") + chat = common.create_chat(alice, bob) + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + + _, dt = sh(args.stop_worker) + note("worker stopped", seconds=dt) + + r = upload(alice["_token"], chat["id"], data, "image/png", "outage.png") + assert r.status == 202 and r.json["status"] == "queued", (r.status, r.json) + att = r.json + note("upload accepted", status=r.status, attachment_id=att["id"], attachment_status=att["status"]) + + time.sleep(3) # быстрый путь relay + sweeper успевают опубликовать + meta = common.request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]).json + content = common.request("GET", "/api/v1/attachments/%s/content" % att["id"], token=bob["_token"]) + assert meta["status"] == "queued", meta + assert content.status == 409, content.status + m1 = common.request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "файл ещё в очереди", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + m2 = common.request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=bob["_token"], + payload={"text": "обычное сообщение"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + assert m1.status == 201 and m2.status == 201, (m1.status, m2.status) + note("while worker is down", metadata_status=meta["status"], content_status=content.status, + message_with_attachment=m1.status, plain_message=m2.status, message_id=m1.json["id"]) + note("broker before restarts", **broker_lag()) + + _, dt = sh(args.restart_api) + note("api replicas restarted", seconds=dt) + _, dt = sh(args.restart_broker) + note("broker restarted (volume kept)", seconds=dt) + time.sleep(2) + meta = common.request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]).json + assert meta["status"] == "queued", meta + stats = broker_lag() + note("after restarts", metadata_status=meta["status"], **stats) + + _, dt = sh(args.start_worker) + note("worker started", seconds=dt) + ready = common.wait_ready(bob["_token"], att["id"], timeout=90) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + status, headers, body = raw_get("/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], bob["_token"]) + assert status == 200 and headers.get_content_type() == "image/jpeg" + assert hashlib.sha256(body).hexdigest() == thumb["sha256"] and jpeg_size(body) == (256, 160) + status, _, original = raw_get("/api/v1/attachments/%s/content" % att["id"], bob["_token"]) + assert status == 200 and original == data + note("ready", attachment_id=att["id"], variants=[v["name"] for v in ready["variants"]], + thumbnail_jpeg_size=list(jpeg_size(body)), original_sha256_ok=True) + print("PASS: accepted job survived worker stop + API restart + broker restart and became ready") + if args.output: + Path(args.output).write_text(json.dumps(timeline, ensure_ascii=False, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main()