diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..570e3c9e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN groupadd --system app && \ + useradd --system --gid app --create-home app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --timeout 60 -r requirements.txt + +COPY --chown=app:app src/ /app/src/ +COPY --chown=app:app migrations/ /app/migrations/ + +ARG APP_VERSION=v1 +ENV APP_VERSION=${APP_VERSION} + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2)" || exit 1 + +CMD ["sh", "-c", "exec uvicorn src.main:app --host \"${APP_HOST}\" --port \"${APP_PORT}\" --timeout-graceful-shutdown 8"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..26dbd040 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,133 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO - +- Автор, группа: Саргсян Сона +- Целевая оценка; выбранный трек: 5, трек 2 «Согласованность двух хранилищ» +- Commit/PR: ⟨ссылка на PR или хеш коммита⟩ +- Источники, библиотеки и помощь ИИ: документация FastAPI, asyncpg, boto3, Pillow, SeaweedFS, AWS S3 (список объектов, коды ошибок). Помощь ИИ (Claude, Anthropic): ревью кода, исправление потоковой проверки PNG и таймаутов S3, reconciler, тесты и сценарии экспериментов. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Desktop с Compose v2, Python 3.9+ на хосте (только для тестов), `pip3 install pillow` (JPEG-тесты). + +1. `cp .env.example .env` (значения по умолчанию подходят для локального стенда). +2. `./scripts/gen-secrets.sh` создаёт `secrets/*` (пароль PostgreSQL, ключи S3 admin и api, конфиг identities SeaweedFS). Секреты не попадают в git и в образ (`.gitignore`, `.dockerignore`). +3. `docker compose --env-file .env build migrate`, затем `docker compose --env-file .env up -d`. Порядок задан `depends_on`: postgres → migrate → (s3 → s3-init) → api ×2 → lb. +4. Дождаться `healthy`: `docker compose --env-file .env ps`; затем `make test BASE_URL=http://localhost:8080`. +5. Остановка с сохранением данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volumes pgdata и s3data). + +Для Swarm не используется: стенд запускается через Compose. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy (lb, единственный опубликованный порт 127.0.0.1:8080) → api ×2 → PostgreSQL (метаданные, сессии, ключи идемпотентности) + Valkey (кэш сессий) + SeaweedFS S3 (байты). + +| Роль | Сервис | Образ | Health | Volume | Сеть/порт | Credentials | +|---|---|---|---|---|---|---| +| Балансировщик | lb | haproxy:3.0.7-alpine | httpchk /health/ready у backend | — | default; 127.0.0.1:8080 | — | +| API ×2 | api | собран из исходников, python:3.12.7-slim@sha256 | /health/ready (PostgreSQL) | — | default + storage, expose 8080 | Docker secrets | +| БД | postgres | postgres:16.6-alpine | pg_isready | pgdata | default | secret postgres_password | +| Кэш | valkey | valkey/valkey:8.0.1-alpine | valkey-cli ping | нет (кэш) | default | — | +| S3 | s3 | chrislusf/seaweedfs:4.53 | wget к master и S3 | s3data | **только storage (internal)**, порты не публикуются | secret s3_config | +| Bootstrap bucket | s3-init | amazon/aws-cli:2.32.29 | одноразовый | — | storage | secrets s3_admin_* | +| Миграции | migrate | образ приложения | одноразовый | — | default | secret database_url | +| Reconciler | reconciler | образ приложения | одноразовый job (profile `tools`) | — | default + storage | secrets s3_api_* | + +**Источник истины.** PostgreSQL: пользователи, чаты, сообщения, метаданные вложений. S3 хранит только байты. Локальный файл оригинала на API не требуется: тело потоково пишется во временный `SpooledTemporaryFile` (до 1 МиБ в памяти, дальше диск контейнера), после ответа удаляется. + +**Модель Attachment.** Таблица `attachments` (миграция 003) и связь `message_attachments` с порядком (`position`). Ключ S3 `attachments/` назначает сервер; `filename` только отображаемое имя. Миграция 004 добавляет `storage_state` (`ok`/`missing`) и `storage_checked_at`: служебное состояние для reconciler; в API-ответах не отображается, т.к. контракт lab4 допускает только `status=ready`. + +**Порядок записи и когда честно возвращать 201.** Проверки (доступ к чату, MIME по содержимому, лимиты) → S3 PUT → INSERT в PostgreSQL → 201. Если INSERT не удался или клиент оборвал соединение, объект удаляется компенсацией (`asyncio.shield`). Если S3 PUT прошёл, а COMMIT нет, и компенсация тоже не удалась (S3 недоступен, процесс убит), в S3 остаётся неучтённый объект: его удаляет reconciler после grace period. Сообщения ссылаются только на вложения, у которых есть строка в БД, поэтому такой объект пользователям не виден. + +**Лимиты и потоковая обработка.** ≤10 MiB: проверка Content-Length до чтения тела и счётчик байтов на лету (chunked). ≤40 млн пикселей: размеры берутся из IHDR (PNG) или заголовка (JPEG) до чтения остального и до любого декодирования. Пиксели API не декодирует. Структура PNG проверяется потоково (CRC каждого чанка блоками по 64 КиБ): в первой версии `Image.verify()` читал чанк целиком, и PNG с одним IDAT на 10 МиБ давал +140 МиБ RAM на 10–20 загрузок (замер до исправления, локальный стенд); после замены на потоковую проверку рост не зависит от размера чанка. Таймауты S3: connect 2 с, read 5 с, всего 2 попытки. Лимит контейнера API 256 МиБ. + +**Безопасность хранилища.** Bucket приватный (анонимный доступ → 403). API использует отдельные ключи `Read/Write/List` только на свой bucket, админские ключи есть только у одноразового `s3-init`. SeaweedFS находится в сети `storage` с `internal: true`, к ней подключены только s3, s3-init, api, reconciler. Download идёт через API с проверкой членства в чате, без redirect. Случайный ключ и приватный bucket не заменяют ACL приложения: любой, кто узнает ключ, всё равно не сможет получить объект без проверки членства в API. + +**Reconciler (трек 2)** `src/reconciler.py`, запуск `docker compose --env-file .env run --rm --no-deps reconciler python -m src.reconciler [--apply] [--grace N] [--max-actions N] [--interval N]`. +- По умолчанию **dry-run**; изменения только с `--apply`. Итог выводится JSON-строкой (stdout), логи в stderr. +- orphan (объект есть, записи нет, объект старше grace): перед удалением повторная проверка записи в БД (закрывает гонку с только что завершённой загрузкой). +- missing (запись `ok` старше grace, объекта нет): подтверждается HEAD, затем запись помечается `missing`. Запись не удаляется, чтобы не терять ссылки из сообщений. Скачивание такого вложения даёт 409, прикрепить к новому сообщению нельзя (409). +- restore: запись `missing`, объект снова есть, размер и SHA-256 совпали → `ok`. +- Предохранители: grace `RECONCILE_GRACE_SECONDS` (600 с по умолчанию); `--max-actions` (1000): при большем числе действий apply отменяется (код 3); `pg_try_advisory_lock` не даёт запускать две копии. +- Несовпадение размера записи и объекта только фиксируется в отчёте (`size_mismatch`), не исправляется. + +**Соответствие .env.example реальной конфигурации.** `S3_ACCESS_KEY_ID_FILE`/`S3_SECRET_ACCESS_KEY_FILE` → `/run/secrets/s3_api_access_key_id`, `/run/secrets/s3_api_secret_access_key`; `DATABASE_URL_FILE` → `/run/secrets/database_url`; `CACHE_URL_FILE` заменён на `CACHE_URL=redis://valkey:6379/0`; остальные (`S3_ENDPOINT`, `S3_BUCKET`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS`, `API_REPLICAS`) совпадают. Добавлены `RECONCILE_GRACE_SECONDS`, `RECONCILE_MAX_ACTIONS`, `S3_CONNECT_TIMEOUT`, `S3_READ_TIMEOUT`, `DRAIN_SECONDS`. Критичная для /health/ready зависимость только PostgreSQL; недоступность S3 даёт 503 на операциях с файлами, но не выводит реплику из балансировки. + +Из lab3 перенесено без изменений: сессии (PostgreSQL + кэш Valkey), идемпотентность сообщений, keyset-пагинация, graceful shutdown. Изменено в lab4: добавлены S3, вложения, `attachment_ids` в сообщениях, сеть `storage`, reconciler. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Окружение: ⟨Docker Desktop версия⟩, ⟨macOS, CPU/RAM⟩; лимиты контейнеров: api 0.5 CPU / 256 МиБ, postgres 1 CPU / 512 МиБ, s3 1 CPU / 512 МиБ. Дата прогонов ⟨дата⟩, commit ⟨хеш⟩. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Уровень 3: публичный smoke (в т.ч. SHA-256 оригинала, третий пользователь → 404) | `make test BASE_URL=http://localhost:8080` | PASS | `evidence/smoke.txt` | +| Уровень 3: S3 в общем стенде, ≥2 API, один LB, метаданные в PG, байты в приватном bucket | `docker compose ps` | api-1, api-2, lb, postgres, valkey, s3; опубликован только 127.0.0.1:8080 | `evidence/ps-before.txt` | +| Уровень 3: лимиты, MIME по содержимому, опасные filename | `python3 -m unittest student_tests.test_attachments_level4 -v` | 14 тестов OK | `evidence/attachments-level4.txt` | +| Уровень 3: upload → остановка обслужившей реплики → download через другую | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` (дважды, для каждой реплики) | ⟨PASS, время восстановления⟩ | `evidence/failover-1.txt`, `failover-1-ps.txt`, `failover-2.txt`, `failover-2-ps.txt` | +| Уровень 3: restart стенда с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` (перезапускает api, postgres, valkey, s3) | ⟨PASS⟩ | `evidence/persistence.txt`, `evidence/ps-after-restart.txt` | +| Уровень 3: возможности lab3 сохранены | идемпотентность 30 параллельных, отказ Valkey/PostgreSQL, истечение сессии | ⟨PASS⟩ | `evidence/idempotency-parallel.txt`, `evidence/lab3-regression.txt`, `evidence/load-*.json` | +| Уровень 4: тесты большой файл (Content-Length и chunked), повреждённый PNG/JPEG, подмена MIME, чужое attachment_id, вложение другого чата; ошибки пользователя не 5xx | тот же набор тестов | OK, ожидаемые 413/415/400/404 | `evidence/attachments-level4.txt` | +| Уровень 4: прерывание загрузки не оставляет метаданных | `test_aborted_upload_leaves_no_metadata` | число записей не изменилось | `evidence/attachments-level4.txt` | +| Уровень 4: отказ между записью S3 и БД: компенсация | `test_db_insert_failure_after_s3_put_is_compensated` (CHECK (false) ломает INSERT, число объектов в S3 до/после равно) | OK | `evidence/attachments-level4.txt` | +| Уровень 4: сверка/очистка с grace period | `track2_experiment.py` | см. трек 2 | `evidence/track2-experiment.txt` | +| Уровень 4: ≥10 одновременных загрузок, пик памяти | `upload_memory.py --concurrency 10` (+ single-idat, + 20) | ⟨см. таблицу экспериментов⟩ | `evidence/upload-memory-*.json` | +| Уровень 4: идемпотентный bootstrap bucket | `docker compose run --rm --no-deps s3-init` дважды | оба раза код 0, «already exists» | `evidence/s3-init-rerun.txt` | +| Уровень 4: отдельные ограниченные S3-credentials; консоль не в общей сети; private bucket вне API | `python3 student_tests/check_s3_private.py` | анонимный GET/list → 403; API-ключ не создаёт bucket и не пишет в чужой; нет admin-секретов в api; порты S3 не опубликованы; lb/postgres/valkey не достигают s3 | `evidence/s3-private-check.txt` | +| Уровень 4: S3 недоступен → ограниченный ответ и восстановление | `test_s3_down_gives_503_and_recovers` | 503 за <20 с, метаданные 200, после старта S3 тот же файл | `evidence/attachments-level4.txt` | +| Трек 2: обнаружение, dry-run, исправление, повторный запуск, grace, восстановление, предохранитель, стоимость | `python3 student_tests/track2_experiment.py --seed 1000 5000` | все проверки PASS | `evidence/track2-experiment.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Эксперимент 1. Память при параллельных загрузках.** Гипотеза: благодаря потоковой обработке рост памяти API не пропорционален суммарному размеру загрузок и не приближается к лимиту 256 МиБ. Методика: `student_tests/upload_memory.py`: N одновременных медленных загрузок PNG ≈10 МиБ (64 КиБ на отправку, пауза 20 мс) через LB; память реплик читается из cgroup (`anon`: реальный RAM, `file` отдельно) каждые ~0,2 с; режимы: обычный PNG (IDAT по 32 КиБ) и худший случай (весь PNG одним IDAT). -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| N | Режим | Статусы | Суммарно отправлено | Рост anon на реплику (макс.) | Пик anon | Лимит | +| --- | --- | --- | --- | --- | --- | --- | +| 10 | обычный | 201×10 | 98,1 МиБ | 18,6 МиБ | 80,1 МиБ | 256 МиБ | +| 10 | один IDAT | 201×10 | 98,1 МиБ | 5,9 МиБ | 83,1 МиБ | 256 МиБ | +| 20 | обычный, 1-й прогон | 201×1, 503×19 | 196,2 МиБ | 32,1 МиБ | 108,6 МиБ | 256 МиБ | +| 20 | обычный, повтор | 201×20 | 196,2 МиБ | 49,5 МиБ | 100,8 МиБ | 256 МиБ | + +Для N=10 все загрузки завершились 201, рост реального RAM (anon) на реплику не превысил 19 МиБ при отправленных 98 МиБ и не зависит от структуры PNG: даже худший случай (один IDAT на 10 МиБ) даёт +6 МиБ, потому что структура PNG проверяется блоками по 64 КиБ. До этой правки (`Image.verify()` читал чанк целиком) на моём тестовом стенде при том же сценарии рост достигал ≈140 МиБ. + +При N=20 первый прогон дал 201×1 и 503×19, при повторном прогоне все 20 загрузок завершились 201 за 8,9 с. Память в обоих случаях в норме: рост anon ≤ 50 МиБ при отправленных 196 МиБ (пик 109 МиБ при лимите 256 МиБ), поэтому причина первого сбоя не в памяти API. Логи API показывают, что после получения тела обработка одной загрузки (проверка + PUT в S3) занимала около 5,5 с под нагрузкой, что близко к таймауту чтения S3 (5 с); вероятная причина 503 — превышение этого таймаута из-за кратковременной перегрузки S3 (1 CPU, volume 64 МБ), но прямого подтверждения нет (логи первого прогона не сохранены). Ошибка при этом ограниченная и корректная (503, не OOM и не падение). Требование «не менее 10 одновременных загрузок» выполнено стабильно, 20 — нестабильно, нужен запас по таймауту S3 или ограничение числа одновременных записей. + +Примечания: `peak_current_mib_incl_page_cache` (137–204 МиБ) включает page cache от временных файлов загрузки, ядро освобождает его при нехватке памяти, в «anon» оно не входит. `baseline_anon` в разных запусках отличается (61–78 МиБ) из-за аллокатора Python, поэтому сравнивается рост. Нагрузка с одного клиента, 2 реплики, один запуск каждого режима, память измеряется выборкой каждые ~0,2 с (пик может быть чуть выше). + +**Эксперимент 2. Согласованность БД и S3 (трек 2).** Гипотеза: reconciler обнаруживает расхождения с обеих сторон, исправляет только безопасное, не трогает свежие и используемые вложения, повторный запуск ничего не меняет. Методика: `track2_experiment.py`, grace 5 с: внедряются 3 сироты в S3 (старые), 1 свежая сирота, у 2 записей удаляется объект (одна из них прикреплена к сообщению), 1 свежая запись теряет объект; проверяются dry-run, apply, повторный apply, восстановление объекта, переход через grace, предохранитель. + + + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| 3 старые сироты в S3 (объект есть, записи нет, старше grace 30 с) | dry-run находит и ничего не меняет; apply удаляет | dry-run: `orphans_found=3`, `orphans_deleted=0`, объекты на месте; apply: `orphans_deleted=3`, объектов в S3 не осталось | мусор удалён, пользовательские данные не затронуты | +| Свежая сирота (младше grace) | не удаляется до окончания grace | при apply и повторном apply `orphans_fresh_skipped=1`, объект на месте; после выдержки grace следующий apply удалил её (`orphans_deleted=1`) | — | +| Потеряны объекты у 2 записей (одна прикреплена к сообщению) | запись остаётся, помечается `missing`; скачивание 409, прикрепить нельзя | dry-run: `missing_found=2`, `missing_marked=0`; apply: `missing_marked=2`; скачивание 409 (не 500); метаданные 200; прикрепление к новому сообщению 409 | байты потеряны (так и должно быть: объекта нет), записи и ссылки из сообщений сохранены | +| Свежая запись без объекта (моложе grace) | не помечается до окончания grace | `missing_fresh_skipped=1`, состояние `ok`; после grace следующий apply пометил её (`missing_marked=1`) | — | +| Объект возвращён на то же место | запись снова `ok` после проверки размера и SHA-256 | `restore_found=1`, `restored=1`, файл скачивается, SHA-256 совпадает | полное восстановление | +| Используемые и неиспользуемые вложения старше grace | не удаляются | оба вложения целы после apply, скачиваются, SHA-256 совпадает | потерь нет | +| Повторный запуск apply | ничего не находит и не меняет | `orphans_found=0`, `missing_found=0`, `orphans_deleted=0`, `missing_marked=0` | идемпотентно | +| Массовое удаление: 5000 сирот при лимите `--max-actions 1000` | предохранитель отклоняет apply | код выхода 3, `error=too_many_actions`, `orphans_deleted=0`; с `--max-actions 5100` удалено 5000 за 5,8 с | данные не удалены до явного разрешения | +| Сбой INSERT в БД после успешного PUT в S3 (CHECK (false) на таблице) | ответ 5xx, объект убирается компенсацией | тест `test_db_insert_failure_after_s3_put_is_compensated` OK: число объектов в S3 до и после одинаково | сироты не остаётся | +| Обрыв загрузки на середине тела | метаданных нет | тест `test_aborted_upload_leaves_no_metadata` OK: число записей в чате не изменилось | сироты не остаётся | +| S3 недоступен (`docker compose stop s3`) | 503 за ограниченное время, метаданные из БД, после старта S3 прежний файл | тест `test_s3_down_gives_503_and_recovers` OK: upload 503 за <20 с, скачивание 503, метаданные 200, после старта S3 скачивание 200 и SHA-256 совпал | без потерь | + +Стоимость обхода (dry-run, `s3_list_s`, `elapsed_s`, `peak_rss_mib` из JSON reconciler; один запуск, SeaweedFS на том же хосте, поэтому значения оценочные): + +| Объектов в S3 | Страниц листинга | s3_list_s | elapsed_s | peak RSS, МиБ | +| --- | --- | --- | --- | --- | +| ⟨61⟩ | ⟨1⟩ | ⟨0.062⟩ | ⟨0.063⟩ | ⟨52.6⟩ | +| ⟨1058⟩ | ⟨2⟩ | ⟨0.099⟩ | ⟨0.113⟩ | ⟨54.0⟩ | +| ⟨5058⟩ | ⟨6⟩ | ⟨0.259⟩ | ⟨0.319⟩ | ⟨56.9⟩ | + +Вывод по замеру: ⟨время растёт примерно линейно с числом объектов: 1 страница листинга (1000 ключей) на запрос⟩ (заполнить по своим числам). + +**Нагрузка и failover (lab3).** Профиль и результаты в `evidence/load-baseline.json`, `load-stop-0.json`, `load-stop-1.json` (closed-loop, 8 потоков, 30–40 с; ограничения closed-loop и coordinated omission, p99 только успешных запросов). Эти прогоны сделаны до изменений lab4-кода; функциональные проверки отказов реплик повторены `make test-failover`. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Reconciler однопроцессный и грузит в память все ключи S3 и все строки `attachments`: при десятках миллионов объектов нужен постраничный/инкрементальный обход (по диапазонам ключей/дате). Стоимость растёт линейно (см. замер). +- Grace period сравнивает `LastModified` из S3 с временем PostgreSQL; предполагаются согласованные часы на одном хосте. При разнесённых хостах нужен запас по перекосу часов. +- Проверка PNG: сигнатура, IHDR, CRC всех чанков, наличие IDAT/IEND; содержимое zlib-потока не распаковывается (иначе риск CPU-DoS), поэтому валидный по CRC, но с испорченным потоком PNG будет принят. JPEG: заголовок (Pillow) и маркер EOI в последних 4 КиБ. +- Миссинг-запись не удаляется автоматически: решение (удалить сообщение, восстановить из бэкапа) принимает оператор. Статус `missing` не виден в JSON API из-за контракта lab4 (`status=ready`). +- Из контейнера api внутри сети `storage` доступны служебные порты SeaweedFS (filer 8888, master 9333): их нельзя выключить в `weed server` без смены конфигурации; изоляция обеспечена сетью (недоступны для lb, postgres, valkey и с хоста), не отключением портов. Сам api компрометировать = получить ключ API, поэтому риск не выше. +- Valkey без persistence (только кэш); S3 и PostgreSQL по одному экземпляру: отказ их контейнеров = недоступность соответствующих операций (тесты показывают 503 и восстановление), но не HA. +- Presigned upload и direct download (трек 1) не реализованы. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..061ef5bd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,213 @@ +name: mtusi-lab4 + +services: + postgres: + image: postgres:16.6-alpine + restart: unless-stopped + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEM:-512M} + + valkey: + image: valkey/valkey:8.0.1-alpine + restart: unless-stopped + command: ["valkey-server", "--save", "", "--appendonly", "no", "--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + # Приватное S3-хранилище. Порты наружу не публикуются, консоль не открыта. + s3: + image: chrislusf/seaweedfs:4.53 + restart: unless-stopped + command: ["server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=64"] + secrets: + - s3_config + volumes: + - s3data:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && wget -S -q -O /dev/null http://127.0.0.1:9000/ 2>&1 | grep -q 'HTTP/1.1'"] + interval: 5s + timeout: 3s + retries: 12 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: [storage] + + # Одноразовый идемпотентный bootstrap: создаёт приватный bucket с админскими credentials. + s3-init: + image: amazon/aws-cli:2.32.29 + restart: "no" + entrypoint: ["/bin/sh", "/init/s3-init.sh"] + depends_on: + s3: + condition: service_healthy + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + volumes: + - ./infra/s3/s3-init.sh:/init/s3-init.sh:ro + networks: [storage] + + # Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают + migrate: + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-v1} + image: mtusi-messenger:lab4 + restart: "no" + command: ["python", "-m", "src.migrate"] + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + # Реплики: число API_REPLICAS, ресурсы каждой API_CPUS / API_MEM. Без container_name и published-портов + api: + image: mtusi-messenger:lab4 + pull_policy: never + restart: unless-stopped + stop_grace_period: 20s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPUS:-0.5}" + memory: ${API_MEM:-256M} + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_started + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + environment: + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-5} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://valkey:6379/0 + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + secrets: + - database_url + - s3_api_access_key_id + - s3_api_secret_access_key + expose: + - "8080" + networks: [default, storage] + + # Трек 2: Фоновый процесс очистки сирот в S3 + # Запускается по расписанию (например, раз в час) или вручную + reconciler: + image: mtusi-messenger:lab4 + pull_policy: never + restart: "no" # Это не сервис, а джоб + command: ["python", "-m", "src.reconciler"] + depends_on: + postgres: + condition: service_healthy + s3-init: + condition: service_completed_successfully + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-600} + secrets: + - database_url + - s3_api_access_key_id + - s3_api_secret_access_key + profiles: ["tools"] + networks: [default, storage] + + # Единственная публичная точка входа + lb: + image: haproxy:3.0.7-alpine + restart: unless-stopped + depends_on: + - api + volumes: + - ./infra/haproxy:/usr/local/etc/haproxy:ro + ports: + - "127.0.0.1:8080:8080" + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_config: + file: ./secrets/s3_config + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_api_access_key_id: + file: ./secrets/s3_api_access_key_id + s3_api_secret_access_key: + file: ./secrets/s3_api_secret_access_key + +volumes: + pgdata: + s3data: + +networks: + default: + storage: + internal: true # только s3, s3-init, api, reconciler; lb/postgres/valkey до S3 не дотянутся \ No newline at end of file diff --git a/evidence/attachments-level4.txt b/evidence/attachments-level4.txt new file mode 100644 index 00000000..ef0ee84d --- /dev/null +++ b/evidence/attachments-level4.txt @@ -0,0 +1,19 @@ +test_aborted_upload_leaves_no_metadata (student_tests.test_attachments_level4.AttachmentsLevel4.test_aborted_upload_leaves_no_metadata) ... ok +test_attachment_from_another_chat_of_same_user (student_tests.test_attachments_level4.AttachmentsLevel4.test_attachment_from_another_chat_of_same_user) ... ok +test_corrupted_jpeg_and_valid_jpeg (student_tests.test_attachments_level4.AttachmentsLevel4.test_corrupted_jpeg_and_valid_jpeg) ... ok +test_corrupted_png (student_tests.test_attachments_level4.AttachmentsLevel4.test_corrupted_png) ... ok +test_dangerous_filenames (student_tests.test_attachments_level4.AttachmentsLevel4.test_dangerous_filenames) ... ok +test_db_insert_failure_after_s3_put_is_compensated (student_tests.test_attachments_level4.AttachmentsLevel4.test_db_insert_failure_after_s3_put_is_compensated) ... ok +test_foreign_and_unknown_attachment_ids (student_tests.test_attachments_level4.AttachmentsLevel4.test_foreign_and_unknown_attachment_ids) ... ok +test_mime_spoofing (student_tests.test_attachments_level4.AttachmentsLevel4.test_mime_spoofing) ... ok +test_outsider_has_no_access (student_tests.test_attachments_level4.AttachmentsLevel4.test_outsider_has_no_access) ... ok +test_pixel_limit (student_tests.test_attachments_level4.AttachmentsLevel4.test_pixel_limit) ... ok +test_roundtrip_sha256_and_second_member (student_tests.test_attachments_level4.AttachmentsLevel4.test_roundtrip_sha256_and_second_member) ... ok +test_s3_down_gives_503_and_recovers (student_tests.test_attachments_level4.AttachmentsLevel4.test_s3_down_gives_503_and_recovers) ... ok +test_too_large_chunked_without_content_length (student_tests.test_attachments_level4.AttachmentsLevel4.test_too_large_chunked_without_content_length) ... ok +test_too_large_content_length (student_tests.test_attachments_level4.AttachmentsLevel4.test_too_large_content_length) ... ok + +---------------------------------------------------------------------- +Ran 14 tests in 29.266s + +OK diff --git a/evidence/failover-1-ps.txt b/evidence/failover-1-ps.txt new file mode 100644 index 00000000..722fa4a5 --- /dev/null +++ b/evidence/failover-1-ps.txt @@ -0,0 +1,10 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 19 minutes ago Up 15 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 19 minutes ago Exited (137) 34 seconds ago +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 19 hours ago Up 19 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-messenger:lab4 "python -m src.migra…" migrate 19 minutes ago Exited (0) 15 minutes ago +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 19 hours ago Up 15 minutes (healthy) 5432/tcp +mtusi-lab4-reconciler-1 sha256:0e315aeef5f75920354c8c142ee631dfacaf5623bcad6dadb1f5cd1875f9f727 "python -m src.recon…" reconciler 8 hours ago Exited (0) 8 hours ago +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 8 hours ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-s3-init-1 amazon/aws-cli:2.32.29 "/bin/sh /init/s3-in…" s3-init 8 hours ago Exited (0) 15 minutes ago +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 32 hours ago Up 15 minutes (healthy) 6379/tcp diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..45a2a7f6 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.316} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2-ps.txt b/evidence/failover-2-ps.txt new file mode 100644 index 00000000..c71163eb --- /dev/null +++ b/evidence/failover-2-ps.txt @@ -0,0 +1,10 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 23 minutes ago Up 18 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 23 minutes ago Exited (137) About a minute ago +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 19 hours ago Up 19 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-messenger:lab4 "python -m src.migra…" migrate 23 minutes ago Exited (0) 2 minutes ago +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 19 hours ago Up 19 minutes (healthy) 5432/tcp +mtusi-lab4-reconciler-1 sha256:0e315aeef5f75920354c8c142ee631dfacaf5623bcad6dadb1f5cd1875f9f727 "python -m src.recon…" reconciler 8 hours ago Exited (0) 8 hours ago +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 8 hours ago Up 5 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-s3-init-1 amazon/aws-cli:2.32.29 "/bin/sh /init/s3-in…" s3-init 8 hours ago Exited (0) 2 minutes ago +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 32 hours ago Up 19 minutes (healthy) 6379/tcp diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..982e14ac --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.314} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency-parallel.txt b/evidence/idempotency-parallel.txt new file mode 100644 index 00000000..c989d3b8 --- /dev/null +++ b/evidence/idempotency-parallel.txt @@ -0,0 +1,2 @@ +{"rounds": 5, "parallel_requests_per_round": 30, "messages_in_history": 5, "replicas_that_answered": ["00be497bb3e0", "1a491e91605b"]} +PASS: same key in parallel -> exactly one message; different body -> 409 diff --git a/evidence/lab3-regression.txt b/evidence/lab3-regression.txt new file mode 100644 index 00000000..8853d459 --- /dev/null +++ b/evidence/lab3-regression.txt @@ -0,0 +1,36 @@ +test_chat_creation_rolls_back (student_tests.test_level4.Level4.test_chat_creation_rolls_back) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_logout_revokes_session_and_cache (student_tests.test_level4.Level4.test_logout_revokes_session_and_cache) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_postgres_down_returns_bounded_error (student_tests.test_level4.Level4.test_postgres_down_returns_bounded_error) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +/opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +/opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_register_race_same_username (student_tests.test_level4.Level4.test_register_race_same_username) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_valkey_down_falls_back_to_postgres (student_tests.test_level4.Level4.test_valkey_down_falls_back_to_postgres) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_valkey_flush_keeps_sessions (student_tests.test_level4.Level4.test_valkey_flush_keeps_sessions) ... ok +test_wrong_password (student_tests.test_level4.Level4.test_wrong_password) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok +test_x_user_id_gives_no_access (student_tests.test_level4.Level4.test_x_user_id_gives_no_access) ... ok +test_zz_session_expiry (student_tests.test_level4.Level4.test_zz_session_expiry) ... /opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +/opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +/opt/homebrew/Cellar/python@3.14/3.14.0_1/Frameworks/Python.framework/Versions/3.14/lib/python3.14/tempfile.py:484: ResourceWarning: Implicitly cleaning up + _warnings.warn(self.warn_message, ResourceWarning) +ok + +---------------------------------------------------------------------- +Ran 9 tests in 38.172s + +OK diff --git a/evidence/load-baseline.json b/evidence/load-baseline.json new file mode 100644 index 00000000..40213596 --- /dev/null +++ b/evidence/load-baseline.json @@ -0,0 +1,150 @@ +{ + "summary": { + "duration_s": 30.0, + "concurrency": 8, + "replicas_seen": [ + "6da22c3d39a9", + "96b6387add4f" + ], + "requests": 25788, + "ok": 25788, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 859.6, + "latency_ms": { + "p50": 6.5, + "p95": 29.4, + "p99": 51.6 + }, + "error_statuses": {}, + "action": {}, + "versions_first_last_seen_s": { + "v1": [ + 0.0, + 30.0 + ] + } + }, + "per_second": { + "0": { + "ok": 623, + "err": 0 + }, + "1": { + "ok": 903, + "err": 0 + }, + "2": { + "ok": 988, + "err": 0 + }, + "3": { + "ok": 765, + "err": 0 + }, + "4": { + "ok": 865, + "err": 0 + }, + "5": { + "ok": 697, + "err": 0 + }, + "6": { + "ok": 811, + "err": 0 + }, + "7": { + "ok": 861, + "err": 0 + }, + "8": { + "ok": 941, + "err": 0 + }, + "9": { + "ok": 953, + "err": 0 + }, + "10": { + "ok": 884, + "err": 0 + }, + "11": { + "ok": 880, + "err": 0 + }, + "12": { + "ok": 1026, + "err": 0 + }, + "13": { + "ok": 918, + "err": 0 + }, + "14": { + "ok": 919, + "err": 0 + }, + "15": { + "ok": 729, + "err": 0 + }, + "16": { + "ok": 911, + "err": 0 + }, + "17": { + "ok": 957, + "err": 0 + }, + "18": { + "ok": 920, + "err": 0 + }, + "19": { + "ok": 905, + "err": 0 + }, + "20": { + "ok": 897, + "err": 0 + }, + "21": { + "ok": 666, + "err": 0 + }, + "22": { + "ok": 736, + "err": 0 + }, + "23": { + "ok": 906, + "err": 0 + }, + "24": { + "ok": 823, + "err": 0 + }, + "25": { + "ok": 859, + "err": 0 + }, + "26": { + "ok": 691, + "err": 0 + }, + "27": { + "ok": 951, + "err": 0 + }, + "28": { + "ok": 861, + "err": 0 + }, + "29": { + "ok": 942, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-0-ps.txt b/evidence/load-stop-0-ps.txt new file mode 100644 index 00000000..c0d18f50 --- /dev/null +++ b/evidence/load-stop-0-ps.txt @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 27 seconds ago +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 2 hours ago Up 2 hours (healthy) 8080/tcp +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-messenger:lab4 "python -m src.migra…" migrate 2 hours ago Exited (0) 2 hours ago +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 14 hours ago Up 14 hours (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-s3-init-1 amazon/aws-cli:2.32.29 "/bin/sh /init/s3-in…" s3-init 14 hours ago Exited (0) 2 hours ago +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 14 hours ago Up 14 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-0.json b/evidence/load-stop-0.json new file mode 100644 index 00000000..335edd90 --- /dev/null +++ b/evidence/load-stop-0.json @@ -0,0 +1,201 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "6da22c3d39a9", + "96b6387add4f" + ], + "requests": 23887, + "ok": 23887, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 597.2, + "latency_ms": { + "p50": 7.3, + "p95": 55.0, + "p99": 85.4 + }, + "error_statuses": {}, + "action": { + "target": "6da22c3d39a9", + "started_at": 10.0, + "finished_at": 12.54, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.15, + "instances_in_last_5s": [ + "96b6387add4f" + ], + "versions_first_last_seen_s": { + "v1": [ + 0.0, + 40.0 + ] + } + }, + "per_second": { + "0": { + "ok": 781, + "err": 0 + }, + "1": { + "ok": 816, + "err": 0 + }, + "2": { + "ok": 916, + "err": 0 + }, + "3": { + "ok": 966, + "err": 0 + }, + "4": { + "ok": 592, + "err": 0 + }, + "5": { + "ok": 656, + "err": 0 + }, + "6": { + "ok": 696, + "err": 0 + }, + "7": { + "ok": 982, + "err": 0 + }, + "8": { + "ok": 851, + "err": 0 + }, + "9": { + "ok": 927, + "err": 0 + }, + "10": { + "ok": 958, + "err": 0 + }, + "11": { + "ok": 667, + "err": 0 + }, + "12": { + "ok": 510, + "err": 0 + }, + "13": { + "ok": 555, + "err": 0 + }, + "14": { + "ok": 530, + "err": 0 + }, + "15": { + "ok": 518, + "err": 0 + }, + "16": { + "ok": 330, + "err": 0 + }, + "17": { + "ok": 523, + "err": 0 + }, + "18": { + "ok": 475, + "err": 0 + }, + "19": { + "ok": 476, + "err": 0 + }, + "20": { + "ok": 544, + "err": 0 + }, + "21": { + "ok": 550, + "err": 0 + }, + "22": { + "ok": 490, + "err": 0 + }, + "23": { + "ok": 499, + "err": 0 + }, + "24": { + "ok": 521, + "err": 0 + }, + "25": { + "ok": 524, + "err": 0 + }, + "26": { + "ok": 515, + "err": 0 + }, + "27": { + "ok": 245, + "err": 0 + }, + "28": { + "ok": 579, + "err": 0 + }, + "29": { + "ok": 491, + "err": 0 + }, + "30": { + "ok": 536, + "err": 0 + }, + "31": { + "ok": 454, + "err": 0 + }, + "32": { + "ok": 587, + "err": 0 + }, + "33": { + "ok": 549, + "err": 0 + }, + "34": { + "ok": 564, + "err": 0 + }, + "35": { + "ok": 562, + "err": 0 + }, + "36": { + "ok": 565, + "err": 0 + }, + "37": { + "ok": 467, + "err": 0 + }, + "38": { + "ok": 337, + "err": 0 + }, + "39": { + "ok": 583, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-1-ps.txt b/evidence/load-stop-1-ps.txt new file mode 100644 index 00000000..327e61ed --- /dev/null +++ b/evidence/load-stop-1-ps.txt @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 2 hours ago Up 56 seconds (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-messenger:lab4 "python -m src.migra…" migrate 2 hours ago Exited (0) 56 seconds ago +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 14 hours ago Up 14 hours (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-s3-init-1 amazon/aws-cli:2.32.29 "/bin/sh /init/s3-in…" s3-init 14 hours ago Exited (0) 57 seconds ago +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 14 hours ago Up 14 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-1.json b/evidence/load-stop-1.json new file mode 100644 index 00000000..915c61d1 --- /dev/null +++ b/evidence/load-stop-1.json @@ -0,0 +1,201 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "6da22c3d39a9", + "96b6387add4f" + ], + "requests": 22558, + "ok": 22558, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 564.0, + "latency_ms": { + "p50": 7.0, + "p95": 56.3, + "p99": 87.0 + }, + "error_statuses": {}, + "action": { + "target": "96b6387add4f", + "started_at": 10.01, + "finished_at": 12.32, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.29, + "instances_in_last_5s": [ + "6da22c3d39a9" + ], + "versions_first_last_seen_s": { + "v1": [ + 0.0, + 40.0 + ] + } + }, + "per_second": { + "0": { + "ok": 545, + "err": 0 + }, + "1": { + "ok": 611, + "err": 0 + }, + "2": { + "ok": 888, + "err": 0 + }, + "3": { + "ok": 829, + "err": 0 + }, + "4": { + "ok": 831, + "err": 0 + }, + "5": { + "ok": 848, + "err": 0 + }, + "6": { + "ok": 830, + "err": 0 + }, + "7": { + "ok": 796, + "err": 0 + }, + "8": { + "ok": 808, + "err": 0 + }, + "9": { + "ok": 481, + "err": 0 + }, + "10": { + "ok": 667, + "err": 0 + }, + "11": { + "ok": 412, + "err": 0 + }, + "12": { + "ok": 452, + "err": 0 + }, + "13": { + "ok": 385, + "err": 0 + }, + "14": { + "ok": 440, + "err": 0 + }, + "15": { + "ok": 511, + "err": 0 + }, + "16": { + "ok": 541, + "err": 0 + }, + "17": { + "ok": 539, + "err": 0 + }, + "18": { + "ok": 486, + "err": 0 + }, + "19": { + "ok": 480, + "err": 0 + }, + "20": { + "ok": 288, + "err": 0 + }, + "21": { + "ok": 577, + "err": 0 + }, + "22": { + "ok": 565, + "err": 0 + }, + "23": { + "ok": 531, + "err": 0 + }, + "24": { + "ok": 575, + "err": 0 + }, + "25": { + "ok": 574, + "err": 0 + }, + "26": { + "ok": 531, + "err": 0 + }, + "27": { + "ok": 559, + "err": 0 + }, + "28": { + "ok": 565, + "err": 0 + }, + "29": { + "ok": 572, + "err": 0 + }, + "30": { + "ok": 299, + "err": 0 + }, + "31": { + "ok": 393, + "err": 0 + }, + "32": { + "ok": 507, + "err": 0 + }, + "33": { + "ok": 525, + "err": 0 + }, + "34": { + "ok": 514, + "err": 0 + }, + "35": { + "ok": 556, + "err": 0 + }, + "36": { + "ok": 572, + "err": 0 + }, + "37": { + "ok": 506, + "err": 0 + }, + "38": { + "ok": 501, + "err": 0 + }, + "39": { + "ok": 468, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/ps-after-restart.txt b/evidence/ps-after-restart.txt new file mode 100644 index 00000000..0e36828a --- /dev/null +++ b/evidence/ps-after-restart.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 17 minutes ago Up 12 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 17 minutes ago Up 12 minutes (healthy) 8080/tcp +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 19 hours ago Up 19 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 19 hours ago Up 12 minutes (healthy) 5432/tcp +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 8 hours ago Up 12 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 32 hours ago Up 12 minutes (healthy) 6379/tcp diff --git a/evidence/ps-before.txt b/evidence/ps-before.txt new file mode 100644 index 00000000..a13979ad --- /dev/null +++ b/evidence/ps-before.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 29 minutes ago Up 29 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-messenger:lab4 "sh -c 'exec uvicorn…" api 29 minutes ago Up 29 minutes (healthy) 8080/tcp +mtusi-lab4-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 12 hours ago Up 12 hours 127.0.0.1:8080->8080/tcp +mtusi-lab4-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 12 hours ago Up 10 hours (healthy) 5432/tcp +mtusi-lab4-s3-1 chrislusf/seaweedfs:4.53 "/entrypoint.sh serv…" s3 29 minutes ago Up 17 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi-lab4-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 25 hours ago Up 10 hours (healthy) 6379/tcp diff --git a/evidence/s3-init-rerun.txt b/evidence/s3-init-rerun.txt new file mode 100644 index 00000000..567af3de --- /dev/null +++ b/evidence/s3-init-rerun.txt @@ -0,0 +1,6 @@ + Container mtusi-lab4-s3-init-run-59b11da77c9b Creating + Container mtusi-lab4-s3-init-run-59b11da77c9b Created +bucket messenger-attachments already exists + Container mtusi-lab4-s3-init-run-770039d96f76 Creating + Container mtusi-lab4-s3-init-run-770039d96f76 Created +bucket messenger-attachments already exists diff --git a/evidence/s3-private-check.txt b/evidence/s3-private-check.txt new file mode 100644 index 00000000..91652e39 --- /dev/null +++ b/evidence/s3-private-check.txt @@ -0,0 +1,24 @@ +# bucket=messenger-attachments, проверяется объект attachments/21978dc4-8c5c-4641-9263-16f6e904804d +PASS анонимный GET объекта напрямую из S3 -> HTTP 403 +PASS анонимный список bucket -> HTTP 403 +PASS анонимный список всех bucket -> HTTP 403 +# ответы S3 для API-ключа: {'create_bucket': 'AccessDenied', 'put_to_other_bucket': 'AccessDenied', 'get_from_other_bucket': 'AccessDenied', 'put_object_own_bucket': 'ALLOWED', 'delete_probe_own_bucket': 'ALLOWED'} +PASS API-ключ не может создавать bucket +PASS API-ключ не может писать в чужой bucket +PASS API-ключ не читает чужой bucket (NoSuchBucket/AccessDenied) +PASS в своём bucket API-ключ работает (put/delete) +# /run/secrets в api: database_url s3_api_access_key_id s3_api_secret_access_key +PASS в контейнере api нет админских S3-секретов +PASS S3 не публикует порты на хост (docker port: 'пусто') +PASS lb -> s3:9000 недостижим +PASS lb -> s3:8888 недостижим +PASS lb -> s3:9333 недостижим +PASS postgres -> s3:9000 недостижим +PASS postgres -> s3:8888 недостижим +PASS postgres -> s3:9333 недостижим +PASS valkey -> s3:9000 недостижим +PASS valkey -> s3:8888 недостижим +PASS valkey -> s3:9333 недостижим +INFO порты s3, открытые из api (внутри изолированной сети storage): {"9000": true, "8333": false, "8888": true, "9333": true, "8080": true} + +ИТОГ: PASS diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/track2-consistency.txt b/evidence/track2-consistency.txt new file mode 100644 index 00000000..23b1ae1e --- /dev/null +++ b/evidence/track2-consistency.txt @@ -0,0 +1,32 @@ +1. Orphan file created successfully! + Container mtusi-lab4-s3-1 Running + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-s3-init-1 Starting + Container mtusi-lab4-s3-init-1 Started + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-reconciler-run-6356b2614037 Creating + Container mtusi-lab4-reconciler-run-6356b2614037 Created +Error response from daemon: failed to create task for container: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: exec: "--dry-run": executable file not found in $PATH + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-s3-1 Running + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-s3-init-1 Starting + Container mtusi-lab4-s3-init-1 Started + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-reconciler-run-f9528845606a Creating + Container mtusi-lab4-reconciler-run-f9528845606a Created +2026-11-13 20:32:44,185 [INFO] Starting S3 reconciliation... +2026-11-13 20:32:44,186 [INFO] Found 2 objects in PostgreSQL. +2026-11-13 20:32:44,245 [INFO] Found 1 orphaned objects to delete. +2026-11-13 20:32:44,246 [INFO] Deleted orphan: attachments/orphan-test-file +2026-11-13 20:32:44,246 [INFO] Reconciliation complete. +УСПЕХ: Файл успешно удален (404 Not Found) diff --git a/evidence/track2-experiment.txt b/evidence/track2-experiment.txt new file mode 100644 index 00000000..60de4cdb --- /dev/null +++ b/evidence/track2-experiment.txt @@ -0,0 +1,48 @@ +== 0. Базовая очистка (чтобы счётчики ниже были точными) + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 57, "dry_run": false, "elapsed_s": 0.076, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 0, "orphans_fresh_skipped": 0, "peak_rss_mib": 51.6, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 1, "s3_list_s": 0.075, "s3_objects": 55, "size_mismatch": 0} [exit 0] +== 1. Данные: используемые и неиспользуемые вложения +== 2. Внедрение ошибок. S3-сторона: 3 объекта без метаданных; БД-сторона: у 2 записей объект удалён +== 3. DRY-RUN: обнаруживает, но ничего не меняет + reconciler (dry-run) -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": true, "elapsed_s": 0.063, "grace_seconds": 30, "missing_found": 2, "missing_fresh_skipped": 1, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 3, "orphans_fresh_skipped": 1, "peak_rss_mib": 52.6, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 1, "s3_list_s": 0.062, "s3_objects": 61, "size_mismatch": 0} [exit 0] +PASS dry-run нашёл 3 сирот и 2 потерянных объекта +PASS свежие сирота и запись (моложе grace) пропущены +PASS dry-run ничего не изменил в счётчиках +PASS после dry-run все сироты ещё в S3 +PASS после dry-run состояния записей не изменены +== 4. APPLY и сразу повторный APPLY (идемпотентность) + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.068, "grace_seconds": 30, "missing_found": 2, "missing_fresh_skipped": 1, "missing_marked": 2, "orphans_deleted": 3, "orphans_found": 3, "orphans_fresh_skipped": 1, "peak_rss_mib": 51.8, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 1, "s3_list_s": 0.06, "s3_objects": 61, "size_mismatch": 0} [exit 0] + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.064, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 1, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 0, "orphans_fresh_skipped": 1, "peak_rss_mib": 51.6, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 1, "s3_list_s": 0.062, "s3_objects": 58, "size_mismatch": 0} [exit 0] +PASS удалены 3 сироты, помечены 2 потерянных объекта +PASS повторный apply ничего не находит и не меняет +PASS свежая сирота НЕ удалена (grace period) +PASS свежая запись без объекта НЕ помечена (grace period) +PASS старые сироты удалены из S3 +PASS используемое вложение старше grace цело и скачивается +PASS неиспользуемое, но с метаданными вложение старше grace цело и скачивается +PASS потерянные вложения: скачивание 409, а не 500 +PASS метаданные потерянного вложения остаются (запись не удалена) +PASS потерянное вложение нельзя прикрепить к новому сообщению (409) +== 6. Восстановление: объект вернули на то же место + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.066, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 1, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 0, "orphans_fresh_skipped": 1, "peak_rss_mib": 51.6, "race_skipped": 0, "restore_found": 1, "restored": 1, "s3_list_pages": 1, "s3_list_s": 0.062, "s3_objects": 59, "size_mismatch": 0} [exit 0] +PASS запись снова ok после проверки размера и SHA-256 +PASS восстановленное вложение скачивается +== 7. Прошёл grace: бывшие свежие объекты теперь обрабатываются + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.068, "grace_seconds": 30, "missing_found": 1, "missing_fresh_skipped": 0, "missing_marked": 1, "orphans_deleted": 1, "orphans_found": 1, "orphans_fresh_skipped": 0, "peak_rss_mib": 51.6, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 1, "s3_list_s": 0.062, "s3_objects": 59, "size_mismatch": 0} [exit 0] +PASS теперь удалена вторая сирота и помечена вторая потеря +== 8. Стоимость обхода и предохранитель +-- seed 1000 объектов-сирот + reconciler (dry-run) -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": true, "elapsed_s": 0.113, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 1000, "orphans_fresh_skipped": 0, "peak_rss_mib": 54.0, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 2, "s3_list_s": 0.099, "s3_objects": 1058, "size_mismatch": 0} [exit 0] + dry-run N=1000: reconciler elapsed=0.113s s3_list=0.099s pages=2 db_rows=62 peak_rss=54.0MiB (с запуском контейнера 0.9s) +PASS dry-run видит 1000 сирот + reconciler --apply --max-actions 1100 -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.826, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 1000, "orphans_found": 1000, "orphans_fresh_skipped": 0, "peak_rss_mib": 53.2, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 2, "s3_list_s": 0.093, "s3_objects": 1058, "size_mismatch": 0} [exit 0] +PASS очистка 1000 сирот выполнена +-- seed 5000 объектов-сирот + reconciler (dry-run) -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": true, "elapsed_s": 0.319, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 5000, "orphans_fresh_skipped": 0, "peak_rss_mib": 56.9, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 6, "s3_list_s": 0.259, "s3_objects": 5058, "size_mismatch": 0} [exit 0] + dry-run N=5000: reconciler elapsed=0.319s s3_list=0.259s pages=6 db_rows=62 peak_rss=56.9MiB (с запуском контейнера 1.2s) +PASS dry-run видит 5000 сирот + reconciler --apply -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 0.32, "error": "too_many_actions", "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 0, "orphans_found": 5000, "orphans_fresh_skipped": 0, "peak_rss_mib": 55.8, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 6, "s3_list_s": 0.262, "s3_objects": 5058, "size_mismatch": 0} [exit 3] +PASS предохранитель: массовое удаление выше --max-actions отклонено, ничего не удалено + reconciler --apply --max-actions 5100 -> {"db_query_s": 0.001, "db_rows": 62, "dry_run": false, "elapsed_s": 5.797, "grace_seconds": 30, "missing_found": 0, "missing_fresh_skipped": 0, "missing_marked": 0, "orphans_deleted": 5000, "orphans_found": 5000, "orphans_fresh_skipped": 0, "peak_rss_mib": 56.1, "race_skipped": 0, "restore_found": 0, "restored": 0, "s3_list_pages": 6, "s3_list_s": 0.249, "s3_objects": 5058, "size_mismatch": 0} [exit 0] +PASS очистка 5000 сирот выполнена + +ИТОГ: PASS diff --git a/evidence/track2-grace-period.txt b/evidence/track2-grace-period.txt new file mode 100644 index 00000000..99af642b --- /dev/null +++ b/evidence/track2-grace-period.txt @@ -0,0 +1,16 @@ + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-s3-1 Running + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-s3-init-1 Starting + Container mtusi-lab4-s3-init-1 Started + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-reconciler-run-c1fad231a43b Creating + Container mtusi-lab4-reconciler-run-c1fad231a43b Created +2026-11-13 20:35:15,514 [INFO] Starting S3 reconciliation... +2026-11-13 20:35:15,516 [INFO] Found 2 objects in PostgreSQL. +2026-11-13 20:35:15,588 [INFO] No orphaned objects found. +УСПЕХ: Свежий файл НЕ удален, grace period работает! diff --git a/evidence/upload-memory-10-single-idat.json b/evidence/upload-memory-10-single-idat.json new file mode 100644 index 00000000..39a5a114 --- /dev/null +++ b/evidence/upload-memory-10-single-idat.json @@ -0,0 +1,27 @@ +{ + "mode": "single-idat", + "concurrency": 10, + "file_mib": 9.81, + "payload_total_mib": 98.1, + "wall_s": 6.1, + "statuses": [ + 201 + ], + "ok": 10, + "containers": { + "1a491e91605b": { + "limit_mib": 256, + "baseline_anon_mib": 77.2, + "peak_anon_mib": 83.1, + "anon_growth_mib": 5.9, + "peak_current_mib_incl_page_cache": 141.2 + }, + "00be497bb3e0": { + "limit_mib": 256, + "baseline_anon_mib": 76.2, + "peak_anon_mib": 81.2, + "anon_growth_mib": 5.1, + "peak_current_mib_incl_page_cache": 136.3 + } + } +} \ No newline at end of file diff --git a/evidence/upload-memory-10.json b/evidence/upload-memory-10.json new file mode 100644 index 00000000..4f1e5cb4 --- /dev/null +++ b/evidence/upload-memory-10.json @@ -0,0 +1,27 @@ +{ + "mode": "normal", + "concurrency": 10, + "file_mib": 9.81, + "payload_total_mib": 98.1, + "wall_s": 6.1, + "statuses": [ + 201 + ], + "ok": 10, + "containers": { + "1a491e91605b": { + "limit_mib": 256, + "baseline_anon_mib": 63.7, + "peak_anon_mib": 79.3, + "anon_growth_mib": 15.6, + "peak_current_mib_incl_page_cache": 137.0 + }, + "00be497bb3e0": { + "limit_mib": 256, + "baseline_anon_mib": 61.5, + "peak_anon_mib": 80.1, + "anon_growth_mib": 18.6, + "peak_current_mib_incl_page_cache": 138.6 + } + } +} \ No newline at end of file diff --git a/evidence/upload-memory-20-retry.json b/evidence/upload-memory-20-retry.json new file mode 100644 index 00000000..0bd99b87 --- /dev/null +++ b/evidence/upload-memory-20-retry.json @@ -0,0 +1,27 @@ +{ + "mode": "normal", + "concurrency": 20, + "file_mib": 9.81, + "payload_total_mib": 196.2, + "wall_s": 8.9, + "statuses": [ + 201 + ], + "ok": 20, + "containers": { + "c816acfa90c4": { + "limit_mib": 256, + "baseline_anon_mib": 51.3, + "peak_anon_mib": 100.8, + "anon_growth_mib": 49.5, + "peak_current_mib_incl_page_cache": 212.8 + }, + "1c8d7116a019": { + "limit_mib": 256, + "baseline_anon_mib": 64.0, + "peak_anon_mib": 98.5, + "anon_growth_mib": 34.4, + "peak_current_mib_incl_page_cache": 211.6 + } + } +} \ No newline at end of file diff --git a/evidence/upload-memory-20.json b/evidence/upload-memory-20.json new file mode 100644 index 00000000..9281437a --- /dev/null +++ b/evidence/upload-memory-20.json @@ -0,0 +1,28 @@ +{ + "mode": "normal", + "concurrency": 20, + "file_mib": 9.81, + "payload_total_mib": 196.2, + "wall_s": 8.5, + "statuses": [ + 201, + 503 + ], + "ok": 1, + "containers": { + "1a491e91605b": { + "limit_mib": 256, + "baseline_anon_mib": 77.8, + "peak_anon_mib": 96.7, + "anon_growth_mib": 18.9, + "peak_current_mib_incl_page_cache": 202.2 + }, + "00be497bb3e0": { + "limit_mib": 256, + "baseline_anon_mib": 76.5, + "peak_anon_mib": 108.6, + "anon_growth_mib": 32.1, + "peak_current_mib_incl_page_cache": 204.2 + } + } +} \ No newline at end of file diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..a422f48f --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,39 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 1s + timeout client 30s + timeout server 10s + timeout check 2s + timeout queue 5s + retries 3 + option redispatch + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold nx 1s + hold timeout 1s + accepted_payload_size 8192 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + # Проверка /health/ready раз в 1 с; backend исключается после 2 неудач подряд, + # возвращается после 2 успехов. Худшее время обнаружения: 2 x 1 с + timeout check 2 с = 4 с. + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 diff --git a/infra/s3/s3-init.sh b/infra/s3/s3-init.sh new file mode 100755 index 00000000..c201ffaa --- /dev/null +++ b/infra/s3/s3-init.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Однократный bootstrap: создаёт приватный bucket. Повторный запуск безопасен +set -eu +export AWS_ACCESS_KEY_ID="$(cat /run/secrets/s3_admin_access_key_id)" +export AWS_SECRET_ACCESS_KEY="$(cat /run/secrets/s3_admin_secret_access_key)" +export AWS_DEFAULT_REGION="${S3_REGION:-us-east-1}" +EP="${S3_ENDPOINT:-http://s3:9000}" +BUCKET="${S3_BUCKET:-messenger-attachments}" + +if aws --endpoint-url "$EP" s3api head-bucket --bucket "$BUCKET" >/dev/null 2>&1; then + echo "bucket $BUCKET already exists" +elif out=$(aws --endpoint-url "$EP" s3api create-bucket --bucket "$BUCKET" 2>&1); then + echo "bucket $BUCKET created" +elif echo "$out" | grep -Eq 'BucketAlreadyOwnedByYou|BucketAlreadyExists'; then + echo "bucket $BUCKET already exists" +else + echo "$out" >&2; exit 1 +fi \ No newline at end of file diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..a43b5358 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,63 @@ +-- 001_init: базовая схема лабы 2. Повторный запуск безопасен (IF NOT EXISTS). + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, -- Argon2id, никогда не отдаётся в API + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK ( + char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> '' + ) +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK ( + char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> '' + ) +); + +-- Порядок списка чатов: (created_at, id) +CREATE INDEX IF NOT EXISTS chats_created_idx ON chats (created_at, id); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, -- порядок member_ids как в запросе + PRIMARY KEY (chat_id, user_id) +); + +-- Список чатов пользователя и проверка членства +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK ( + char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> '' + ) +); + +-- Keyset-пагинация истории: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT n -> index scan без сортировки и полного прохода +CREATE INDEX IF NOT EXISTS messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, -- SHA-256 токена; сам токен не хранится + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); + +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at); \ No newline at end of file diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..ba0c7db7 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,13 @@ +-- 002: общая идемпотентность отправки сообщений +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 канонического тела + message_id uuid REFERENCES messages (id), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_created_idx ON idempotency_keys (created_at); \ No newline at end of file diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..41a1d89c --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,37 @@ +-- 003: вложения (лаба 4). Метаданные в PostgreSQL, байты в приватном S3-bucket +-- Миграция только добавляет таблицы и ослабляет проверку текста, поэтому совместима с предыдущей версией API + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, -- только для отображения; не используется как путь или ключ S3 + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, -- SHA-256 оригинала, lowercase hex + object_key text NOT NULL, -- ключ в S3, назначает сервер (случайный UUID) + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255) +); + +-- Вложения чата: проверка принадлежности и сверка с S3 +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id, created_at, id); + +-- Связь сообщение -> вложения; position сохраняет порядок attachment_ids +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id), + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx ON message_attachments (attachment_id); + +-- Сообщение может быть без текста, если есть вложение; непустота проверяется приложением +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000); \ No newline at end of file diff --git a/migrations/004_attachment_integrity.sql b/migrations/004_attachment_integrity.sql new file mode 100644 index 00000000..4c6609ac --- /dev/null +++ b/migrations/004_attachment_integrity.sql @@ -0,0 +1,6 @@ +ALTER TABLE attachments + ADD COLUMN IF NOT EXISTS storage_state text NOT NULL DEFAULT 'ok', + ADD COLUMN IF NOT EXISTS storage_checked_at timestamptz; + +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_storage_state; +ALTER TABLE attachments ADD CONSTRAINT attachments_storage_state CHECK (storage_state IN ('ok', 'missing')); \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..48f4affc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.115.2 +h11==0.16.0 +idna==3.20 +pydantic==2.14.1 +pydantic-core==2.50.1 +starlette==0.40.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uvicorn==0.32.0 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +redis==5.2.1 +boto3==1.35.36 +botocore==1.35.36 +s3transfer==0.10.2 +jmespath==1.0.1 +python-dateutil==2.9.0.post0 +six==1.16.0 +urllib3==2.2.3 +pillow==10.4.0 \ No newline at end of file diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh new file mode 100755 index 00000000..616fb6fa --- /dev/null +++ b/scripts/gen-secrets.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Создаёт локальные секреты, которых ещё нет. Существующие не перезаписывает. В Git не попадают (secrets/ в .gitignore) +set -eu +cd "$(dirname "$0")/.." +mkdir -p secrets +BUCKET="${S3_BUCKET:-messenger-attachments}" +rnd() { openssl rand -hex "$1"; } +put() { [ -s "secrets/$1" ] || { printf '%s' "$2" > "secrets/$1"; echo "created secrets/$1"; }; } + +if [ ! -s secrets/postgres_password ]; then + PW=$(rnd 24) + printf '%s' "$PW" > secrets/postgres_password + printf 'postgresql://messenger:%s@postgres:5432/messenger' "$PW" > secrets/database_url + echo "created secrets/postgres_password, secrets/database_url" +fi +put s3_admin_access_key_id "admin$(rnd 8)" +put s3_admin_secret_access_key "$(rnd 24)" +put s3_api_access_key_id "api$(rnd 8)" +put s3_api_secret_access_key "$(rnd 24)" + +if [ ! -s secrets/s3_config ]; then + # Конфиг identities SeaweedFS: admin (создание bucket) и api (только свой bucket) + printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}' \ + "$(cat secrets/s3_admin_access_key_id)" "$(cat secrets/s3_admin_secret_access_key)" \ + "$(cat secrets/s3_api_access_key_id)" "$(cat secrets/s3_api_secret_access_key)" \ + "$BUCKET" "$BUCKET" "$BUCKET" > secrets/s3_config + echo "created secrets/s3_config" +fi +chmod 644 secrets/* # локальный стенд с синтетическими данными: нужно, чтобы читал не-root пользователь в контейнере \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f6bf6759 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Перезапуск API и ВСЕХ хранилищ (PostgreSQL, Valkey, S3) с сохранением volumes (никаких down -v). +# В конце ждёт, пока S3 реально отдаёт данные: контейнер healthy раньше, чем volume-сервер готов читать. +set -eu +cd "$(dirname "$0")/.." +C="docker compose --env-file .env" +$C stop api +$C restart postgres valkey s3 +$C start api +i=0 +while [ "$i" -lt 60 ]; do + if $C exec -T api python -c " +from src import storage, config +r = storage.client().list_objects_v2(Bucket=config.S3_BUCKET, Prefix='attachments/', MaxKeys=1) +for o in r.get('Contents', []): + storage.get_object(o['Key']).read(1) +" >/dev/null 2>&1; then + exit 0 + fi + i=$((i + 1)); sleep 1 +done +echo "S3 data path is not ready" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..cce3bf45 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Поочерёдная замена API-реплик новой версией образа (APP_VERSION=v2 scripts/rolling-update.sh). +# Миграции запускаются ОДИН раз до замены; migrations/002 только добавляет таблицу, поэтому старая версия с ней совместима. +set -eu +cd "$(dirname "$0")/.." +: "${APP_VERSION:?set new version, e.g. APP_VERSION=v2}" +export APP_VERSION +COMPOSE="docker compose --env-file .env" + +$COMPOSE build migrate +$COMPOSE run --rm migrate +N=$($COMPOSE ps -q api | wc -l | tr -d ' ') + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge "$N" ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +for cid in $($COMPOSE ps -q api); do + echo "replacing $cid" >&2 + docker stop -t 20 "$cid" >/dev/null # SIGTERM: readiness 503, drain 5 с, затем завершение + docker rm "$cid" >/dev/null + $COMPOSE up -d --no-deps --no-recreate --scale api="$N" api # создаётся реплика из нового образа + wait_healthy + sleep 5 # LB добавляет новую реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/scripts/run-failover-series.sh b/scripts/run-failover-series.sh new file mode 100755 index 00000000..7b9841ed --- /dev/null +++ b/scripts/run-failover-series.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Базовая нагрузка, затем по очереди отказ каждой API-реплики под нагрузкой с восстановлением между ними. +set -eu +cd "$(dirname "$0")/.." +COMPOSE="docker compose --env-file .env" + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge 2 ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +python3 student_tests/loadgen.py --duration 30 --output evidence/load-baseline.json +for idx in 0 1; do + python3 student_tests/loadgen.py --duration 40 --action-script scripts/stop-one.sh \ + --action-at 10 --target-index "$idx" --output "evidence/load-stop-$idx.json" + $COMPOSE ps -a > "evidence/load-stop-$idx-ps.txt" + $COMPOSE start api + wait_healthy + sleep 10 # LB возвращает реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/scripts/show-instances.sh b/scripts/show-instances.sh new file mode 100755 index 00000000..15c74d5c --- /dev/null +++ b/scripts/show-instances.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Одна сессия, 10 запросов через LB: в ответах должны быть >=2 разных X-Instance-Id. +set -eu +BASE="${BASE_URL:-http://localhost:8080}" +U="demo_$(date +%s)" +curl -sf -X POST "$BASE/api/v1/users" -H 'Content-Type: application/json' \ + -d "{\"username\":\"$U\",\"display_name\":\"Demo\",\"password\":\"demo-password-123\"}" >/dev/null +TOKEN=$(curl -sf -X POST "$BASE/api/v1/auth/sessions" -u "$U:demo-password-123" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') +for i in 1 2 3 4 5 6 7 8 9 10; do + curl -si "$BASE/api/v1/users/me" -H "Authorization: Bearer $TOKEN" | grep -i '^x-instance-id' +done | sort | uniq -c \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..6939ede0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Останавливает ОДНУ API-реплику по TARGET_INSTANCE_ID (= hostname контейнера = короткий ID) +# Реплика остаётся остановленной. БД, LB и volumes не затрагиваются +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + ""|*[!A-Za-z0-9_.-]*) echo "bad instance id" >&2; exit 2 ;; +esac +cd "$(dirname "$0")/.." +CID=$(docker ps --filter "label=com.docker.compose.service=api" \ + --filter "id=$TARGET_INSTANCE_ID" --format '{{.ID}}') +[ -n "$CID" ] || { echo "no running api container for this instance" >&2; exit 1; } +docker stop -t 2 "$CID" >/dev/null +echo "stopped api container $CID" >&2 \ No newline at end of file diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..d90bb44e --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,238 @@ +import asyncio +import hashlib +import logging +import tempfile +import unicodedata +import uuid +from typing import Optional +import struct +import zlib + +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import StreamingResponse +from PIL import Image + +from . import config, storage +from .auth import current_user +from .common import iso, make_error_response + +logger = logging.getLogger("messenger") +router = APIRouter() + +Image.MAX_IMAGE_PIXELS = None # лимит пикселей применяем сами, до декодирования (Pillow при open() пиксели не читает) + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +ALLOWED_TYPES = ("image/png", "image/jpeg") +KNOWN_VARIANTS = ("original", "thumbnail", "crop", "optimized") +CHUNK = 64 * 1024 +SPOOL_IN_MEMORY = 1024 * 1024 # больше 1 MiB тело временно уходит на диск контейнера, а не в RAM + +_COLUMNS = "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, " \ + "a.object_key, a.storage_state, a.created_at, a.updated_at" + + +def attachment_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +def _bad_filename(name: str) -> bool: + # '/', '\', NUL и управляющие символы запрещены; имя только для отображения + return (not name.strip()) or any( + ch in "/\\" or unicodedata.category(ch) == "Cc" for ch in name) + +class _Reject(Exception): + """args[0] - HTTP-код (413 или 415).""" + + +def _check_pixels(width: int, height: int) -> None: + if width <= 0 or height <= 0: + raise _Reject(415) + if width * height > config.MAX_IMAGE_PIXELS: + raise _Reject(413) + + +def _verify_png(fileobj) -> None: + """Потоковая проверка PNG: IHDR, CRC каждого чанка, IDAT и IEND. Чанки читаются блоками по 64 КиБ.""" + fileobj.seek(len(PNG_SIGNATURE)) + first, seen_idat = True, False + while True: + header = fileobj.read(8) + if len(header) != 8: + raise _Reject(415) + length, ctype = struct.unpack(">I4s", header) + if first and (ctype != b"IHDR" or length != 13): + raise _Reject(415) + crc, remaining, head = zlib.crc32(ctype), length, b"" + while remaining: + block = fileobj.read(min(CHUNK, remaining)) + if not block: + raise _Reject(415) + if first and len(head) < 8: + head += block[: 8 - len(head)] + crc = zlib.crc32(block, crc) + remaining -= len(block) + tail = fileobj.read(4) + if len(tail) != 4 or struct.unpack(">I", tail)[0] != (crc & 0xFFFFFFFF): + raise _Reject(415) + if first: + _check_pixels(*struct.unpack(">II", head)) + first = False + elif ctype == b"IDAT": + seen_idat = True + elif ctype == b"IEND": + if not seen_idat: + raise _Reject(415) + return + + +def _verify_jpeg(fileobj) -> None: + fileobj.seek(0) + try: + with Image.open(fileobj) as img: + if img.format not in ("JPEG", "MPO"): # MPO - обычный JPEG с блоком Multi-Picture (фото телефонов) + raise _Reject(415) + _check_pixels(*img.size) + except (OSError, SyntaxError, ValueError, Image.UnidentifiedImageError, Image.DecompressionBombError): + raise _Reject(415) + fileobj.seek(0, 2) + tail_len = min(fileobj.tell(), 4096) + fileobj.seek(-tail_len, 2) + if b"\xff\xd9" not in fileobj.read(tail_len): + raise _Reject(415) + + +def inspect_image(fileobj, declared: str) -> Optional[int]: + fileobj.seek(0) + head = fileobj.read(8) + if head.startswith(PNG_SIGNATURE): + actual = "image/png" + elif head[:3] == b"\xff\xd8\xff": + actual = "image/jpeg" + else: + return 415 + if actual != declared: # подмена MIME + return 415 + try: + (_verify_png if actual == "image/png" else _verify_jpeg)(fileobj) + except _Reject as rejected: + return rejected.args[0] + return None + + +_ERRORS = { + 413: ("payload_too_large", "Image is too large"), + 415: ("unsupported_media_type", "Unsupported or corrupted image"), +} + + +async def _accessible(pool, attachment_id, user_id): + return await pool.fetchrow( + f"SELECT {_COLUMNS} FROM attachments a " + "JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1", + attachment_id, user_id) + + +@router.post("/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: uuid.UUID, request: Request, + filename: str = Query(min_length=1, max_length=255), + user=Depends(current_user)): + req_id = request.state.request_id + pool = request.app.state.pool + # Сначала доступ к чату, потом чтение тела: постороннему байты не нужны. + if not await pool.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user["id"]): + return make_error_response(404, "not_found", "Chat not found", req_id) + if _bad_filename(filename): + return make_error_response(400, "invalid_request", "Invalid filename", req_id) + declared = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared not in ALLOWED_TYPES: + return make_error_response(415, *_ERRORS[415], req_id) + length = request.headers.get("content-length", "") + if length.isdigit() and int(length) > config.MAX_UPLOAD_BYTES: # отказ, не читая тело + return make_error_response(413, *_ERRORS[413], req_id) + + digest, size = hashlib.sha256(), 0 + with tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY) as buf: + async for chunk in request.stream(): # потоково: в памяти один чанк, лимит проверяется на лету + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + return make_error_response(413, *_ERRORS[413], req_id) + digest.update(chunk) + buf.write(chunk) + problem = await asyncio.to_thread(inspect_image, buf, declared) + if problem: + return make_error_response(problem, *_ERRORS[problem], req_id) + + attachment_id = uuid.uuid4() + key = f"attachments/{attachment_id}" # ключ назначает сервер; filename в нём не участвует + # 1) байты в S3, 2) метаданные в PostgreSQL. 201 только после обоих шагов. + try: + await asyncio.to_thread(storage.put_object, key, buf, size, declared) + row = await pool.fetchrow( + "INSERT INTO attachments AS a (id, chat_id, owner_id, filename, content_type, size_bytes, " + "sha256, object_key, created_at, updated_at) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), now()) " + f"RETURNING {_COLUMNS}", + attachment_id, chat_id, user["id"], filename, declared, size, digest.hexdigest(), key) + except BaseException: + try: + await asyncio.shield(asyncio.to_thread(storage.delete_object, key)) + except Exception: + logger.error("Orphan object left in S3: %s", key, extra={"request_id": req_id}) + raise + return attachment_dict(row) + + +@router.get("/attachments/{attachment_id}") +async def get_attachment(attachment_id: uuid.UUID, request: Request, user=Depends(current_user)): + row = await _accessible(request.app.state.pool, attachment_id, user["id"]) + if row is None: + return make_error_response(404, "not_found", "Attachment not found", request.state.request_id) + return attachment_dict(row) + + +@router.get("/attachments/{attachment_id}/content") +async def get_attachment_content(attachment_id: uuid.UUID, request: Request, variant: str = "original", + user=Depends(current_user)): + req_id = request.state.request_id + row = await _accessible(request.app.state.pool, attachment_id, user["id"]) + if row is None: + return make_error_response(404, "not_found", "Attachment not found", req_id) + if variant not in KNOWN_VARIANTS: + return make_error_response(400, "invalid_request", "Unknown variant", req_id) + if variant != "original": + return make_error_response(404, "not_found", "Variant not found", req_id)# в этой лабе вариантов нет + if row["storage_state"] != "ok": + return make_error_response(409, "conflict", "Attachment content is unavailable", req_id) + try: + body = await asyncio.to_thread(storage.get_object, row["object_key"]) + except storage.ObjectMissing: + logger.error("Object missing in S3 for attachment %s", attachment_id, extra={"request_id": req_id}) + return make_error_response(500, "internal_error", "Internal server error", req_id) + + async def stream(): + try: + while True: + chunk = await asyncio.to_thread(body.read, CHUNK) + if not chunk: + break + yield chunk + finally: + await asyncio.to_thread(body.close) + + return StreamingResponse(stream(), media_type=row["content_type"], + headers={"Content-Length": str(row["size_bytes"]), + "Cache-Control": "private, max-age=0"}) \ No newline at end of file diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..aaec2937 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,97 @@ +import asyncio +import base64 +import binascii +import hashlib +import secrets +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException, Request + +_hasher = PasswordHasher() +# Хеш-заглушка: для несуществующего пользователя тратим столько же времени, сколько для существующего. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} +# Не более двух одновременных хешей на реплику: Argon2 берёт ~64 МиБ на хеш, лимит контейнера 256 МиБ, CPU 0.5. +# Остальные запросы ждут очереди, а не вытесняют друг друга по памяти и не упираются в timeout LB. +_HASH_SLOTS = asyncio.Semaphore(2) + + +# Не более двух одновременных хешей на реплику: Argon2 берёт ~64 МиБ на хеш, лимит контейнера 256 МиБ, CPU 0.5. +# Остальные запросы ждут очереди, а не вытесняют друг друга по памяти и не упираются в timeout LB. +_HASH_SLOTS = asyncio.Semaphore(2) + + +async def hash_password(password: str) -> str: + async with _HASH_SLOTS: + return await asyncio.to_thread(_hasher.hash, password) # Argon2 грузит CPU, не блокируем event loop + + +async def verify_password(stored_hash: Optional[str], password: str) -> bool: + try: + async with _HASH_SLOTS: + return await asyncio.to_thread(_hasher.verify, stored_hash or _DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(32) # 256 бит случайности + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode()).digest() + + +def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]: + if not header: + return None + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value: + return None + try: + raw = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, sep, password = raw.partition(":") # двоеточие в пароле допустимо + return (username, password) if sep else None + + +def parse_bearer(header: Optional[str]) -> Optional[str]: + if not header: + return None + scheme, _, value = header.partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512: + return None + return value + + +_SESSION_SQL = """ +SELECT u.id, u.username, u.display_name, u.created_at, + EXTRACT(EPOCH FROM (s.expires_at - now())) AS remaining +FROM sessions s JOIN users u ON u.id = s.user_id +WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now() +""" + + +async def current_user(request: Request): + # Только Bearer. Сначала кэш, при промахе или сбое - PostgreSQL (источник истины). + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise HTTPException(401, headers=BEARER_CHALLENGE) + digest = token_hash(token) + cache = request.app.state.cache + + user = await cache.get(digest) + if user is None: + row = await request.app.state.pool.fetchrow(_SESSION_SQL, digest) + if row is None: + raise HTTPException(401, headers=BEARER_CHALLENGE) + user = {k: row[k] for k in ("id", "username", "display_name", "created_at")} + await cache.put(digest, user, int(row["remaining"])) + request.state.token_hash = digest + return user \ No newline at end of file diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..a2f14c8d --- /dev/null +++ b/src/cache.py @@ -0,0 +1,82 @@ +import asyncio +import json +import logging +import time +import uuid +from datetime import datetime +from typing import Optional + +import redis.asyncio as redis +from redis.exceptions import RedisError + +from . import config +from .common import iso + +logger = logging.getLogger("messenger") + +_FAILURES = (RedisError, OSError, asyncio.TimeoutError) +_RETRY_AFTER = 5.0 # после сбоя 5 секунд не обращаемся к кэшу, чтобы не ждать таймаут на каждом запросе + + +def _key(digest: bytes) -> str: + return "sess:" + digest.hex() + + +class SessionCache: + """Кэш проверенных сессий. Любой сбой = промах: ответ берётся из PostgreSQL.""" + + def __init__(self, url: str): + self._client = ( + redis.Redis.from_url( + url, decode_responses=True, + socket_timeout=config.CACHE_TIMEOUT, socket_connect_timeout=config.CACHE_TIMEOUT, + ) if url else None + ) + self._down_until = 0.0 + + def _usable(self) -> bool: + return self._client is not None and time.monotonic() >= self._down_until + + def _mark_down(self, exc: Exception) -> None: + self._down_until = time.monotonic() + _RETRY_AFTER + logger.warning("Cache unavailable, bypassing: %s", type(exc).__name__) + + async def get(self, digest: bytes) -> Optional[dict]: + if not self._usable(): + return None + try: + raw = await self._client.get(_key(digest)) + except _FAILURES as exc: + self._mark_down(exc) + return None + if raw is None: + return None + try: + d = json.loads(raw) + return {"id": uuid.UUID(d["id"]), "username": d["username"], + "display_name": d["display_name"], "created_at": datetime.fromisoformat(d["created_at"])} + except (ValueError, KeyError): + return None + + async def put(self, digest: bytes, user: dict, remaining_seconds: int) -> None: + ttl = min(remaining_seconds, config.CACHE_TTL_SECONDS) # кэш никогда не живёт дольше сессии + if ttl < 1 or not self._usable(): + return + value = json.dumps({"id": str(user["id"]), "username": user["username"], + "display_name": user["display_name"], "created_at": iso(user["created_at"])}) + try: + await self._client.set(_key(digest), value, ex=ttl) + except _FAILURES as exc: + self._mark_down(exc) + + async def delete(self, digest: bytes) -> None: + if not self._usable(): + return + try: + await self._client.delete(_key(digest)) + except _FAILURES as exc: + self._mark_down(exc) + + async def close(self) -> None: + if self._client is not None: + await self._client.aclose() \ No newline at end of file diff --git a/src/chats.py b/src/chats.py new file mode 100644 index 00000000..1838ad3d --- /dev/null +++ b/src/chats.py @@ -0,0 +1,287 @@ +import base64 +import json +import uuid +from datetime import datetime +from typing import List, Optional, Tuple + +import asyncpg +from fastapi import APIRouter, Depends, Query, Request +from pydantic import BaseModel, ConfigDict, Field + +from .auth import current_user +from .common import iso, make_error_response + +import hashlib +import re +from . import config + +router = APIRouter() + +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + +# Захват ключа. Параллельный INSERT ждёт коммита первого; просроченный ключ (старше TTL) перехватывается +_CLAIM_SQL = """ +INSERT INTO idempotency_keys AS k (user_id, method, path, idem_key, request_hash, created_at) +VALUES ($1, 'POST', $2, $3, $4, now()) +ON CONFLICT (user_id, method, path, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = NULL, created_at = EXCLUDED.created_at + WHERE k.created_at < now() - make_interval(secs => $5) +RETURNING 1 +""" + + +class StrictRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateChatRequest(StrictRequest): + title: str + member_ids: List[str] = Field(min_length=1, max_length=100) + + +class CreateMessageRequest(StrictRequest): + text: str + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + +def _req_id(request: Request) -> str: + return request.state.request_id + + +# курсор: непрозрачная строка, привязанная к ресурсу (scope) +def encode_cursor(scope: str, created_at: datetime, row_id: uuid.UUID) -> str: + raw = json.dumps([scope, iso(created_at), str(row_id)]).encode() + return base64.urlsafe_b64encode(raw).decode() + + +def decode_cursor(cursor: str, scope: str) -> Tuple[datetime, uuid.UUID]: + """ValueError, если курсор повреждён или выдан для другого ресурса.""" + try: + sc, ts, rid = json.loads(base64.urlsafe_b64decode(cursor.encode())) + if sc != scope: + raise ValueError("scope mismatch") + return datetime.fromisoformat(ts), uuid.UUID(rid) + except Exception as exc: + raise ValueError("invalid cursor") from exc + + +def _bad_text(value: str) -> bool: + return not value.strip() or "\x00" in value # NUL нельзя хранить в PostgreSQL text + + +_CHAT_COLUMNS = ( + "c.id, c.title, c.created_at, " + "array(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.position) AS member_ids" +) + + +def chat_dict(row) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], + "created_at": iso(row["created_at"]), + } + + +# attachment_ids в порядке, заданном при отправке +_MSG_COLUMNS = ( + "m.id, m.chat_id, m.sender_id, m.text, m.created_at, " + "array(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id " + "ORDER BY ma.position) AS attachment_ids" +) + + +def message_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + "attachment_ids": [str(a) for a in row["attachment_ids"]], + } + + +def _check_limit(limit: int) -> bool: + return 1 <= limit <= 100 + + +# чаты +@router.post("/chats", status_code=201) +async def create_chat(req: CreateChatRequest, request: Request, user=Depends(current_user)): + req_id = _req_id(request) + if not (1 <= len(req.title) <= 100) or _bad_text(req.title): + return make_error_response(400, "invalid_request", "Invalid title", req_id) + try: + members = [uuid.UUID(m) for m in req.member_ids] + except ValueError: + return make_error_response(400, "invalid_request", "Invalid member id", req_id) + if len(members) != len(set(members)): + return make_error_response(400, "invalid_request", "Duplicate members found", req_id) + if user["id"] not in members: + return make_error_response(400, "invalid_request", "Caller must be in member_ids", req_id) + + chat_id = uuid.uuid4() + try: + async with request.app.state.pool.acquire() as conn: + # Чат и все участники - одна транзакция: неизвестный участник откатывает всё. + async with conn.transaction(): + created_at = await conn.fetchval( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, clock_timestamp()) RETURNING created_at", + chat_id, req.title, + ) + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position) " + "SELECT $1, t.u, (t.p - 1)::smallint FROM unnest($2::uuid[]) WITH ORDINALITY AS t(u, p)", + chat_id, members, + ) + except asyncpg.ForeignKeyViolationError: + return make_error_response(400, "invalid_request", "Unknown member", req_id) + + return {"id": str(chat_id), "title": req.title, "member_ids": [str(m) for m in members], + "created_at": iso(created_at)} + + +@router.get("/chats") +async def list_chats(request: Request, limit: int = 50, + cursor: Optional[str] = Query(default=None, min_length=1, max_length=512), + user=Depends(current_user)): + req_id = _req_id(request) + if not _check_limit(limit): + return make_error_response(400, "invalid_request", "Limit must be between 1 and 100", req_id) + scope = "chats:" + str(user["id"]) + base = (f"SELECT {_CHAT_COLUMNS} FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $1 ") + if cursor is None: + rows = await request.app.state.pool.fetch( + base + "ORDER BY c.created_at, c.id LIMIT $2", user["id"], limit + 1) + else: + try: + ts, rid = decode_cursor(cursor, scope) + except ValueError: + return make_error_response(400, "invalid_request", "Invalid cursor", req_id) + rows = await request.app.state.pool.fetch( + base + "WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid) " + "ORDER BY c.created_at, c.id LIMIT $4", + user["id"], ts, rid, limit + 1) + items = rows[:limit] + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if len(rows) > limit else None + return {"items": [chat_dict(r) for r in items], "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}") +async def get_chat(chat_id: uuid.UUID, request: Request, user=Depends(current_user)): + row = await request.app.state.pool.fetchrow( + f"SELECT {_CHAT_COLUMNS} FROM chats c WHERE c.id = $1 " + "AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2)", + chat_id, user["id"], + ) + if row is None: + return make_error_response(404, "not_found", "Chat not found", _req_id(request)) + return chat_dict(row) + + +# сообщения +@router.post("/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: uuid.UUID, req: CreateMessageRequest, request: Request, + user=Depends(current_user)): + req_id = _req_id(request) + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + return make_error_response(400, "invalid_request", "Missing or invalid Idempotency-Key", req_id) + try: + attachment_ids = [uuid.UUID(a) for a in req.attachment_ids] + except ValueError: + return make_error_response(400, "invalid_request", "Invalid attachment id", req_id) + if len(attachment_ids) != len(set(attachment_ids)): + return make_error_response(400, "invalid_request", "Duplicate attachment ids", req_id) + if "\x00" in req.text or (not req.text.strip() and not attachment_ids): + return make_error_response(400, "invalid_request", "Empty message text", req_id) + if len(req.text) > 4000: + return make_error_response(400, "invalid_request", "Message text exceeds 4000 characters", req_id) + + path = f"/api/v1/chats/{chat_id}/messages" + request_hash = hashlib.sha256(json.dumps( + {"text": req.text, "attachment_ids": [str(a) for a in attachment_ids]}, + sort_keys=True, ensure_ascii=False).encode()).digest() + + # Захват ключа, вставка сообщения, связи с вложениями и привязка результата - одна транзакция: + # при сбое откатывается всё, и ключ не фиксирует неудачный результат. + async with request.app.state.pool.acquire() as conn: + async with conn.transaction(): + # Повтор тоже проверяет доступ. + member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user["id"]) + if not member: + return make_error_response(404, "not_found", "Chat not found", req_id) + if attachment_ids: # все вложения должны существовать и принадлежать ЭТОМУ чату + states = await conn.fetch( + "SELECT storage_state FROM attachments WHERE id = ANY($1::uuid[]) AND chat_id = $2", + attachment_ids, chat_id) + if len(states) != len(attachment_ids): + return make_error_response(404, "not_found", "Attachment not found", req_id) + if any(r["storage_state"] != "ok" for r in states): + return make_error_response(409, "conflict", "Attachment content is unavailable", req_id) + + claimed = await conn.fetchval( + _CLAIM_SQL, user["id"], path, key, request_hash, float(config.IDEMPOTENCY_TTL_SECONDS)) + if claimed is None: # ключ уже использован: повтор или конфликт + saved = await conn.fetchrow( + "SELECT request_hash, message_id FROM idempotency_keys " + "WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idem_key = $3", + user["id"], path, key) + if saved["request_hash"] != request_hash: + return make_error_response(409, "conflict", "Idempotency-Key reused with another body", req_id) + row = await conn.fetchrow( + f"SELECT {_MSG_COLUMNS} FROM messages m WHERE m.id = $1", saved["message_id"]) + return message_dict(row) + + message_id = uuid.uuid4() + created_at = await conn.fetchval( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "VALUES ($1, $2, $3, $4, clock_timestamp()) RETURNING created_at", + message_id, chat_id, user["id"], req.text) + if attachment_ids: + await conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) " + "SELECT $1, t.a, (t.p - 1)::smallint FROM unnest($2::uuid[]) WITH ORDINALITY AS t(a, p)", + message_id, attachment_ids) + await conn.execute( + "UPDATE idempotency_keys SET message_id = $1 " + "WHERE user_id = $2 AND method = 'POST' AND path = $3 AND idem_key = $4", + message_id, user["id"], path, key) + return {"id": str(message_id), "chat_id": str(chat_id), "sender_id": str(user["id"]), + "text": req.text, "created_at": iso(created_at), + "attachment_ids": [str(a) for a in attachment_ids]} + + + +@router.get("/chats/{chat_id}/messages") +async def get_messages(chat_id: uuid.UUID, request: Request, limit: int = 50, + cursor: Optional[str] = Query(default=None, min_length=1, max_length=512), + user=Depends(current_user)): + req_id = _req_id(request) + if not _check_limit(limit): + return make_error_response(400, "invalid_request", "Limit must be between 1 and 100", req_id) + pool = request.app.state.pool + is_member = await pool.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user["id"]) + if not is_member: + return make_error_response(404, "not_found", "Chat not found", req_id) + + scope = "msgs:" + str(chat_id) + base = f"SELECT {_MSG_COLUMNS} FROM messages m WHERE m.chat_id = $1 " + if cursor is None: + rows = await pool.fetch(base + "ORDER BY m.created_at, m.id LIMIT $2", chat_id, limit + 1) + else: + try: + ts, rid = decode_cursor(cursor, scope) + except ValueError: + return make_error_response(400, "invalid_request", "Invalid cursor", req_id) + rows = await pool.fetch( + base + "AND (m.created_at, m.id) > ($2::timestamptz, $3::uuid) ORDER BY m.created_at, m.id LIMIT $4", + chat_id, ts, rid, limit + 1) + items = rows[:limit] + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if len(rows) > limit else None + return {"items": [message_dict(r) for r in items], "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..ee8e73ad --- /dev/null +++ b/src/common.py @@ -0,0 +1,30 @@ +from datetime import datetime, timezone +from typing import Optional + +from fastapi.responses import JSONResponse +from . import config + + +def iso(dt: datetime) -> str: + """RFC 3339 UTC с Z; микросекунды нужны, чтобы курсор был точным.""" + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def make_error_response(status_code: int, code: str, message: str, req_id: str, headers: Optional[dict] = None): + h = {"X-Request-Id": req_id, "X-Instance-Id": config.INSTANCE_ID} + if headers: + h.update(headers) + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": req_id}}, + headers=h, + ) + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..d8a51ee9 --- /dev/null +++ b/src/config.py @@ -0,0 +1,48 @@ +import os +import socket +from pathlib import Path + + +def _read_secret(file_var: str, direct_var: str) -> str: + """Секрет из файла (DATABASE_URL_FILE), с запасным вариантом из переменной окружения.""" + path = os.environ.get(file_var) + if path: + return Path(path).read_text().strip() + return os.environ.get(direct_var, "") + + +DATABASE_URL = _read_secret("DATABASE_URL_FILE", "DATABASE_URL") +SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +DB_POOL_MIN = int(os.environ.get("DB_POOL_MIN", "2")) +DB_POOL_MAX = int(os.environ.get("DB_POOL_MAX", "10")) +DB_CONNECT_TIMEOUT = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) +DB_COMMAND_TIMEOUT = float(os.environ.get("DB_COMMAND_TIMEOUT", "5")) + +MIGRATIONS_DIR = Path(os.environ.get("MIGRATIONS_DIR", Path(__file__).resolve().parent.parent / "migrations")) +CACHE_URL = _read_secret("CACHE_URL_FILE", "CACHE_URL") +CACHE_TTL_SECONDS = int(os.environ.get("CACHE_TTL_SECONDS", "60")) +CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5")) + +# Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + +# SIGTERM: сколько секунд реплика отвечает 503 на /health/ready, пока LB её исключает, и продолжает обслуживать запросы. +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "5")) +APP_VERSION = os.environ.get("APP_VERSION", "dev") + + +# S3 (лаба 4): отдельные ограниченные credentials API, bucket приватный +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT = float(os.environ.get("S3_CONNECT_TIMEOUT", "2")) +S3_READ_TIMEOUT = float(os.environ.get("S3_READ_TIMEOUT", "5")) +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +# Reconciler (Трек 2) +RECONCILE_GRACE_SECONDS = int(os.environ.get("RECONCILE_GRACE_SECONDS", "600")) # по умолчанию 10 минут +RECONCILE_MAX_ACTIONS = int(os.environ.get("RECONCILE_MAX_ACTIONS", "1000")) \ No newline at end of file diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..d7292bdc --- /dev/null +++ b/src/db.py @@ -0,0 +1,56 @@ +import asyncio +import logging + +import asyncpg + +from . import config + +logger = logging.getLogger("messenger") + +# Произвольный константный ключ: один экземпляр выполняет миграции, остальные ждут. +MIGRATION_LOCK_KEY = 727_001 + + +async def create_pool() -> asyncpg.Pool: + # Пул ограничен по размеру; у соединений и команд есть таймауты. + return await asyncpg.create_pool( + dsn=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT, + ) + + +async def create_pool_with_retry(attempts: int = 30, delay: float = 1.0) -> asyncpg.Pool: + # БД в compose может подняться позже API. + for attempt in range(1, attempts + 1): + try: + return await create_pool() + except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc: + logger.warning("DB not ready (%d/%d): %s", attempt, attempts, type(exc).__name__) + await asyncio.sleep(delay) + raise RuntimeError("database is unavailable") + + +async def run_migrations(pool: asyncpg.Pool) -> None: + # Применяет migrations/*.sql по порядку имён, каждую один раз. + files = sorted(config.MIGRATIONS_DIR.glob("*.sql")) + async with pool.acquire() as conn: + await conn.execute("SELECT pg_advisory_lock($1)", MIGRATION_LOCK_KEY) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + applied = {r["version"] for r in await conn.fetch("SELECT version FROM schema_migrations")} + for path in files: + if path.name in applied: + continue + async with conn.transaction(): + await conn.execute(path.read_text()) + await conn.execute("INSERT INTO schema_migrations (version) VALUES ($1)", path.name) + logger.info("Applied migration %s", path.name) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", MIGRATION_LOCK_KEY) \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..4e99d913 --- /dev/null +++ b/src/main.py @@ -0,0 +1,298 @@ +import asyncio +import logging +import re +import sys +import uuid +from contextlib import asynccontextmanager + +import asyncpg +import signal +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from pydantic import BaseModel, ConfigDict +from starlette.exceptions import HTTPException + +from . import config +from .auth import ( + BASIC_CHALLENGE, + current_user, + hash_password, + new_token, + parse_basic, + token_hash, + verify_password, +) +from .chats import router as chats_router +from .common import iso, make_error_response, user_dict +from .db import create_pool_with_retry +from .cache import SessionCache + +from botocore.exceptions import BotoCoreError, ClientError +from .attachments import router as attachments_router + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] [req_id=%(request_id)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) + + +class RequestIdFilter(logging.Filter): + def filter(self, record): + if not hasattr(record, "request_id"): + record.request_id = "-" + return True + + +logger = logging.getLogger("messenger") +logger.addFilter(RequestIdFilter()) + +is_shutting_down = False +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") + + +def _install_drain_handler(app: FastAPI) -> None: + """SIGTERM: сначала /health/ready -> 503, и только через DRAIN_SECONDS обычное завершение uvicorn.""" + loop = asyncio.get_running_loop() + original = signal.getsignal(signal.SIGTERM) + if not callable(original): + return + + def on_sigterm(signum, frame): + if app.state.draining: + return + app.state.draining = True + logger.info("SIGTERM received: draining for %s s", config.DRAIN_SECONDS) + loop.call_soon_threadsafe( + loop.call_later, config.DRAIN_SECONDS, original, signum, frame + ) + + try: + signal.signal(signal.SIGTERM, on_sigterm) + except ValueError: # не главный поток (например, в тестах) + pass + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.draining = False + app.state.pool = await create_pool_with_retry() + app.state.cache = SessionCache(config.CACHE_URL) + _install_drain_handler(app) + yield + global is_shutting_down + is_shutting_down = True + await app.state.cache.close() + await app.state.pool.close() + logger.info("Application shutdown complete") + + +app = FastAPI(title="MTUSI Messenger - Lab 2", lifespan=lifespan) +app.include_router(chats_router, prefix="/api/v1") +app.include_router(attachments_router, prefix="/api/v1") + + +@app.middleware("http") +async def middleware_pipeline(request: Request, call_next): + req_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = req_id + if is_shutting_down and request.url.path != "/health/live": + return make_error_response( + 503, "unavailable", "Server is shutting down", req_id + ) + logger.info( + f"Incoming request {request.method} {request.url.path}", + extra={"request_id": req_id}, + ) + response: Response = await call_next(request) + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = config.INSTANCE_ID + response.headers["X-App-Version"] = config.APP_VERSION + logger.info( + f"Completed {request.method} {request.url.path} with status {response.status_code}", + extra={"request_id": req_id}, + ) + return response + + +def _req_id(request: Request) -> str: + return getattr(request.state, "request_id", str(uuid.uuid4())) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return make_error_response( + 400, "invalid_request", "Validation error", _req_id(request) + ) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + code = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + }.get( + exc.status_code, + "internal_error" if exc.status_code >= 500 else "invalid_request", + ) + return make_error_response( + exc.status_code, code, "Request failed", _req_id(request), exc.headers + ) + + +async def dependency_unavailable_handler(request: Request, exc: Exception): + """БД недоступна или не ответила в срок: ограниченная 503, без деталей и без доверия анониму.""" + logger.error( + "Dependency failure: %s", + type(exc).__name__, + extra={"request_id": _req_id(request)}, + ) + return make_error_response( + 503, "unavailable", "Service temporarily unavailable", _req_id(request) + ) + + +for _exc in ( + asyncpg.PostgresError, + asyncpg.InterfaceError, + OSError, + asyncio.TimeoutError, + BotoCoreError, + ClientError, +): + app.add_exception_handler(_exc, dependency_unavailable_handler) + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + logger.error( + "Unhandled exception: %s", + type(exc).__name__, + extra={"request_id": _req_id(request)}, + ) + return make_error_response( + 500, "internal_error", "Internal server error", _req_id(request) + ) + + +class StrictRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUserRequest(StrictRequest): + username: str + display_name: str + password: str + + +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def health_ready(request: Request): + # Критичная зависимость для готовности - PostgreSQL. При SIGTERM реплика сразу «не готова». + if request.app.state.draining: + return make_error_response(503, "unavailable", "Draining", _req_id(request)) + await request.app.state.pool.fetchval("SELECT 1", timeout=2) + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def register_user(req: CreateUserRequest, request: Request): + req_id = _req_id(request) + if not USERNAME_RE.fullmatch(req.username): + return make_error_response( + 400, "invalid_request", "Invalid username format", req_id + ) + if ( + not (1 <= len(req.display_name) <= 100) + or not req.display_name.strip() + or "\x00" in req.display_name + ): + return make_error_response( + 400, "invalid_request", "Invalid display_name", req_id + ) + if not (12 <= len(req.password) <= 128): + return make_error_response( + 400, "invalid_request", "Invalid password length", req_id + ) + + pw_hash = await hash_password(req.password) + + # Используем ON CONFLICT DO NOTHING, чтобы избежать долгого ожидания блокировки + # при одновременной регистрации одного и того же username. + row = await request.app.state.pool.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, $3, $4, clock_timestamp()) " + "ON CONFLICT (username) DO NOTHING " + "RETURNING id, username, display_name, created_at", + uuid.uuid4(), + req.username, + req.display_name, + pw_hash, + ) + + if row is None: + # Если row is None, значит произошел конфликт (пользователь уже существует) + return make_error_response(409, "conflict", "Username already exists", req_id) + + return user_dict(row) + + +@app.get("/api/v1/users/me") +async def get_current_user(user=Depends(current_user)): + return user_dict(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + req_id = _req_id(request) + creds = parse_basic(request.headers.get("Authorization")) + if creds is None: + return make_error_response( + 401, "unauthorized", "Unauthorized", req_id, BASIC_CHALLENGE + ) + username, password = creds + pool = request.app.state.pool + row = await pool.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + ok = await verify_password(row["password_hash"] if row else None, password) + if row is None or not ok: + return make_error_response( + 401, "unauthorized", "Unauthorized", req_id, BASIC_CHALLENGE + ) + + token = new_token() + expires_at = await pool.fetchval( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at) " + "VALUES ($1, $2, now(), now() + make_interval(secs => $3)) RETURNING expires_at", + token_hash(token), + row["id"], + float(config.SESSION_TTL_SECONDS), + ) + return { + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires_at), + "user": user_dict(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request, user=Depends(current_user)): + await request.app.state.pool.execute( + "UPDATE sessions SET revoked_at = now() WHERE token_hash = $1", + request.state.token_hash, + ) + await request.app.state.cache.delete(request.state.token_hash) + return Response(status_code=204) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..52541c51 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +import asyncio +import logging +import sys + +from .db import create_pool_with_retry, run_migrations + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + +async def main() -> None: + pool = await create_pool_with_retry() + try: + await run_migrations(pool) + finally: + await pool.close() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/src/reconciler.py b/src/reconciler.py new file mode 100644 index 00000000..fe470a93 --- /dev/null +++ b/src/reconciler.py @@ -0,0 +1,160 @@ +"""Сверка PostgreSQL и S3 (трек 2). По умолчанию DRY-RUN: ничего не меняет. Изменения только с --apply. + +orphan : объект в S3 без записи в БД и старше grace -> удаляется (перед удалением повторная проверка в БД). +missing : запись в БД (старше grace), а объекта в S3 нет (подтверждено HEAD) -> storage_state='missing'. + Запись НЕ удаляется: на неё могут ссылаться сообщения. +restore : запись 'missing', объект снова есть, размер и SHA-256 совпали -> storage_state='ok'. +Свежие объекты и записи (моложе grace) не трогаются: загрузка пишет S3 -> БД, между ними есть окно. +""" +import argparse +import asyncio +import hashlib +import json +import logging +import resource +import sys +import time +from datetime import timedelta + +from botocore.exceptions import ClientError + +from . import config, storage +from .db import create_pool_with_retry + +logging.basicConfig(level=logging.INFO, stream=sys.stderr, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger("reconciler") + +LOCK_KEY = 727_002 +PREFIX = "attachments/" + + +def list_s3(): + """Все объекты bucket с префиксом: {key: (LastModified, Size)} и число страниц листинга.""" + objects, pages = {}, 0 + paginator = storage.client().get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=PREFIX): + pages += 1 + for obj in page.get("Contents", []): + objects[obj["Key"]] = (obj["LastModified"], obj["Size"]) + return objects, pages + + +def head_size(key): + try: + return storage.client().head_object(Bucket=config.S3_BUCKET, Key=key)["ContentLength"] + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"): + return None + raise + + +def sha256_of(key): + body, digest = storage.get_object(key), hashlib.sha256() + try: + for chunk in iter(lambda: body.read(64 * 1024), b""): + digest.update(chunk) + finally: + body.close() + return digest.hexdigest() + + +async def reconcile(pool, apply, grace, max_actions): + started = time.monotonic() + result = {"dry_run": not apply, "grace_seconds": grace} + async with pool.acquire() as conn: + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_KEY): + logger.warning("Another reconciler is running, exiting") + result["error"] = "locked" + return result, 2 + try: + now = await conn.fetchval("SELECT now()") + cutoff = now - timedelta(seconds=grace) + + t0 = time.monotonic() + objects, pages = await asyncio.to_thread(list_s3) + result["s3_list_s"] = round(time.monotonic() - t0, 3) + result["s3_objects"], result["s3_list_pages"] = len(objects), pages + + t0 = time.monotonic() + rows = await conn.fetch( + "SELECT id, object_key, size_bytes, sha256, storage_state, created_at FROM attachments", timeout=300) + result["db_query_s"] = round(time.monotonic() - t0, 3) + result["db_rows"] = len(rows) + by_key = {r["object_key"]: r for r in rows} + + orphans = [k for k, (lm, _) in objects.items() if k not in by_key and lm < cutoff] + result["orphans_fresh_skipped"] = sum(1 for k, (lm, _) in objects.items() if k not in by_key and lm >= cutoff) + missing = [r for r in rows if r["storage_state"] == "ok" and r["object_key"] not in objects + and r["created_at"] < cutoff] + result["missing_fresh_skipped"] = sum(1 for r in rows if r["storage_state"] == "ok" + and r["object_key"] not in objects and r["created_at"] >= cutoff) + restore = [r for r in rows if r["storage_state"] == "missing" and r["object_key"] in objects + and objects[r["object_key"]][1] == r["size_bytes"]] + result["size_mismatch"] = sum(1 for r in rows if r["storage_state"] == "ok" + and r["object_key"] in objects and objects[r["object_key"]][1] != r["size_bytes"]) + result.update(orphans_found=len(orphans), missing_found=len(missing), restore_found=len(restore), + orphans_deleted=0, missing_marked=0, restored=0, race_skipped=0) + for k in orphans: + logger.info("%s orphan: %s", "WOULD DELETE" if not apply else "found", k) + for r in missing: + logger.warning("%s missing object for attachment %s (%s)", "WOULD MARK" if not apply else "found", + r["id"], r["object_key"]) + if result["size_mismatch"]: + logger.warning("%d attachments have size different from S3 object (report only)", result["size_mismatch"]) + + actions = len(orphans) + len(missing) + len(restore) + if apply and actions > max_actions: + logger.error("Refusing to apply %d actions (limit %d). Check S3/DB settings or raise --max-actions", + actions, max_actions) + result["error"] = "too_many_actions" + return result, 3 + + if apply: + for key in orphans: # повторная проверка прямо перед удалением: закрывает гонку со свежим commit + if await conn.fetchval("SELECT 1 FROM attachments WHERE object_key = $1", key): + result["race_skipped"] += 1 + continue + await asyncio.to_thread(storage.delete_object, key) + result["orphans_deleted"] += 1 + for r in missing: # подтверждаем HEAD, чтобы не поверить неполному листингу + if await asyncio.to_thread(head_size, r["object_key"]) is not None: + result["race_skipped"] += 1 + continue + await conn.execute("UPDATE attachments SET storage_state = 'missing', storage_checked_at = now() " + "WHERE id = $1 AND storage_state = 'ok'", r["id"]) + result["missing_marked"] += 1 + for r in restore: + if await asyncio.to_thread(sha256_of, r["object_key"]) == r["sha256"]: + await conn.execute("UPDATE attachments SET storage_state = 'ok', storage_checked_at = now() " + "WHERE id = $1 AND storage_state = 'missing'", r["id"]) + result["restored"] += 1 + return result, 0 + finally: + result["elapsed_s"] = round(time.monotonic() - started, 3) + result["peak_rss_mib"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1) + await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_KEY) + + +async def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--apply", action="store_true", help="выполнить изменения (по умолчанию dry-run)") + parser.add_argument("--grace", type=int, default=config.RECONCILE_GRACE_SECONDS) + parser.add_argument("--max-actions", type=int, default=config.RECONCILE_MAX_ACTIONS) + parser.add_argument("--interval", type=int, default=0, help="повторять каждые N секунд (0 - один запуск)") + args = parser.parse_args() + pool = await create_pool_with_retry() + code = 0 + try: + while True: + result, code = await reconcile(pool, args.apply, args.grace, args.max_actions) + print(json.dumps(result, sort_keys=True), flush=True) + if not args.interval: + break + await asyncio.sleep(args.interval) + finally: + await pool.close() + sys.exit(code) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..468958f0 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,63 @@ +# Клиент S3 (SeaweedFS). Синхронный boto3: вызывается из потоков через asyncio.to_thread +import logging +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from . import config + +logger = logging.getLogger("messenger") + +_client = None +_lock = threading.Lock() + + +class ObjectMissing(Exception): + """Объект есть в метаданных, но отсутствует в S3 (рассинхронизация)""" + + +def client(): + global _client + if _client is None: + with _lock: + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=config.S3_CONNECT_TIMEOUT, + read_timeout=config.S3_READ_TIMEOUT, + retries={"total_max_attempts": 2, "mode": "standard"}, + max_pool_connections=20, + ), + ) + return _client + + +def put_object(key: str, fileobj, size: int, content_type: str) -> None: + """Потоковая запись из файлового объекта; в память целиком содержимое не загружается.""" + fileobj.seek(0) + client().put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, + ContentLength=size, ContentType=content_type) + + +def get_object(key: str): + """Возвращает StreamingBody; чтение чанками выполняет вызывающий.""" + try: + response = client().get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"): + raise ObjectMissing(key) from exc + raise + return response["Body"] + + +def delete_object(key: str) -> None: + client().delete_object(Bucket=config.S3_BUCKET, Key=key) \ No newline at end of file diff --git a/student_tests/check_s3_private.py b/student_tests/check_s3_private.py new file mode 100644 index 00000000..5ad520ee --- /dev/null +++ b/student_tests/check_s3_private.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Проверка приватности S3 и ограниченных прав API-ключа. Нужен запущенный стенд. Ничего не удаляет. +Запуск: python3 student_tests/check_s3_private.py | tee evidence/s3-private-check.txt""" +import json +import subprocess +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FAILED = [] + + +def compose(*args, check=True): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, + capture_output=True, text=True, check=check) + + +def report(ok, text): + print(("PASS " if ok else "FAIL ") + text) + if not ok: + FAILED.append(text) + + +def api_py(code): + return compose("exec", "-T", "api", "python", "-c", code).stdout.strip() + + +key = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", + "SELECT object_key FROM attachments ORDER BY created_at DESC LIMIT 1").stdout.strip() +if not key: + sys.exit("В БД нет вложений: сначала выполните make test") +bucket = api_py("from src import config; print(config.S3_BUCKET)") +print(f"# bucket={bucket}, проверяется объект {key}") + +# 1. Анонимный доступ к S3 напрямую, минуя API (из контейнера api, где S3 достижим по сети) +anon = api_py(f""" +import urllib.request, urllib.error, json +def code(u): + try: return urllib.request.urlopen(u, timeout=5).status + except urllib.error.HTTPError as e: return e.code +print(json.dumps([code("http://s3:9000/{bucket}/{key}"), code("http://s3:9000/{bucket}/"), code("http://s3:9000/")])) +""") +get_obj, list_bucket, list_all = json.loads(anon) +report(get_obj in (401, 403), f"анонимный GET объекта напрямую из S3 -> HTTP {get_obj}") +report(list_bucket in (401, 403), f"анонимный список bucket -> HTTP {list_bucket}") +report(list_all in (401, 403), f"анонимный список всех bucket -> HTTP {list_all}") + +# 2. Права API-ключа ограничены: чужой bucket и создание bucket запрещены +rights = json.loads(api_py(f""" +import json, uuid +from botocore.exceptions import ClientError +from src import storage +c = storage.client(); res = {{}} +def t(name, f): + try: f(); res[name] = "ALLOWED" + except ClientError as e: res[name] = e.response["Error"]["Code"] +other = "other-" + uuid.uuid4().hex[:10] +t("create_bucket", lambda: c.create_bucket(Bucket=other)) +t("put_to_other_bucket", lambda: c.put_object(Bucket="not-my-bucket", Key="x", Body=b"x")) +t("get_from_other_bucket", lambda: c.get_object(Bucket="not-my-bucket", Key="x")) +t("put_object_own_bucket", lambda: c.put_object(Bucket="{bucket}", Key="attachments/_rights_probe", Body=b"x")) +t("delete_probe_own_bucket", lambda: c.delete_object(Bucket="{bucket}", Key="attachments/_rights_probe")) +if res["create_bucket"] == "ALLOWED": c.delete_bucket(Bucket=other) +print(json.dumps(res)) +""")) +print("# ответы S3 для API-ключа:", rights) +report(rights["create_bucket"] != "ALLOWED", "API-ключ не может создавать bucket") +report(rights["put_to_other_bucket"] != "ALLOWED", "API-ключ не может писать в чужой bucket") +report(rights["get_from_other_bucket"] not in ("ALLOWED",) and rights["get_from_other_bucket"] != "NoSuchBucket" + or rights["get_from_other_bucket"] == "NoSuchBucket", "API-ключ не читает чужой bucket (NoSuchBucket/AccessDenied)") +report(rights["put_object_own_bucket"] == "ALLOWED" and rights["delete_probe_own_bucket"] == "ALLOWED", + "в своём bucket API-ключ работает (put/delete)") + +# 3. Админские ключи не попали в API +secrets = api_py("import os; print(' '.join(sorted(os.listdir('/run/secrets'))))") +print("# /run/secrets в api:", secrets) +report("admin" not in secrets, "в контейнере api нет админских S3-секретов") + +# 4. Порты S3 не опубликованы на хост +cid = compose("ps", "-q", "s3").stdout.strip() +published = subprocess.run(["docker", "port", cid], capture_output=True, text=True).stdout.strip() +report(published == "", f"S3 не публикует порты на хост (docker port: '{published or 'пусто'}')") + +# 5. Из контейнеров вне сети storage S3 недостижим +for svc in ("lb", "postgres", "valkey"): + for port in (9000, 8888, 9333): + r = compose("exec", "-T", svc, "wget", "-T", "2", "-O", "/dev/null", f"http://s3:{port}/", check=False) + err = (r.stderr + r.stdout).lower() + reached = r.returncode == 0 or "server returned error" in err or "http/1" in err + report(not reached, f"{svc} -> s3:{port} недостижим") + +# 6. Информация: что доступно из api внутри сети storage +ports = api_py(""" +import socket, json +res = {} +for p in (9000, 8333, 8888, 9333, 8080): + s = socket.socket(); s.settimeout(1) + res[p] = s.connect_ex(("s3", p)) == 0 +print(json.dumps(res)) +""") +print("INFO порты s3, открытые из api (внутри изолированной сети storage):", ports) + +print("\nИТОГ:", "FAIL" if FAILED else "PASS") +sys.exit(1 if FAILED else 0) \ No newline at end of file diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 00000000..b649bc94 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,34 @@ +import base64 +import json +import urllib.error +import urllib.request +import uuid + + +def call(base, method, path, body=None, headers=None, timeout=3.0): + """Возвращает (status, headers с ключами в нижнем регистре, bytes). Сетевые ошибки поднимаются наружу.""" + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if data else {} + h.update(headers or {}) + req = urllib.request.Request(base + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, {k.lower(): v for k, v in r.headers.items()}, r.read() + except urllib.error.HTTPError as e: + return e.code, {k.lower(): v for k, v in e.headers.items()}, e.read() + + +def setup(base): + """Синтетический пользователь, сессия и чат из одного участника. Возвращает (auth_headers, chat_id).""" + name = "load_" + uuid.uuid4().hex[:12] + password = "load-password-" + uuid.uuid4().hex[:8] + s, _, b = call(base, "POST", "/api/v1/users", {"username": name, "display_name": "Load", "password": password}) + assert s == 201, f"register failed: {s}" + uid = json.loads(b)["id"] + cred = base64.b64encode(f"{name}:{password}".encode()).decode() + s, _, b = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, f"login failed: {s}" + auth = {"Authorization": "Bearer " + json.loads(b)["token"]} + s, _, b = call(base, "POST", "/api/v1/chats", {"title": "load", "member_ids": [uid]}, auth) + assert s == 201, f"chat failed: {s}" + return auth, json.loads(b)["id"] \ No newline at end of file diff --git a/student_tests/loadgen.py b/student_tests/loadgen.py new file mode 100644 index 00000000..aca9941d --- /dev/null +++ b/student_tests/loadgen.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики). +Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают.""" +import argparse +import json +import os +import subprocess +import threading +import time +import uuid + +from common import call, setup + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--duration", type=float, default=40) + ap.add_argument("--concurrency", type=int, default=8) + ap.add_argument("--action-script") + ap.add_argument("--action-at", type=float, default=10) + ap.add_argument("--target-index", type=int, default=0) + ap.add_argument("--output") + a = ap.parse_args() + + auth, chat = setup(a.base_url) + path = f"/api/v1/chats/{chat}/messages" + ids = set() + for _ in range(40): # какие реплики видны через LB + _, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth) + ids.add(h.get("x-instance-id")) + ids = sorted(i for i in ids if i) + + records, lock = [], threading.Lock() # (t, status, latency, instance) + t_start = time.monotonic() + t_end = t_start + a.duration + + def worker(): + while time.monotonic() < t_end: + t0 = time.monotonic() + try: + s, h, _ = call(a.base_url, "POST", path, {"text": "load"}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3) + inst, ver = h.get("x-instance-id"), h.get("x-app-version") + except Exception: + s, inst, ver = 0, None, None + with lock: + records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver)) + + threads = [threading.Thread(target=worker) for _ in range(a.concurrency)] + for t in threads: + t.start() + + action = {} + if a.action_script: + time.sleep(a.action_at) + target = ids[a.target_index % len(ids)] + t0 = time.monotonic() - t_start + rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target}, + capture_output=True, timeout=300).returncode + action = {"target": target, "started_at": round(t0, 2), + "finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc} + for t in threads: + t.join() + + ok = [r for r in records if r[1] == 201] + bad = [r for r in records if r[1] != 201] + lat = sorted(r[2] for r in ok) + pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None + summary = { + "duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids, + "requests": len(records), "ok": len(ok), "errors": len(bad), + "error_rate": round(len(bad) / max(1, len(records)), 4), + "rps_ok": round(len(ok) / a.duration, 1), + "latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)}, + "error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}}, + "action": action, + } + if action: + errs_after = [r[0] for r in bad if r[0] >= action["started_at"]] + times = sorted(r[0] for r in ok if r[0] >= action["started_at"]) + gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)] + summary["errors_after_action"] = len(errs_after) + summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0 + summary["max_stall_s"] = round(max(gaps), 2) if gaps else None + summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]}) + + vers = {} + for r in ok: + if r[4]: + v = vers.setdefault(r[4], [r[0], r[0]]) + v[0], v[1] = min(v[0], r[0]), max(v[1], r[0]) + summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()} + + timeline = {} + for t, s, *_ in records: + b = timeline.setdefault(int(t), {"ok": 0, "err": 0}) + b["ok" if s == 201 else "err"] += 1 + print(json.dumps(summary, ensure_ascii=False, indent=2)) + if a.output: + with open(a.output, "w") as f: + json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_attachments_level4.py b/student_tests/test_attachments_level4.py new file mode 100644 index 00000000..61099c14 --- /dev/null +++ b/student_tests/test_attachments_level4.py @@ -0,0 +1,281 @@ +"""Тесты вложений, уровень 4. Нужен запущенный стенд. +Запуск: python3 -m unittest student_tests.test_attachments_level4 -v +Тесты отказов сами останавливают/запускают s3 и временно ломают INSERT в БД; volumes не удаляются.""" +import base64 +import hashlib +import http.client +import json +import os +import socket +import struct +import subprocess +import time +import unittest +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote, urlparse + +BASE = os.environ.get("BASE_URL", "http://localhost:8080") +URL = urlparse(BASE) +ROOT = Path(__file__).resolve().parents[1] +PASSWORD = "correct-horse-battery" +MAX_BYTES = 10 * 1024 * 1024 +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +SIG = b"\x89PNG\r\n\x1a\n" + +try: + from PIL import Image + import io +except ImportError: # JPEG-тесты будут пропущены + Image = None + + +def http_call(method, path, body=None, headers=None, raw=None, timeout=60): + """Возвращает (status, headers, bytes). Если сервер ответил раньше, чем клиент дослал тело, читаем ответ.""" + h = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + if data is not None and not hasattr(data, "__next__"): + h["Content-Length"] = str(len(data)) + conn = http.client.HTTPConnection(URL.hostname, URL.port, timeout=timeout) + try: + try: + conn.request(method, path, body=data, headers=h, encode_chunked=hasattr(data, "__next__")) + except (BrokenPipeError, ConnectionResetError): + pass + resp = conn.getresponse() + return resp.status, {k.lower(): v for k, v in resp.getheaders()}, resp.read() + finally: + conn.close() + + +def jcall(method, path, body=None, headers=None): + status, _, raw = http_call(method, path, body, headers) + return status, (json.loads(raw) if raw else None) + + +def compose(*args): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, + check=True, capture_output=True, text=True).stdout + + +def psql(sql): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", sql).strip() + + +def reconcile_summary(): + out = compose("run", "--rm", "--no-deps", "-T", "reconciler", "python", "-m", "src.reconciler") + return json.loads(out.strip().splitlines()[-1]) + + +def wait_ready(timeout=90): + end = time.time() + timeout + while time.time() < end: + try: + if http_call("GET", "/health/ready")[0] == 200: + return + except OSError: + pass + time.sleep(1) + raise AssertionError("API not ready") + + +def new_user(): + name = "u" + uuid.uuid4().hex[:12] + st, user = jcall("POST", "/api/v1/users", {"username": name, "display_name": "T", "password": PASSWORD}) + assert st == 201, st + basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode() + st, sess = jcall("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + assert st == 201, st + return user, {"Authorization": "Bearer " + sess["token"]} + + +def new_chat(auth, member_ids): + st, chat = jcall("POST", "/api/v1/chats", {"title": "att", "member_ids": member_ids}, auth) + assert st == 201, st + return chat["id"] + + +def upload(auth, chat, data, ctype="image/png", filename="a.png"): + return http_call("POST", f"/api/v1/chats/{chat}/attachments?filename={quote(filename, safe='')}", + headers={**auth, "Content-Type": ctype}, raw=data) + + +def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +def png_1bit(w, h): + rows = b"".join(b"\x00" + b"\x00" * ((w + 7) // 8) for _ in range(h)) + return SIG + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 1, 0, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"") + + +def jpeg_bytes(): + buf = io.BytesIO() + Image.new("RGB", (32, 32), (200, 30, 30)).save(buf, "JPEG") + return buf.getvalue() + + +class AttachmentsLevel4(unittest.TestCase): + @classmethod + def setUpClass(cls): + wait_ready() + cls.alice, cls.a = new_user() + cls.bob, cls.b = new_user() + cls.eve, cls.e = new_user() + cls.chat = new_chat(cls.a, [cls.alice["id"], cls.bob["id"]]) # alice + bob + cls.chat_b = new_chat(cls.a, [cls.alice["id"]]) # другой чат alice + cls.chat_e = new_chat(cls.e, [cls.eve["id"]]) # чат eve + cls.att = cls.ok(cls.a, cls.chat) + cls.att_b = cls.ok(cls.a, cls.chat_b) + cls.att_e = cls.ok(cls.e, cls.chat_e) + + @classmethod + def ok(cls, auth, chat): + st, _, raw = upload(auth, chat, SAMPLE) + assert st == 201, st + return json.loads(raw) + + def send(self, auth, chat, ids): + return jcall("POST", f"/api/v1/chats/{chat}/messages", {"text": "", "attachment_ids": ids}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}) + + def assertStatus(self, got, want): + self.assertEqual(got, want) + self.assertLess(got, 500, "пользовательская ошибка не должна быть 5xx") + + # --- базовый сценарий --- + def test_roundtrip_sha256_and_second_member(self): + self.assertEqual(self.att["sha256"], hashlib.sha256(SAMPLE).hexdigest()) + self.assertEqual(self.att["size_bytes"], len(SAMPLE)) + st, msg = self.send(self.a, self.chat, [self.att["id"]]) + self.assertEqual(st, 201) + st, page = jcall("GET", f"/api/v1/chats/{self.chat}/messages", headers=self.b) + self.assertEqual(st, 200) + self.assertIn(self.att["id"], [i for m in page["items"] for i in m["attachment_ids"]]) + st, h, raw = http_call("GET", f"/api/v1/attachments/{self.att['id']}/content", headers=self.b) + self.assertEqual((st, h["content-type"]), (200, "image/png")) + self.assertEqual(hashlib.sha256(raw).hexdigest(), self.att["sha256"]) + + # --- размер и пиксели --- + def test_too_large_content_length(self): + st, _, _ = upload(self.a, self.chat, SIG + os.urandom(MAX_BYTES + 1)) + self.assertStatus(st, 413) + + def test_too_large_chunked_without_content_length(self): + def gen(): + yield SIG + for _ in range(11): + yield os.urandom(1024 * 1024) + st, _, _ = http_call("POST", f"/api/v1/chats/{self.chat}/attachments?filename=a.png", + headers={**self.a, "Content-Type": "image/png", "Transfer-Encoding": "chunked"}, raw=gen()) + self.assertStatus(st, 413) + + def test_pixel_limit(self): + st, _, _ = upload(self.a, self.chat, png_1bit(8000, 8000)) # 64 млн пикселей, файл несколько КиБ + self.assertStatus(st, 413) + st, _, _ = upload(self.a, self.chat, png_1bit(6000, 6000)) # 36 млн: допустимо + self.assertEqual(st, 201) + + # --- повреждённые файлы и подмена MIME --- + def test_corrupted_png(self): + flipped = bytearray(SAMPLE) + flipped[len(flipped) // 2] ^= 0xFF + cases = {"truncated": SAMPLE[:-20], "bad_crc": bytes(flipped), "random_after_signature": SIG + os.urandom(2000), + "empty": b"", "only_signature": SIG} + for name, data in cases.items(): + with self.subTest(name): + self.assertStatus(upload(self.a, self.chat, data)[0], 415) + + @unittest.skipIf(Image is None, "pip3 install pillow") + def test_corrupted_jpeg_and_valid_jpeg(self): + good = jpeg_bytes() + st, _, raw = upload(self.a, self.chat, good, "image/jpeg", "a.jpg") + self.assertEqual(st, 201) + self.assertEqual(json.loads(raw)["content_type"], "image/jpeg") + self.assertStatus(upload(self.a, self.chat, good[:-30], "image/jpeg", "a.jpg")[0], 415) + + def test_mime_spoofing(self): + cases = [("png as jpeg", SAMPLE, "image/jpeg"), ("gif as png", b"GIF89a" + b"0" * 200, "image/png"), + ("text as png", b"hello world" * 50, "image/png"), ("svg as png", b"", "image/png"), + ("png as text/plain", SAMPLE, "text/plain"), ("png as octet-stream", SAMPLE, "application/octet-stream")] + for name, data, ctype in cases: + with self.subTest(name): + self.assertStatus(upload(self.a, self.chat, data, ctype)[0], 415) + if Image is not None: + self.assertStatus(upload(self.a, self.chat, jpeg_bytes(), "image/png")[0], 415) + + def test_dangerous_filenames(self): + for name in ["../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\n.png", " ", "x" * 256]: + with self.subTest(repr(name)): + self.assertStatus(upload(self.a, self.chat, SAMPLE, filename=name)[0], 400) + st, _, raw = upload(self.a, self.chat, SAMPLE, filename="фото №1 (копия).png") + self.assertEqual(st, 201) # имя только для отображения; ключ S3 назначает сервер + self.assertEqual(json.loads(raw)["filename"], "фото №1 (копия).png") + + # --- доступ и привязка --- + def test_foreign_and_unknown_attachment_ids(self): + self.assertStatus(self.send(self.a, self.chat, [self.att_e["id"]])[0], 404) # вложение чужого чата и чужого пользователя + self.assertStatus(self.send(self.a, self.chat, [str(uuid.uuid4())])[0], 404) + self.assertStatus(self.send(self.a, self.chat, ["not-a-uuid"])[0], 400) + self.assertStatus(self.send(self.a, self.chat, [self.att["id"], self.att["id"]])[0], 400) + + def test_attachment_from_another_chat_of_same_user(self): + self.assertStatus(self.send(self.a, self.chat, [self.att_b["id"]])[0], 404) # chat_b, а не chat + self.assertStatus(self.send(self.a, self.chat_b, [self.att_b["id"]])[0], 201) + + def test_outsider_has_no_access(self): + for path in (f"/api/v1/attachments/{self.att['id']}", f"/api/v1/attachments/{self.att['id']}/content"): + self.assertStatus(http_call("GET", path, headers=self.e)[0], 404) + self.assertStatus(http_call("GET", path)[0], 401) + self.assertStatus(upload(self.e, self.chat, SAMPLE)[0], 404) + self.assertStatus(self.send(self.e, self.chat, [self.att["id"]])[0], 404) + + # --- обрыв загрузки --- + def test_aborted_upload_leaves_no_metadata(self): + before = psql(f"SELECT count(*) FROM attachments WHERE chat_id = '{self.chat_b}'") + sock = socket.create_connection((URL.hostname, URL.port), timeout=10) + head = (f"POST /api/v1/chats/{self.chat_b}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: {self.a['Authorization']}\r\nContent-Type: image/png\r\n" + f"Content-Length: {5 * 1024 * 1024}\r\n\r\n").encode() + sock.sendall(head + SIG + os.urandom(1024 * 1024)) + sock.close() # обрыв на середине тела + time.sleep(3) + self.assertEqual(psql(f"SELECT count(*) FROM attachments WHERE chat_id = '{self.chat_b}'"), before) + + # --- отказы зависимостей --- + def test_s3_down_gives_503_and_recovers(self): + compose("stop", "s3") + try: + t = time.time() + self.assertEqual(upload(self.a, self.chat, SAMPLE)[0], 503) + self.assertLess(time.time() - t, 20, "ответ при недоступном S3 должен быть ограничен по времени") + self.assertEqual(http_call("GET", f"/api/v1/attachments/{self.att['id']}/content", headers=self.a)[0], 503) + self.assertEqual(http_call("GET", f"/api/v1/attachments/{self.att['id']}", headers=self.a)[0], 200) # метаданные из БД + finally: + compose("start", "s3") + end = time.time() + 90 + while time.time() < end: + st, _, raw = http_call("GET", f"/api/v1/attachments/{self.att['id']}/content", headers=self.b) + if st == 200: + break + time.sleep(2) + self.assertEqual(st, 200) + self.assertEqual(hashlib.sha256(raw).hexdigest(), self.att["sha256"]) + + def test_db_insert_failure_after_s3_put_is_compensated(self): + before = reconcile_summary()["s3_objects"] + psql("ALTER TABLE attachments ADD CONSTRAINT zz_fail CHECK (false) NOT VALID") # ломает только новые INSERT + try: + st, _, _ = upload(self.a, self.chat, SAMPLE) + self.assertIn(st, (500, 503)) + finally: + psql("ALTER TABLE attachments DROP CONSTRAINT zz_fail") + self.assertEqual(reconcile_summary()["s3_objects"], before, "объект остался в S3: компенсация не сработала") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/student_tests/test_idempotency_parallel.py b/student_tests/test_idempotency_parallel.py new file mode 100644 index 00000000..56706014 --- /dev/null +++ b/student_tests/test_idempotency_parallel.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""30 параллельных POST с одним Idempotency-Key через LB: ровно одно сообщение, везде 201 и один id.""" +import json +import sys +import threading +import uuid + +from common import call, setup + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +THREADS, ROUNDS = 30, 5 + +auth, chat = setup(BASE) +path = f"/api/v1/chats/{chat}/messages" +replicas = set() + +for rnd in range(ROUNDS): + key = uuid.uuid4().hex + barrier = threading.Barrier(THREADS) + results = [None] * THREADS + + def work(i, rnd=rnd, key=key, barrier=barrier, results=results): + barrier.wait() + s, h, b = call(BASE, "POST", path, {"text": f"parallel {rnd}"}, {**auth, "Idempotency-Key": key}, timeout=10) + results[i] = (s, h.get("x-instance-id"), json.loads(b) if s == 201 else None) + + ts = [threading.Thread(target=work, args=(i,)) for i in range(THREADS)] + [t.start() for t in ts] + [t.join() for t in ts] + codes = {r[0] for r in results} + msg_ids = {r[2]["id"] for r in results if r[0] == 201} + replicas |= {r[1] for r in results} + assert codes == {201}, f"round {rnd}: codes {codes}" + assert len(msg_ids) == 1, f"round {rnd}: {len(msg_ids)} different messages" + s, _, _ = call(BASE, "POST", path, {"text": "other body"}, {**auth, "Idempotency-Key": key}) + assert s == 409, f"round {rnd}: other body gave {s}" + +s, _, b = call(BASE, "GET", path + "?limit=100", headers=auth) +count = len(json.loads(b)["items"]) +assert count == ROUNDS, f"history has {count} messages, expected {ROUNDS}" +print(json.dumps({"rounds": ROUNDS, "parallel_requests_per_round": THREADS, + "messages_in_history": count, "replicas_that_answered": sorted(r for r in replicas if r)})) +print("PASS: same key in parallel -> exactly one message; different body -> 409") \ No newline at end of file diff --git a/student_tests/test_lab2.py b/student_tests/test_lab2.py new file mode 100644 index 00000000..37dd02fc --- /dev/null +++ b/student_tests/test_lab2.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 + +import concurrent.futures +import json +import os +import sys +import uuid +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime +import base64 + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") +PASSWORD = "correct-horse-battery" + + +def request(method, path, expected, body=None, headers=None, raw_body=None): + url = BASE_URL + path + + request_headers = dict(headers or {}) + + if raw_body is not None: + data = raw_body + elif body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + else: + data = None + + if body is not None or raw_body is not None: + request_headers.setdefault("Content-Type", "application/json") + + req = urllib.request.Request( + url, + data=data, + method=method, + headers=request_headers, + ) + + try: + with urllib.request.urlopen(req, timeout=10) as response: + status = response.status + response_headers = response.headers + response_body = response.read() + + except urllib.error.HTTPError as exc: + status = exc.code + response_headers = exc.headers + response_body = exc.read() + + if status != expected: + raise AssertionError( + f"{method} {path}: expected HTTP {expected}, got {status}. " + f"Body={response_body!r}" + ) + + request_id = response_headers.get("X-Request-Id") + if not request_id: + raise AssertionError( + f"{method} {path}: missing X-Request-Id" + ) + + parsed = None + + if response_body: + content_type = response_headers.get_content_type() + + if content_type == "application/json": + parsed = json.loads(response_body.decode("utf-8")) + + if status >= 400: + codes = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", + 409: "conflict", 413: "payload_too_large", 415: "unsupported_media_type", + 429: "rate_limited", 500: "internal_error", 503: "unavailable", + } + if not isinstance(parsed, dict) or set(parsed) != {"error"}: + raise AssertionError(f"Invalid error response: {parsed!r}") + error = parsed["error"] + if not isinstance(error, dict) or set(error) != {"code", "message", "request_id"}: + raise AssertionError(f"Invalid error fields: {error!r}") + if error["code"] != codes[status] or error["request_id"] != request_id: + raise AssertionError(f"Wrong error code or request ID: {error!r}") + if not isinstance(error["message"], str) or not error["message"].strip(): + raise AssertionError("Error message must be a nonempty string") + + return { + "status": status, + "headers": response_headers, + "json": parsed, + "bytes": response_body, + } + + +def unique_username(prefix): + # username должен соответствовать [a-z0-9_]{3,32} + suffix = uuid.uuid4().hex[:12] + return f"{prefix}_{suffix}" + + +def basic(username, password=PASSWORD): + raw = f"{username}:{password}".encode("utf-8") + return "Basic " + base64.b64encode(raw).decode() + + +def create_user(prefix="usr", login=True): + user = request("POST", "/api/v1/users", 201, body={ + "username": unique_username(prefix), + "display_name": f"Test {prefix}", + "password": PASSWORD, + })["json"] + if login: + session = request("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(user["username"])})["json"] + user["token"] = session["token"] + return user + + +def auth(user): + return {"Authorization": "Bearer " + user["token"]} + + +def create_chat(owner, members, title="Student test chat"): + result = request( + "POST", + "/api/v1/chats", + 201, + headers=auth(owner), + body={ + "title": title, + "member_ids": [u["id"] for u in members], + }, + ) + + return result["json"] + + +def create_message(user, chat, text): + return request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + headers=auth(user), + body={"text": text}, + )["json"] + + +def test_malformed_json(): + for body in (b'{"username": "broken",', b"\xff"): + request("POST", "/api/v1/users", 400, raw_body=body) + + print("PASS malformed JSON and invalid UTF-8 -> 400 with error envelope") + + +def test_duplicate_username(): + username = unique_username("dup") + + body = { + "username": username, + "display_name": "First", + "password": PASSWORD, + } + + request( + "POST", + "/api/v1/users", + 201, + body=body, + ) + + request( + "POST", + "/api/v1/users", + 409, + body={ + "username": username, + "display_name": "Second", + "password": PASSWORD, + }, + ) + + print("PASS duplicate username -> 409") + + +def test_outsider_chat(): + alice = create_user("alice") + bob = create_user("bob") + eve = create_user("eve") + + chat = create_chat( + alice, + [alice, bob], + "Private chat", + ) + + request( + "GET", + f"/api/v1/chats/{chat['id']}", + 404, + headers=auth(eve), + ) + + request( + "GET", + f"/api/v1/chats/{chat['id']}/messages", + 404, + headers=auth(eve), + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 404, + headers=auth(eve), + body={"text": "Not allowed"}, + ) + page = request("GET", "/api/v1/chats", 200, headers=auth(eve))["json"] + if page != {"items": [], "next_cursor": None}: + raise AssertionError("Outsider can see a private chat in the list") + + print("PASS outsider cannot access chat -> 404") + + +def test_empty_and_whitespace_message(): + alice = create_user("emptya") + bob = create_user("emptyb") + + chat = create_chat( + alice, + [alice, bob], + "Empty message test", + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + headers=auth(alice), + body={"text": ""}, + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + headers=auth(alice), + body={"text": " \t\n"}, + ) + + print("PASS empty/whitespace message -> 400") + + +def test_extra_json_field(): + request( + "POST", + "/api/v1/users", + 400, + body={ + "username": unique_username("extra"), + "display_name": "Extra", + "password": PASSWORD, + "admin": True, + }, + ) + + print("PASS extra JSON field -> 400") + + +def test_multi_page_history(): + alice = create_user("pagea") + bob = create_user("pageb") + + chat = create_chat( + alice, + [alice, bob], + "Pagination test", + ) + + expected = [] + + for i in range(7): + message = create_message( + alice, + chat, + f"pagination-message-{i}", + ) + + expected.append(message) + + expected.sort( + key=lambda message: ( + datetime.fromisoformat( + message["created_at"].replace("Z", "+00:00") + ), + uuid.UUID(message["id"]).int, + ) + ) + + expected_ids = [ + message["id"] + for message in expected + ] + + collected = [] + cursor = None + + for _ in range(len(expected) + 1): + path = ( + f"/api/v1/chats/{chat['id']}/messages" + f"?limit=2" + ) + + if cursor is not None: + path += "&cursor=" + urllib.parse.quote( + cursor, + safe="", + ) + + page = request( + "GET", + path, + 200, + headers=auth(bob), + )["json"] + + ids = [item["id"] for item in page["items"]] + collected.extend(ids) + + cursor = page["next_cursor"] + + if cursor is None: + break + else: + raise AssertionError("Pagination did not reach the last page") + + if collected != expected_ids: + raise AssertionError( + "Pagination lost, duplicated or reordered messages.\n" + f"Expected: {expected_ids}\n" + f"Received: {collected}" + ) + + if len(collected) != len(set(collected)): + raise AssertionError( + "Pagination returned duplicate messages" + ) + + print( + f"PASS pagination: {len(collected)} messages " + "read through multiple pages" + ) + + +def test_concurrent_100_messages(): + users = [ + create_user(f"c{i}") + for i in range(10) + ] + + chat = create_chat( + users[0], + users, + "Concurrency test", + ) + + def worker(client_index): + sent = [] + + for message_index in range(10): + text = ( + f"client-{client_index}-" + f"message-{message_index}" + ) + + result = create_message( + users[client_index], + chat, + text, + ) + + sent.append(result) + + return sent + + all_sent = [] + + with concurrent.futures.ThreadPoolExecutor( + max_workers=10 + ) as executor: + futures = [ + executor.submit(worker, i) + for i in range(10) + ] + + for future in concurrent.futures.as_completed( + futures + ): + all_sent.extend(future.result()) + + if len(all_sent) != 100: + raise AssertionError( + f"Expected 100 successful POSTs, got {len(all_sent)}" + ) + + page = request( + "GET", + f"/api/v1/chats/{chat['id']}/messages?limit=100", + 200, + headers=auth(users[0]), + )["json"] + + messages = page["items"] + + if len(messages) != 100: + raise AssertionError( + f"Expected exactly 100 stored messages, " + f"got {len(messages)}" + ) + + ids = [message["id"] for message in messages] + + if len(set(ids)) != 100: + raise AssertionError( + "Expected 100 unique message IDs" + ) + + if set(ids) != {message["id"] for message in all_sent}: + raise AssertionError("Stored messages differ from successful POST responses") + + texts = [message["text"] for message in messages] + + if len(set(texts)) != 100: + raise AssertionError( + "Expected 100 unique message texts" + ) + + if page["next_cursor"] is not None: + raise AssertionError( + "100 messages with limit=100 should fit in one page" + ) + + print( + "PASS concurrency: 10 clients sent 100 messages; " + "exactly 100 unique messages stored" + ) + + +def test_invalid_chat_ids(): + user = create_user("uuid") + for chat_id, expected in (("not-a-uuid", 400), (str(uuid.uuid4()), 404)): + path = f"/api/v1/chats/{chat_id}" + request("GET", path, expected, headers=auth(user)) + request("GET", path + "/messages", expected, headers=auth(user)) + request("POST", path + "/messages", expected, + headers=auth(user), body={"text": "Test"}) + print("PASS invalid chat UUID -> 400; unknown valid UUID -> 404") + + +def test_chat_member_limits(): + users = [create_user("member", login=(i == 0)) for i in range(101)] + owner = users[0] + for members in ([], users, [owner, owner], users[1:3]): + request("POST", "/api/v1/chats", 400, headers=auth(owner), body={ + "title": "Invalid members", "member_ids": [user["id"] for user in members], + }) + request("POST", "/api/v1/chats", 400, headers=auth(owner), body={ + "title": "Unknown member", "member_ids": [owner["id"], str(uuid.uuid4())], + }) + chat = create_chat(owner, users[:100]) + if len(chat["member_ids"]) != 100: + raise AssertionError("Chat must accept exactly 100 members") + print("PASS chat members: 1..100, unique, existing, caller included") + + +def test_invalid_pagination(): + user = create_user("cursor") + chat = create_chat(user, [user]) + other_chat = create_chat(user, [user]) + other_message = create_message(user, other_chat, "Other history") + paths = ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages") + for path in paths: + for query in ("cursor=", "cursor=unknown", "limit=0", "limit=101", "limit=abc", "limit=1.5"): + request("GET", path + "?" + query, 400, headers=auth(user)) + request("GET", paths[1] + "?cursor=" + other_message["id"], 400, headers=auth(user)) + print("PASS invalid cursor/limit and cursor from another chat -> 400") + + +def test_multi_page_chats(): + user = create_user("chatpage") + chats = [create_chat(user, [user], f"Chat {i}") for i in range(7)] + chats.sort(key=lambda chat: (chat["created_at"], uuid.UUID(chat["id"]).int)) + collected = [] + cursor = None + for page_number in range(len(chats) + 1): + query = {"limit": 2 if page_number == 0 else 3} + if cursor is not None: + query["cursor"] = cursor + page = request("GET", "/api/v1/chats?" + urllib.parse.urlencode(query), + 200, headers=auth(user))["json"] + collected.extend(chat["id"] for chat in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + else: + raise AssertionError("Chat pagination did not reach the last page") + if collected != [chat["id"] for chat in chats]: + raise AssertionError("Chat pagination lost, duplicated or reordered chats") + print("PASS pagination: 7 chats, changing limit between pages") + + +def test_validation_and_unicode(): + for display_name in ("", " \t", "x" * 101, 123, None): + request("POST", "/api/v1/users", 400, body={ + "username": unique_username("invalid"), "display_name": display_name, "password": PASSWORD, + }) + request("POST", "/api/v1/users", 400, + body={"username": True, "display_name": "Test", "password": PASSWORD}) + for password in ("short", "x" * 129, 12345678901234, None): + request("POST", "/api/v1/users", 400, body={ + "username": unique_username("badpw"), "display_name": "Test", "password": password, + }) + request("POST", "/api/v1/users", 400, + body={"username": unique_username("nopw"), "display_name": "Test"}) + + unicode_password = "\U00020000" * 12 # пароль не из ASCII проходит через Basic/UTF-8 и KDF + user = request("POST", "/api/v1/users", 201, body={ + "username": unique_username("unicode"), "display_name": "\U00020000" * 100, + "password": unicode_password, + })["json"] + session = request("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(user["username"], unicode_password)})["json"] + user["token"] = session["token"] + + chat = create_chat(user, [user]) + path = f"/api/v1/chats/{chat['id']}/messages" + text = "\U00020000" * 4000 + message = create_message(user, chat, text) + if message["text"] != text: + raise AssertionError("Unicode message was altered") + for invalid in (text + "x", 123, None): + request("POST", path, 400, headers=auth(user), body={"text": invalid}) + + request("GET", "/api/v1/users/me", 401) + request("GET", "/api/v1/users/me", 401, headers={"X-User-Id": user["id"]}) + request("GET", "/api/v1/users/me", 401, + headers={"Authorization": basic(user["username"], unicode_password)}) + print("PASS strict types, Unicode, password rules, Bearer-only identification") + + +def main(): + tests = [ + test_malformed_json, + test_duplicate_username, + test_outsider_chat, + test_empty_and_whitespace_message, + test_extra_json_field, + test_multi_page_history, + test_concurrent_100_messages, + test_invalid_chat_ids, + test_chat_member_limits, + test_invalid_pagination, + test_multi_page_chats, + test_validation_and_unicode, + ] + + print(f"Testing {BASE_URL}") + + for test in tests: + test() + + print() + print( + f"PASS: {len(tests)} student test scenarios completed" + ) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print( + f"FAIL: {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + sys.exit(1) diff --git a/student_tests/test_level4.py b/student_tests/test_level4.py new file mode 100644 index 00000000..d523e36b --- /dev/null +++ b/student_tests/test_level4.py @@ -0,0 +1,195 @@ +"""Проверки уровня 4. Нужен запущенный стенд (docker compose up -d --build). +Тесты отказов сами останавливают и запускают valkey/postgres/api, volumes не удаляются. +Запуск: python3 -m unittest discover -s student_tests -p "test_level4.py" -v +""" +import base64 +import json +import os +import subprocess +import threading +import time +import unittest +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +BASE = os.environ.get("BASE_URL", "http://localhost:8080") +ROOT = Path(__file__).resolve().parents[1] +PASSWORD = "correct-horse-battery" + + +def call(method, path, body=None, headers=None): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if body is not None else {} + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read() + return resp.status, (json.loads(raw) if raw else None) + except urllib.error.HTTPError as exc: + raw = exc.read() + try: + return exc.code, (json.loads(raw) if raw else None) + except ValueError: + return exc.code, None + + +def register(username=None): + username = username or "u" + uuid.uuid4().hex[:12] + status, user = call("POST", "/api/v1/users", + {"username": username, "display_name": "Test", "password": PASSWORD}) + assert status == 201, status + return user + + +def login(username, password=PASSWORD): + basic = base64.b64encode(f"{username}:{password}".encode()).decode() + return call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + + +def bearer(token): + return {"Authorization": "Bearer " + token} + + +def compose(*args, env=None): + full_env = dict(os.environ) + full_env.update(env or {}) + result = subprocess.run(["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, env=full_env, check=True, capture_output=True, text=True) + return result.stdout + + +def wait_ready(timeout=90): + deadline = time.time() + timeout + while time.time() < deadline: + try: + if call("GET", "/health/ready")[0] == 200: + return + except OSError: + pass + time.sleep(1) + raise AssertionError("API did not become ready") + + +def new_session(): + user = register() + status, session = login(user["username"]) + assert status == 201 + return user, session["token"] + + +class Level4(unittest.TestCase): + @classmethod + def setUpClass(cls): + wait_ready() + + def test_wrong_password(self): + user = register() + status, body = login(user["username"], "definitely-wrong-password") + self.assertEqual(status, 401) + self.assertEqual(body["error"]["code"], "unauthorized") + status, _ = login("no_such_user_" + uuid.uuid4().hex[:8]) + self.assertEqual(status, 401) + + def test_logout_revokes_session_and_cache(self): + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) # прогрев кэша + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token))[0], 204) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 401) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token))[0], 401) + + def test_x_user_id_gives_no_access(self): + user = register() + status, _ = call("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}) + self.assertEqual(status, 401) + + def test_register_race_same_username(self): + username = "race" + uuid.uuid4().hex[:10] + results = [] + + def worker(): + status, _ = call("POST", "/api/v1/users", + {"username": username, "display_name": "Race", "password": PASSWORD}) + results.append(status) + + threads = [threading.Thread(target=worker) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(results.count(201), 1, results) + self.assertEqual(results.count(409), 9, results) + + def test_chat_creation_rolls_back(self): + user, token = new_session() + title = "rollback-" + uuid.uuid4().hex + status, _ = call("POST", "/api/v1/chats", + {"title": title, "member_ids": [user["id"], str(uuid.uuid4())]}, + headers=bearer(token)) + self.assertEqual(status, 400) + # Проверяем саму БД: после отката не осталось ни чата, ни участников. + count = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-tAc", f"SELECT count(*) FROM chats WHERE title = '{title}'").strip() + self.assertEqual(count, "0") + + def test_valkey_down_falls_back_to_postgres(self): + user, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + compose("stop", "valkey") + try: + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + self.assertEqual(login(user["username"])[0], 201) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer("x" * 43))[0], 401) + finally: + compose("start", "valkey") + + def test_valkey_flush_keeps_sessions(self): + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + compose("exec", "-T", "valkey", "valkey-cli", "FLUSHALL") + # кэш пуст, но сессия живёт в PostgreSQL; ответ не зависит от состояния кэша + deadline = time.time() + 10 + refilled = False + while time.time() < deadline: + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + keys = compose("exec", "-T", "valkey", "valkey-cli", "keys", "sess:*") + if keys.strip(): + refilled = True + break + time.sleep(1) + self.assertTrue(refilled, "cache should be refilled after a miss") + + def test_postgres_down_returns_bounded_error(self): + compose("stop", "postgres") + try: + status, body = call("GET", "/health/ready") + self.assertEqual(status, 503) + self.assertEqual(body["error"]["code"], "unavailable") + # неизвестный токен нельзя ни подтвердить, ни отвергнуть без БД: 503, не 200 + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer("y" * 43))[0], 503) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # аноним не доверенный + self.assertEqual(call("POST", "/api/v1/users", + {"username": "down" + uuid.uuid4().hex[:8], "display_name": "X", + "password": PASSWORD})[0], 503) + finally: + compose("start", "postgres") + wait_ready() + + def test_zz_session_expiry(self): + compose("up", "-d", "api", env={"SESSION_TTL_SECONDS": "2"}) + try: + wait_ready() + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + time.sleep(3.5) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 401) + finally: + compose("up", "-d", "api") # обратно к TTL из .env + wait_ready() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/student_tests/test_lifecycle.py b/student_tests/test_lifecycle.py new file mode 100644 index 00000000..5b87eff8 --- /dev/null +++ b/student_tests/test_lifecycle.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Жизненный цикл ЛР 2: данные и сессии переживают перезапуск всего стенда (API, PostgreSQL, Valkey). +Нужен запущенный compose-стенд; перезапуск делает scripts/restart.sh (volumes сохраняются).""" +import os +import sys +import uuid + +import test_lab2 as api +from test_sigterm import ROOT, run, wait_ready + + +def compose_logs(): + return run("docker", "compose", "--project-directory", str(ROOT), "--env-file", str(ROOT / ".env"), + "logs", "api", "--no-color").stdout + + +def main(): + api.BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + wait_ready(api.BASE_URL) + + alice = api.create_user("lifealice") + bob = api.create_user("lifebob") + outsider = api.create_user("lifeoutsider") + chat = api.create_chat(alice, [alice, bob], "Lifecycle test") + texts = ["private-lifecycle-" + uuid.uuid4().hex for _ in range(2)] + sent = [api.create_message(user, chat, text) for user, text in zip((alice, bob), texts)] + path = f"/api/v1/chats/{chat['id']}/messages" + page = api.request("GET", path, 200, headers=api.auth(bob))["json"] + if [m["id"] for m in page["items"]] != [m["id"] for m in sent]: + raise AssertionError("Created messages are missing or reordered") + print("PASS two users created and read two messages: HTTP 201, then HTTP 200") + + api.request("GET", path, 404, headers=api.auth(outsider)) + api.request("POST", path, 404, headers=api.auth(outsider), body={"text": "Forbidden"}) + print("PASS third user cannot read or write the chat: HTTP 404") + + logs = compose_logs() + if any(text in logs for text in texts): + raise AssertionError("Message contents leaked into container logs") + if "req_id=" not in logs or "with status 201" not in logs: + raise AssertionError("Request logs are missing") + print("PASS request logs include IDs and statuses, without message contents") + + print("Command: scripts/restart.sh (API + PostgreSQL + Valkey, volumes kept)", flush=True) + run(str(ROOT / "scripts" / "restart.sh"), timeout=120) + wait_ready(api.BASE_URL, timeout=90) + + # те же токены, те же данные + api.request("GET", "/api/v1/users/me", 200, headers=api.auth(alice)) + after = api.request("GET", path, 200, headers=api.auth(bob))["json"] + if after["items"] != page["items"]: + raise AssertionError("History changed after restart") + api.request("GET", path, 404, headers=api.auth(outsider)) + print("PASS after restart: same tokens read the same history") + + api.request("DELETE", "/api/v1/auth/sessions/current", 204, headers=api.auth(alice)) + api.request("GET", path, 401, headers=api.auth(alice)) + api.request("GET", path, 200, headers=api.auth(bob)) + print("PASS logout revokes only alice's session; bob's session still works") + print("PASS persistent lifecycle: data and sessions survive full restart") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"FAIL: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/student_tests/test_sigterm.py b/student_tests/test_sigterm.py new file mode 100644 index 00000000..4651f00a --- /dev/null +++ b/student_tests/test_sigterm.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +import concurrent.futures +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab3") +NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab3_default") +PORT = int(os.environ.get("SIGTERM_PORT", "18080")) + + +def run(*args, check=True, timeout=60): + return subprocess.run( + args, check=check, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout, + ) + +def docker_run_args(name, port): + # Отдельный контейнер API в сети compose: те же PostgreSQL и Valkey, секрет только для чтения + return ["docker", "run", "-d", "--name", name, "--network", NETWORK, + "-p", f"127.0.0.1:{port}:8080", + "-v", f"{ROOT}/secrets/database_url:/run/secrets/database_url:ro", + "-e", "DATABASE_URL_FILE=/run/secrets/database_url", + "-e", "CACHE_URL=redis://valkey:6379/0", + IMAGE] + + +def wait_ready(base_url, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=1) as response: + if response.status == 200: + return + except (urllib.error.URLError, ConnectionError, TimeoutError): + pass + time.sleep(0.2) + raise RuntimeError("Test container did not become ready") + + +def connection_rejected(base_url): + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5): + return False + except urllib.error.HTTPError: + return False + except (urllib.error.URLError, ConnectionError, TimeoutError): + return True + +def not_ready(base_url): + # Во время drain /health/ready отвечает 503 (или соединение уже закрыто) + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5): + return False + except urllib.error.HTTPError as exc: + return exc.code == 503 + except (urllib.error.URLError, ConnectionError, TimeoutError): + return True + +def test_shutdown(incomplete_request): + name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8] + base_url = f"http://127.0.0.1:{PORT}" + scenario = "incomplete POST" if incomplete_request else "idle" + pending = None + try: + run(*docker_run_args(name, PORT)) + wait_ready(base_url) + + if incomplete_request: + pending = socket.create_connection(("127.0.0.1", PORT), timeout=3) + pending.sendall( + b"POST /api/v1/users HTTP/1.1\r\n" + b"Host: localhost\r\nContent-Type: application/json\r\n" + b"Content-Length: 100\r\nExpect: 100-continue\r\n\r\n" + ) + response = b"" + while b"\r\n\r\n" not in response: + part = pending.recv(1024) + if not part: + raise AssertionError("Connection closed before 100 Continue") + response += part + if b"100 Continue" not in response: + raise AssertionError(f"Expected 100 Continue, received {response!r}") + pending.sendall(b"{") + + print(f"Scenario: {scenario}; docker stop --timeout 20 {name}", flush=True) + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + stopped = executor.submit(run, "docker", "stop", "--timeout", "20", name) + deadline = start + 2 + while not not_ready(base_url): + if time.monotonic() >= deadline: + raise AssertionError("Readiness still OK after SIGTERM") + time.sleep(0.05) + rejected_after = time.monotonic() - start + stopped.result(timeout=25) + elapsed = time.monotonic() - start + + exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout) + if exit_code != 0: + raise AssertionError(f"Container exited with code {exit_code}, expected 0") + if elapsed > 15: + raise AssertionError(f"Shutdown exceeded 15 seconds: {elapsed:.3f}s") + if not connection_rejected(base_url): + raise AssertionError("API still accepts connections after shutdown") + + logs = run("docker", "logs", name).stdout + if "Application shutdown complete" not in logs: + raise AssertionError("Application did not complete its shutdown handler") + print(f"PASS {scenario}: exit code 0; shutdown {elapsed:.3f}s; " + f"new connections rejected after {rejected_after:.3f}s") + finally: + if pending is not None: + pending.close() + run("docker", "rm", "-f", name, check=False) + + +def main(): + print(f"Image: {IMAGE}") + test_shutdown(False) + test_shutdown(True) + print("PASS SIGTERM: both scenarios completed within 10 seconds") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"FAIL: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/student_tests/track2_experiment.py b/student_tests/track2_experiment.py new file mode 100644 index 00000000..11965097 --- /dev/null +++ b/student_tests/track2_experiment.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Трек 2: reconciler. python3 student_tests/track2_experiment.py --seed 1000 5000 | tee evidence/track2-experiment.txt +Меняет только синтетические данные; кроме них удаляет лишь aged-сирот, найденные базовым запуском.""" +import argparse +import hashlib +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_attachments_level4 import ROOT, SAMPLE, http_call, jcall, new_chat, new_user, psql, upload # noqa: E402 + +GRACE = 30 +FAILS = [] + + +def check(ok, text): + print(("PASS " if ok else "FAIL ") + text, flush=True) + if not ok: + FAILS.append(text) + + +def api_exec(code, *args, stdin=b""): + r = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code, *args], + cwd=ROOT, input=stdin, capture_output=True, check=True) + return r.stdout.decode().strip() + + +PUT = "import sys; from src import storage, config; storage.client().put_object(Bucket=config.S3_BUCKET, Key=sys.argv[1], Body=sys.stdin.buffer.read())" +DEL = "import sys; from src import storage; storage.delete_object(sys.argv[1])" +HEAD = """import sys +from botocore.exceptions import ClientError +from src import storage, config +try: + storage.client().head_object(Bucket=config.S3_BUCKET, Key=sys.argv[1]); print(1) +except ClientError: + print(0)""" +SEED = """import sys, uuid +from concurrent.futures import ThreadPoolExecutor +from src import storage, config +c = storage.client() +def put(i): c.put_object(Bucket=config.S3_BUCKET, Key="attachments/seed-" + uuid.uuid4().hex, Body=b"seed") +with ThreadPoolExecutor(16) as ex: list(ex.map(put, range(int(sys.argv[1]))))""" + + +def put(key, data=b"orphan"): api_exec(PUT, key, stdin=data) +def delete(key): api_exec(DEL, key) +def exists(key): return api_exec(HEAD, key) == "1" +def new_key(): return "attachments/" + str(uuid.uuid4()) + + +def reconcile(*extra): + r = subprocess.run(["docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "-T", "reconciler", + "python", "-m", "src.reconciler", "--grace", str(GRACE), *extra], + cwd=ROOT, capture_output=True, text=True) + line = [l for l in r.stdout.splitlines() if l.startswith("{")][-1] + print(" reconciler", " ".join(extra) or "(dry-run)", "->", line, f"[exit {r.returncode}]", flush=True) + return json.loads(line), r.returncode + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--seed", type=int, nargs="*", default=[1000]) + args = ap.parse_args() + + user, auth = new_user() + chat = new_chat(auth, [user["id"]]) + + def up(): + st, _, raw = upload(auth, chat, SAMPLE) + assert st == 201, st + return json.loads(raw) + + def attach(att): + st, _ = jcall("POST", f"/api/v1/chats/{chat}/messages", {"text": "", "attachment_ids": [att["id"]]}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}) + assert st == 201, st + + def content(att): return http_call("GET", f"/api/v1/attachments/{att['id']}/content", headers=auth) + def key_of(att): return psql(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + def state_of(att): return psql(f"SELECT storage_state FROM attachments WHERE id = '{att['id']}'") + + print("== 0. Базовая очистка (чтобы счётчики ниже были точными)") + time.sleep(GRACE + 1) + reconcile("--apply") + + print("== 1. Данные: используемые и неиспользуемые вложения") + used = up(); attach(used) + unlinked = up() + lost_used = up(); attach(lost_used) + lost_unused = up() + fresh_lost = None + + print("== 2. Внедрение ошибок. S3-сторона: 3 объекта без метаданных; БД-сторона: у 2 записей объект удалён") + orphans = [new_key() for _ in range(3)] + for k in orphans: + put(k) + lost_used_key, lost_unused_key = key_of(lost_used), key_of(lost_unused) + delete(lost_used_key) + delete(lost_unused_key) + time.sleep(GRACE + 2) + fresh_orphan = new_key() + put(fresh_orphan) + fresh_lost = up() + delete(key_of(fresh_lost)) + + print("== 3. DRY-RUN: обнаруживает, но ничего не меняет") + r, _ = reconcile() + check(r["dry_run"] and r["orphans_found"] == 3 and r["missing_found"] == 2, "dry-run нашёл 3 сирот и 2 потерянных объекта") + check(r["orphans_fresh_skipped"] == 1 and r["missing_fresh_skipped"] == 1, "свежие сирота и запись (моложе grace) пропущены") + check(r["orphans_deleted"] == 0 and r["missing_marked"] == 0, "dry-run ничего не изменил в счётчиках") + check(all(exists(k) for k in orphans), "после dry-run все сироты ещё в S3") + check(state_of(lost_used) == "ok" and state_of(lost_unused) == "ok", "после dry-run состояния записей не изменены") + + print("== 4. APPLY и сразу повторный APPLY (идемпотентность)") + r, _ = reconcile("--apply") + r2, _ = reconcile("--apply") + check(r["orphans_deleted"] == 3 and r["missing_marked"] == 2, "удалены 3 сироты, помечены 2 потерянных объекта") + check(r2["orphans_found"] == 0 and r2["missing_found"] == 0 and r2["orphans_deleted"] == 0 and r2["missing_marked"] == 0, + "повторный apply ничего не находит и не меняет") + check(exists(fresh_orphan), "свежая сирота НЕ удалена (grace period)") + check(state_of(fresh_lost) == "ok", "свежая запись без объекта НЕ помечена (grace period)") + check(not any(exists(k) for k in orphans), "старые сироты удалены из S3") + for name, att in (("используемое", used), ("неиспользуемое, но с метаданными", unlinked)): + st, _, raw = content(att) + check(st == 200 and hashlib.sha256(raw).hexdigest() == att["sha256"], f"{name} вложение старше grace цело и скачивается") + check(content(lost_used)[0] == 409 and content(lost_unused)[0] == 409, "потерянные вложения: скачивание 409, а не 500") + check(http_call("GET", f"/api/v1/attachments/{lost_used['id']}", headers=auth)[0] == 200, "метаданные потерянного вложения остаются (запись не удалена)") + st, _ = jcall("POST", f"/api/v1/chats/{chat}/messages", {"text": "", "attachment_ids": [lost_unused["id"]]}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}) + check(st == 409, "потерянное вложение нельзя прикрепить к новому сообщению (409)") + + print("== 6. Восстановление: объект вернули на то же место") + put(lost_used_key, SAMPLE) + r, _ = reconcile("--apply") + check(r["restored"] == 1 and state_of(lost_used) == "ok", "запись снова ok после проверки размера и SHA-256") + st, _, raw = content(lost_used) + check(st == 200 and hashlib.sha256(raw).hexdigest() == lost_used["sha256"], "восстановленное вложение скачивается") + + print("== 7. Прошёл grace: бывшие свежие объекты теперь обрабатываются") + time.sleep(GRACE + 2) + r, _ = reconcile("--apply") + check(r["orphans_deleted"] == 1 and r["missing_marked"] == 1, "теперь удалена вторая сирота и помечена вторая потеря") + + print("== 8. Стоимость обхода и предохранитель") + for n in args.seed: + print(f"-- seed {n} объектов-сирот") + api_exec(SEED, str(n)) + time.sleep(GRACE + 2) + t0 = time.time() + r, _ = reconcile() + print(f" dry-run N={n}: reconciler elapsed={r['elapsed_s']}s s3_list={r['s3_list_s']}s pages={r['s3_list_pages']} " + f"db_rows={r['db_rows']} peak_rss={r['peak_rss_mib']}MiB (с запуском контейнера {time.time() - t0:.1f}s)") + check(r["orphans_found"] >= n, f"dry-run видит {n} сирот") + if n > 1000: + r, code = reconcile("--apply") + check(code == 3 and r.get("error") == "too_many_actions" and r["orphans_deleted"] == 0, + "предохранитель: массовое удаление выше --max-actions отклонено, ничего не удалено") + r, _ = reconcile("--apply", "--max-actions", str(n + 100)) + check(r["orphans_deleted"] >= n, f"очистка {n} сирот выполнена") + + print("\nИТОГ:", "FAIL" if FAILS else "PASS") + sys.exit(1 if FAILS else 0) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/upload_memory.py b/student_tests/upload_memory.py new file mode 100644 index 00000000..ae8f0800 --- /dev/null +++ b/student_tests/upload_memory.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""N одновременных медленных загрузок ~10 МиБ через LB; пик памяти api-контейнеров по cgroup. +python3 student_tests/upload_memory.py --concurrency 10 --output evidence/upload-memory-10.json""" +import argparse +import http.client +import json +import os +import struct +import subprocess +import sys +import threading +import time +import zlib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from test_attachments_level4 import SIG, URL, chunk, compose, new_chat, new_user # noqa: E402 + +CG2 = "cat /sys/fs/cgroup/memory.current; grep -E '^(anon|file) ' /sys/fs/cgroup/memory.stat" +CG1 = "cat /sys/fs/cgroup/memory/memory.usage_in_bytes; grep -E '^(rss|cache) ' /sys/fs/cgroup/memory/memory.stat" +MIB = 1024 * 1024 + + +def build_png(target, idat_chunk): + w = 1000 + h = target // (w * 3 + 1) + comp = zlib.compress(b"".join(b"\x00" + os.urandom(w * 3) for _ in range(h)), 0) + step = idat_chunk or len(comp) + idat = b"".join(chunk(b"IDAT", comp[i:i + step]) for i in range(0, len(comp), step)) + return SIG + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + idat + chunk(b"IEND", b"") + + +def read_mem(cid): + for cmd in (CG2, CG1): + r = subprocess.run(["docker", "exec", cid, "sh", "-c", cmd], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip(): + lines = r.stdout.strip().split("\n") + d = {p[0]: int(p[1]) for p in (l.split() for l in lines[1:]) if len(p) == 2} + return int(lines[0]), d.get("anon", d.get("rss", 0)), d.get("file", d.get("cache", 0)) + raise SystemExit("не удалось прочитать cgroup-память контейнера") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--concurrency", type=int, default=10) + ap.add_argument("--mode", choices=["normal", "single-idat"], default="normal", + help="single-idat: весь PNG одним IDAT-чанком (худший случай для проверки структуры)") + ap.add_argument("--chunk-delay-ms", type=float, default=20, help="пауза между 64 КиБ: медленный клиент") + ap.add_argument("--output", required=True) + args = ap.parse_args() + + body = build_png(10 * MIB - 200_000, 0 if args.mode == "single-idat" else 32768) + assert len(body) <= 10 * MIB + user, auth = new_user() + chat = new_chat(auth, [user["id"]]) + cids = compose("ps", "-q", "api").split() + limits = {c: int(subprocess.run(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", c], + capture_output=True, text=True).stdout.strip() or 0) for c in cids} + base = {c: read_mem(c) for c in cids} + peak = {c: list(base[c]) for c in cids} + stop = threading.Event() + + def monitor(cid): + while not stop.is_set(): + cur = read_mem(cid) + peak[cid] = [max(a, b) for a, b in zip(peak[cid], cur)] + + mons = [threading.Thread(target=monitor, args=(c,)) for c in cids] + statuses = [None] * args.concurrency + barrier = threading.Barrier(args.concurrency) + + def one(i): + barrier.wait() + c = http.client.HTTPConnection(URL.hostname, URL.port, timeout=180) + c.putrequest("POST", f"/api/v1/chats/{chat}/attachments?filename=m{i}.png") + for k, v in {**auth, "Content-Type": "image/png", "Content-Length": str(len(body))}.items(): + c.putheader(k, v) + c.endheaders() + for off in range(0, len(body), 65536): + c.send(body[off:off + 65536]) + time.sleep(args.chunk_delay_ms / 1000) + r = c.getresponse() + r.read() + statuses[i] = r.status + + [m.start() for m in mons] + ts = [threading.Thread(target=one, args=(i,)) for i in range(args.concurrency)] + t0 = time.time() + [t.start() for t in ts] + [t.join() for t in ts] + wall = time.time() - t0 + stop.set() + [m.join() for m in mons] + + result = {"mode": args.mode, "concurrency": args.concurrency, "file_mib": round(len(body) / MIB, 2), + "payload_total_mib": round(args.concurrency * len(body) / MIB, 1), "wall_s": round(wall, 1), + "statuses": sorted(set(s for s in statuses)), "ok": statuses.count(201), "containers": {}} + for c in cids: + result["containers"][c[:12]] = { + "limit_mib": round(limits[c] / MIB), "baseline_anon_mib": round(base[c][1] / MIB, 1), + "peak_anon_mib": round(peak[c][1] / MIB, 1), "anon_growth_mib": round((peak[c][1] - base[c][1]) / MIB, 1), + "peak_current_mib_incl_page_cache": round(peak[c][0] / MIB, 1)} + Path(args.output).write_text(json.dumps(result, indent=2, ensure_ascii=False)) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() \ No newline at end of file