From b78cadb8936c1bc28f7629fa5a3b91dc1a983a86 Mon Sep 17 00:00:00 2001 From: angelika005 Date: Thu, 29 Oct 2026 13:02:56 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=204=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22303=20=E2=80=94=20=D0=91=D1=80=D0=B5=D0=B7=D0=B3=D1=83?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 +- .gitattributes | 5 + .github/pull_request_template.md | 30 +- .github/workflows/solution.yml | 25 ++ DEFENSE.md | 39 ++ Dockerfile | 15 + REPORT.md | 158 +++++++- compose.test.yaml | 24 ++ compose.yaml | 158 ++++++++ evidence/acceptance.txt | 3 + evidence/after-restart.json | 44 +++ evidence/before-restart.json | 44 +++ evidence/bootstrap-repeat.txt | 3 + evidence/check.txt | 1 + evidence/compose-config.txt | 0 evidence/docker-environment.json | 8 + evidence/failover.txt | 3 + evidence/final-containers.json | 44 +++ evidence/final-smoke.txt | 1 + evidence/host-memory.json | 3 + evidence/integrity-and-secrets.txt | 2 + evidence/mc-version.txt | 4 + evidence/message-load.json | 17 + evidence/message-load.txt | 17 + evidence/minio-version.txt | 4 + evidence/openapi-validation.txt | 1 + evidence/persistence.txt | 2 + evidence/postgres-outage.txt | 3 + evidence/postgres-reconcile-abort.txt | 4 + evidence/production-containers.txt | 7 + evidence/production-smoke.txt | 1 + evidence/reconciliation.txt | 3 + evidence/restore-api.txt | 18 + evidence/restore-postgres.txt | 2 + evidence/restore-s3.txt | 2 + evidence/restore-upload-replica.txt | 14 + evidence/s3-outage.txt | 3 + evidence/s3-reconcile-abort.txt | 4 + evidence/smoke.txt | 1 + evidence/stop-fault-api.txt | 2 + evidence/stop-postgres.txt | 2 + evidence/stop-s3.txt | 2 + evidence/stop-upload-replica.txt | 2 + evidence/test-services.txt | 49 +++ evidence/upload-failover.json | 1 + evidence/upload-load-runner.txt | 2 + evidence/upload-load.txt | 1 + evidence/verification.json | 7 + infra/Dockerfile.minio | 19 + infra/api-policy.json | 1 + infra/bootstrap.sh | 12 + infra/haproxy.cfg | 26 ++ infra/minio-start.sh | 5 + infra/reconcile-policy.json | 1 + migrations/001_initial.sql | 39 ++ requirements.lock | 26 ++ requirements.txt | 6 + scripts/migrate.py | 7 + scripts/restart.cmd | 3 + scripts/restart.sh | 4 + scripts/setup.py | 21 ++ scripts/stop-one.cmd | 7 + scripts/stop-one.sh | 5 + scripts/verify.py | 120 ++++++ src/__init__.py | 1 + src/app.py | 508 ++++++++++++++++++++++++++ src/reconcile.py | 146 ++++++++ src/storage.py | 30 ++ src/sweep.py | 11 + student-tests/acceptance.py | 119 ++++++ student-tests/outage.py | 18 + student-tests/reconciliation.py | 101 +++++ student-tests/upload_load.py | 88 +++++ 73 files changed, 2076 insertions(+), 36 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/solution.yml create mode 100644 DEFENSE.md create mode 100644 Dockerfile create mode 100644 compose.test.yaml create mode 100644 compose.yaml create mode 100644 evidence/acceptance.txt create mode 100644 evidence/after-restart.json create mode 100644 evidence/before-restart.json create mode 100644 evidence/bootstrap-repeat.txt create mode 100644 evidence/check.txt create mode 100644 evidence/compose-config.txt create mode 100644 evidence/docker-environment.json create mode 100644 evidence/failover.txt create mode 100644 evidence/final-containers.json create mode 100644 evidence/final-smoke.txt create mode 100644 evidence/host-memory.json create mode 100644 evidence/integrity-and-secrets.txt create mode 100644 evidence/mc-version.txt create mode 100644 evidence/message-load.json create mode 100644 evidence/message-load.txt create mode 100644 evidence/minio-version.txt create mode 100644 evidence/openapi-validation.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/postgres-outage.txt create mode 100644 evidence/postgres-reconcile-abort.txt create mode 100644 evidence/production-containers.txt create mode 100644 evidence/production-smoke.txt create mode 100644 evidence/reconciliation.txt create mode 100644 evidence/restore-api.txt create mode 100644 evidence/restore-postgres.txt create mode 100644 evidence/restore-s3.txt create mode 100644 evidence/restore-upload-replica.txt create mode 100644 evidence/s3-outage.txt create mode 100644 evidence/s3-reconcile-abort.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stop-fault-api.txt create mode 100644 evidence/stop-postgres.txt create mode 100644 evidence/stop-s3.txt create mode 100644 evidence/stop-upload-replica.txt create mode 100644 evidence/test-services.txt create mode 100644 evidence/upload-failover.json create mode 100644 evidence/upload-load-runner.txt create mode 100644 evidence/upload-load.txt create mode 100644 evidence/verification.json create mode 100644 infra/Dockerfile.minio create mode 100644 infra/api-policy.json create mode 100644 infra/bootstrap.sh create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-start.sh create mode 100644 infra/reconcile-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100644 scripts/migrate.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/setup.py create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify.py create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/reconcile.py create mode 100644 src/storage.py create mode 100644 src/sweep.py create mode 100644 student-tests/acceptance.py create mode 100644 student-tests/outage.py create mode 100644 student-tests/reconciliation.py create mode 100644 student-tests/upload_load.py diff --git a/.dockerignore b/.dockerignore index bc16e556..43d520f3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,5 +16,6 @@ backups .venv __pycache__ node_modules -evidence .DS_Store +evidence +*.pyc diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..cfb626a8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.cfg text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..d89c4437 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ -## Лабораторная 4 +## Лабораторная 4 — БВТ2303 — Брезгунова Анжелика — на 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Трек 2: согласованность PostgreSQL и S3. Реализованы синхронный PNG/JPEG upload, Attachment, ACL download, две API за HAProxy и persistent MinIO. Сохранён минимум lab3: Bearer-сессии, Argon2id, keyset pagination и транзакционная идемпотентность сообщений. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Reconciler dry-run/apply удаляет только старые orphan, защищает fresh/used, карантинит missing после grace и проверяет SHA-256 при restore. Sweeper работает раз в минуту с grace=3600. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `python scripts/setup.py`, `docker compose --env-file .env up -d --build`. API — `http://127.0.0.1:8084`. Полный сценарий: подключить `compose.test.yaml`, затем `python -X utf8 scripts/verify.py`. Подробности и raw evidence — в [REPORT.md](../REPORT.md). + +Offline, OpenAPI, smoke/persistence/failover, 21 негативная проверка, 12 конкурентных повторов через две API, S3/DB outage, ошибки после PUT/на COMMIT, dry-run/apply/repeat и restore — PASS. При 12 клиентах и двух размерах PNG ошибок 0; максимальный cgroup peak API — 184.5 MiB. + +Ограничения: один host и одиночные хранилища; restore потерянных bytes требует backup; полного bit-rot audit нет. Локальный стенд с синтетическими данными. Библиотеки, источники MinIO/mc и помощь OpenAI Codex раскрыты в отчёте. PR пока не опубликован. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Python-эквиваленты make check / make test выполнены на Windows; вывод приложен. +- [x] Сценарии отказов/безопасности выбранного уровня воспроизведены. +- [x] Секреты находятся вне Git; отчёт и evidence не содержат tokens. +- [x] README задания сохранён; REPORT.md содержит инструкцию решения. diff --git a/.github/workflows/solution.yml b/.github/workflows/solution.yml new file mode 100644 index 00000000..8079d011 --- /dev/null +++ b/.github/workflows/solution.yml @@ -0,0 +1,25 @@ +name: lab4 solution +on: [push, pull_request] +permissions: + contents: read +jobs: + acceptance: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: python scripts/check_contract.py && python scripts/validate_openapi.py + - run: python scripts/setup.py + - run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml --profile tests up -d --build api1 api2 lb fault-api sweeper + - run: python scripts/verify.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab4-evidence + path: evidence/ + - run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml --profile tests down + if: always() diff --git a/DEFENSE.md b/DEFENSE.md new file mode 100644 index 00000000..41f624f7 --- /dev/null +++ b/DEFENSE.md @@ -0,0 +1,39 @@ +# Памятка для защиты — Брезгунова Анжелика, БВТ2303 + +Цель: **5, трек 2**. Показать `REPORT.md`, `compose.yaml`, `src/app.py`, `src/reconcile.py` и фактические результаты `evidence/`. + +## Демонстрация + +1. `docker compose --env-file .env ps`: две здоровые API, PostgreSQL, MinIO, LB и sweeper; только LB имеет host port 8084. +2. `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084`: PNG → сообщение → участник скачал те же bytes, SHA-256 совпал; посторонний получил 404. +3. `evidence/upload-failover.json`: загрузка api1 → api1 остановлена → download api2, checksum совпал. +4. `before-restart.json`, `after-restart.json`, `persistence.txt`: основные сервисы действительно restart, данные и прежняя сессия сохранены. +5. `acceptance.txt`: 21 негативная проверка, JPEG, private bucket и ограниченная S3 policy. +6. `reconciliation.txt`: dry-run без удаления, apply старых orphan, repeat без повторных изменений, fresh сохранён; missing сохраняет message link, restore проверяет bytes; 10 параллельных originals не потеряны. +7. `upload-load.txt`: 12 клиента, 2 настоящих PNG, 0 ошибок, peak до 184.5 MiB; объяснить ограничения короткого эксперимента. + +## Вопросы и ответы + +**Почему volume одной API-реплики не S3?** Другой процесс не имеет её файловую систему. Общий MinIO хранит обязательный оригинал. Временный API tmpfs нужен во время запроса и не участвует в download после 201. + +**Когда можно вернуть 201?** После успешного PUT и подтверждённого COMMIT метаданных. Клиент получает ready. + +**PUT прошёл, COMMIT не прошёл?** Возможен orphan, который найдёт sweeper после grace. При неясном COMMIT нельзя сразу удалять объект: БД могла подтвердить запись. Сверка повторно проверяет PostgreSQL под блокировкой. + +**Зачем lock и grace одновременно?** Shared lock покрывает PUT→COMMIT, exclusive lock не даёт очистке пересечь эту операцию. Grace защищает свежие объекты и аварийные окна. В обычном режиме — час, 2 секунды только в эксперименте. + +**Почему UUID key и private bucket не заменяют ACL?** Знание ID не доказывает участие в чате. API проверяет membership; прямой S3 GET без credentials запрещён. + +**Как проверяется MIME?** По формату Pillow и полной декодируемости после проверки dimensions. PNG с image/jpeg — 415; bytes/pixels oversize — 413; опасное имя — 400. + +**Как повторы сообщения работают на разных API?** Ключ scoped по user+chat+POST messages. Lock и транзакция связывают Message и результат. Одинаковый body возвращает тот же ID; другой — 409. Отсутствующие attachment_ids равны []. + +**Что исправляет missing-repair?** Сохраняет metadata/ссылки, карантинит устойчиво отсутствующий оригинал и возвращает 503. Restore требует backup: SHA-256 не содержит bytes. После возврата файла сверка проверяет фактическое содержимое и снимает карантин. + +**Почему unused Attachment не удаляется?** Upload и отправка сообщения — разные запросы, пользователь может прикрепить файл позже. Orphan — объект без metadata, а не без ссылки в Message. + +**Как ограничена RAM?** Четыре upload на реплику, 10 MiB/файл, 40M pixels, tmpfs 64 MiB, таймауты и backpressure. Decode требует RAM пропорционально pixels; hash/download идут по 64 KiB. Cgroup memory.peak включает tmpfs, RSS описывает процесс; они различаются. + +**Границы доказательства?** Конкретные отказы, две нагрузки и один ноутбук. Это не HA host/диска, не максимальный production RPS, не полный bit-rot audit и не удалённый TLS deployment. + +Полная повторная демонстрация: подключить `compose.test.yaml` по отчёту и выполнить `python -X utf8 scripts/verify.py`. Он явно останавливает только сервисы `mtusi-lab4`, сохраняя volumes. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ae2b9a6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12.11-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +COPY requirements.lock . +RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home app +COPY src src +COPY migrations migrations +COPY scripts scripts +COPY tests tests +COPY student-tests student-tests +COPY contracts contracts +COPY lab.json . +USER app +CMD ["python", "-m", "uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8080", "--no-access-log", "--limit-concurrency", "64", "--timeout-keep-alive", "5"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..2cb80d20 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,155 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Автор: Брезгунова Анжелика, БВТ2303. Целевая оценка: 5. Трек 2 — согласованность PostgreSQL и S3.** + +Решение выполнено только в `mtusi2026lab4`: самостоятельный сервер сохраняет обязательный минимум lab3, соседние лабораторные не изменялись. Обязательные `contracts/`, публичные `tests/`, `README.md` и `COURSE.md` сохранены. Код, настройки и секреты соседних работ не переносились. Commit/PR не опубликованы; заполненный шаблон находится в `.github/pull_request_template.md`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Linux-контейнерами, Compose v2 и Python 3.9+ для публичных тестов. Проверено на Windows с Python 3.11.9; runtime API — Python 3.12.11. Первый build требует доступа к GitHub, PyPI и Docker Hub. Все команды выполняются **из lab4**. Проект `mtusi-lab4`, его volumes и сети отдельные; наружный порт **8084**, поскольку 8080 занят другой работой. + +```sh +python scripts/setup.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +docker compose --env-file .env ps +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084 +``` + +`setup.py` генерирует случайные локальные secrets в `secrets/` и `.env` с портом/TTL, сохраняя существующие файлы. Они исключены из Git и Docker build context. Значений credentials в YAML и отчёте нет. Порт изменяется через `LAB4_PORT` в `.env`. Готовность: `GET http://127.0.0.1:8084/health/ready`, дождаться 200. Внутренние API-порты, PostgreSQL, S3 и консоль MinIO не публикуются на host. + +Закреплены `postgres:17.6-alpine3.22`, `haproxy:3.0.11-alpine`, Python 3.12.11 и Go 1.24.2. MinIO/mc собираются из официальных исходников по тегам `RELEASE.2025-04-22T22-12-26Z` / `RELEASE.2025-04-16T18-13-26Z`: готовые образы оказались недоступны при проверке. Проверенные commit IDs — `0d7408fc9969caf07de6a8c3a84f9fbb10a6739e` / `b00526b153a31b36767991a4f5ce2cced435ee8e`. Source build показывает префикс DEVELOPMENT, дату и commit выбранных релизов; реальные версии сохранены в `evidence/minio-version.txt` и `mc-version.txt`. Все Python-зависимости, включая транзитивные, закреплены в `requirements.lock`. + +`migrate` выполняет `migrations/001_initial.sql` с advisory lock и идемпотентным созданием схемы. `bootstrap` создаёт private bucket, запрещает anonymous access, создаёт отдельных API/reconciler пользователей с policy. Их exit=0 — нормальный результат; API ждёт успешного завершения обоих. `sweeper` сверяет хранилища каждую минуту с grace=3600 секунд. + +Полная проверка на Windows без Make: + +```powershell +python -X utf8 scripts/check_contract.py +python -m venv .venv +.venv\Scripts\python -m pip install -r requirements-dev.txt +.venv\Scripts\python -X utf8 scripts/validate_openapi.py +docker compose --env-file .env -f compose.yaml -f compose.test.yaml --profile tests up -d --build api1 api2 lb fault-api sweeper +python -X utf8 scripts/verify.py +``` + +На Linux вместо `.venv\Scripts\python` — `.venv/bin/python`. Make-эквиваленты: `make check validate PYTHON=.venv/bin/python` и `make test BASE_URL=http://127.0.0.1:8084`. `verify.py` запускает неизменённые публичные тесты, свои сценарии, реальные restart/stop hooks и сохраняет санитизированные результаты в `evidence/`. Он явно останавливает/возвращает только сервисы `mtusi-lab4`, создаёт синтетические данные и никогда не удаляет volumes. `--quick` пропускает load. На Linux скрипт сам устанавливает executable bit для hooks; отдельно можно выполнить `chmod +x scripts/restart.sh scripts/stop-one.sh`. Windows hooks — `.cmd`. + +`compose.test.yaml` включает внутренние метрики и test-runner с административными credentials для внесения ошибок. `fault-api` имеет отдельный профиль, не входит в upstream LB. Метрики запрещены на LB и выключены в обычном стенде. После испытаний вернуть обычный режим: + +```sh +docker compose --env-file .env up -d api1 api2 sweeper lb +docker compose --env-file .env -f compose.yaml -f compose.test.yaml --profile tests stop fault-api +``` + +Безопасная остановка: `docker compose --env-file .env down`. **Намеренное удаление только учебных данных lab4:** `docker compose --env-file .env down --volumes`; эта команда не использовалась в проверках persistence. Нельзя менять PostgreSQL password независимо от БД с сохранённым volume. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy :8084] + LB --> A1[API api1] + LB --> A2[API api2] + A1 --> P[(PostgreSQL: metadata и sessions)] + A2 --> P + A1 --> S[(Приватный MinIO: bytes)] + A2 --> S + R[Периодическая сверка] --> P + R --> S +``` + +HAProxy проверяет `/health/ready`, исключает неготовые API, обновляет адреса через Docker DNS после пересоздания контейнеров. Только LB подключён к сети с публикацией порта. `live` зависит от жизни процесса; `ready` требует доступную PostgreSQL со схемой и S3 bucket, поскольку upload — обязательный синхронный путь. + +PostgreSQL — источник истины для пользователей, membership, чатов, сообщений, сессий и Attachment. Сессия хранит SHA-256 случайного bearer token с энтропией 256 бит и абсолютный expiry; restart не продлевает TTL. Пароли — Argon2id (`time_cost=2`, `memory_cost=19456 KiB`, `parallelism=1`), без усечения UTF-8. Basic используется только для login, `X-User-Id` не меняет личность. Membership проверяется перед metadata/download; посторонний получает 404. + +Message и результат idempotency фиксируются одной транзакцией. Advisory lock по user+chat+key сериализует повторы на разных API. Канонизация считает отсутствие `attachment_ids` и `[]` эквивалентными; TTL ключа — 24 часа. Keyset по `(created_at,id)` использует HMAC-подписанный cursor со scope пользователя/чата. На неизменной истории нет повторов; между страницами нет snapshot, конкурентная запись с временем до cursor может быть пропущена. Время назначает одна PostgreSQL, но скачок её часов остаётся ограничением. + +Attachment заранее принадлежит чату; `message_attachments` содержит внешние ключи и порядок ссылок. Key — `originals/`; filename — только отображаемое имя. В API нет ключей bucket, внутренних адресов или credentials. + +Upload принимает raw PNG/JPEG. Content-Length проверяется заранее, chunked — накопительно до **10485760 bytes**. Четыре upload-слота на реплику; ожидание слота ≤1 сек (далее 429), приём ≤30 сек. Поток пишется во временный файл на tmpfs 64 MiB, ≤10 MiB на файл. Это staging только во время запроса: после 201 он закрыт/удалён, download не зависит от API filesystem. Pillow проверяет формат/MIME, dimensions до полной распаковки (**40M pixels**), структуру и декодируемость; многокадровые файлы отклоняются. Опасный filename — 400, corrupt/MIME — 415, byte/pixel overflow — 413. JSON ограничен 64 KiB/10 сек; неверные поля, типы и surrogate — 400. Download/hash идут потоками по 64 KiB, без redirect. + +Транзакционная граница upload: после ACL берётся shared advisory lock, S3 PUT сохраняет оригинал с транспортной проверкой MD5 и SHA-256 metadata, далее INSERT Attachment и COMMIT. **201 ready возвращается только после COMMIT.** При неясном результате COMMIT нельзя сразу удалить объект: БД могла сохранить metadata. Компенсация выполняется сверкой подтверждённого состояния под exclusive lock. API read-only и без persistent volume; `postgres-data` и `s3-data` принадлежат хранилищам. + +API S3 policy разрешает Get/Put только `originals/*` и ListBucket для HEAD/readiness, без DeleteObject/admin. Reconciler — List/Get/Delete в prefix, без Put. Root credentials доступны только MinIO, bootstrap и тестовому оператору. Консоль слушает `127.0.0.1:9001` внутри S3-контейнера, недоступна общей сети/host. + +Соответствие `.env.example`: API внутри слушает 0.0.0.0:8080, наружный порт задаёт `LAB4_PORT`; TTL используется напрямую; DB/S3 credentials — через `*_FILE`, endpoint/bucket/limits заданы Compose. Две явно именованные реплики; sessions в PostgreSQL, `CACHE_URL_FILE` не используется. Idempotency TTL соответствует фиксированным 24 часам. Обязательный минимум lab3 реализован без Redis/Valkey. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Полный запуск `python -X utf8 scripts/verify.py` — **PASS, 10.10.2026** (`evidence/verification.json`). Ниже имена файлов относятся к `evidence/`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Сценарий и наблюдение | Доказательство | +| --- | --- | --- | +| Offline / OpenAPI | Оба валидатора PASS, Python-эквиваленты Make на Windows | `check.txt`, `openapi-validation.txt` | +| Обязательный lab3/4 API | Smoke: Unicode, ACL, pagination, sessions, idempotency, SHA-256 PNG, ссылка в Message, logout | `smoke.txt`, `final-smoke.txt` | +| Две API и persistent DB/S3 | 2 API, PostgreSQL, MinIO, LB; API без state volume | `final-containers.json`, `compose.yaml` | +| PNG/JPEG, ограничения, чужие ID | JPEG byte roundtrip; 21 негативная проверка, >10 MiB с/без Content-Length, >40M pixels, повреждение, MIME, filename, ACL/другой чат, JSON | `acceptance.txt`, `student-tests/acceptance.py` | +| Конкурентная идемпотентность | 12 повторов через 2 реплики — один Message ID | `acceptance.txt` | +| Остановка upload-реплики | Upload api1 → stop api1 → download api2, bytes совпали | `upload-failover.json` | +| Persistence всего стенда | API, DB, S3, LB, sweeper реально restart; прежний token и bytes сохранены | `persistence.txt`, `before-restart.json`, `after-restart.json` | +| Failover под записью | 71 подтверждение, 2 переходные ошибки, recovery 2.625 сек, без потери/дублей | `failover.txt` | +| Недоступность S3/DB | live=200, ready=503, mutation=503; сверка abort без ложной очистки | `s3-outage.txt`, `postgres-outage.txt`, `*-reconcile-abort.txt` | +| Обрыв/ошибка после PUT | TCP disconnect не создал metadata; отказ после PUT и реальный deferred COMMIT trigger создали обнаруживаемый orphan | `acceptance.txt`, `reconciliation.txt` | +| Private bucket / least privilege / bootstrap | Anonymous GET=403, API delete и PUT вне prefix запрещены, повтор bootstrap exit=0 | `acceptance.txt`, `bootstrap-repeat.txt` | +| ≥10 upload, RAM и таймауты | 12 клиента, 2 размера, 36 запросов на размер, 0 ошибок; RSS и kernel cgroup peak измерены | `upload-load.txt` | +| Завершённый трек 2 | dry-run/apply/repeat, grace, orphan/missing, restore SHA-256, fresh/used protection, cost ×3 | `reconciliation.txt` | -## Эксперименты и отказы +Добавлен CI `solution.yml`: build, публичные и свои проверки, сохранение evidence как artifact. Удалённый CI в этой работе не запускался. Контейнерные состояния до/после подтверждают scope restart hook, а не только его успешный exit. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Исследование согласованности — трек 2 -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Гипотеза:** общий lock на PUT→COMMIT и exclusive lock на каждое решение сверки вместе с grace удаляют orphan после ошибки БД, сохраняя подтверждённые и ещё сохраняемые файлы. Missing исправляется безопасным карантином без удаления metadata/ссылок, затем проверяемым восстановлением. -## Ограничения и следующие шаги +**Алгоритм:** S3 ListObjectsV2 и metadata keyset по ID, ограниченные страницы (default=100, тест=7). Для orphan exclusive lock охватывает повторную проверку отсутствия metadata и удаление; возраст подтверждается вторым HEAD. Существующая metadata защищает объект независимо от наличия ссылки в Message. Missing/corrupt headers создают `missing_observations`; второе наблюдение после grace меняет внутреннюю availability на missing, сохраняя Attachment и ссылки. Download и новые ссылки дают 503; metadata сохраняется. Возвращённый оригинал проверяется фактическим SHA-256 по bytes, затем снимается карантин. Dry-run не пишет даже observations. Ошибки сети, permission и БД прерывают проход, не считаются отсутствием объекта. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Каждое решение имеет свою транзакцию, проход не глобально атомарен. После abort безопасные уже выполненные действия могут сохраниться; повтор завершает работу. Несколько reconciler сериализуют решения одним lock. Авария API освобождает transaction lock, но orphan не удаляется до grace. Периодический sweeper предотвращает вечное накопление потерянных объектов. + +**Методика:** `student-tests/reconciliation.py` делает PUT без metadata, отказ тестового API после PUT и реальную PostgreSQL-ошибку на COMMIT через deferred trigger; далее dry-run, apply и повтор. Потом удаляется S3 оригинал, используемый сообщением; проверяются наблюдения до/после grace, сохранность ссылки, restore точных bytes и download. Затем 10 upload параллельно со сверкой и 3 cost-обхода неизменного набора. Операторские credentials есть только в test-runner. В тесте grace=2 сек, обычный стенд — **3600 сек**, период — 60 сек. Часы должны быть синхронизированы; короткий тестовый grace не используется в эксплуатации. + +**Результат:** dry-run нашёл 5 старых orphan, удалений 0, 1 свежий сохранён. Три orphan созданы текущими отказами, два остались от диагностических запусков. Apply удалил 5; repeat удалил 0, снова сохранив свежий. Когда свежий тестовый orphan действительно достиг grace, он был безопасно удалён последующим проходом. Первое missing-наблюдение не изменило availability; после grace 1 объект попал в карантин, message link сохранился. После restore проверены 111817 bytes и снят 1 карантин. Все **10** конкурентно сохранённых originals скачались без изменений. Используемый объект reconciler не удалял. + +Cost: 28 объектов/28 metadata, page=7, **34 S3 запроса / 146 SQL statements**, 1.0823 / 0.8517 / 0.8582 сек, среднее **0.9307 сек**, 0 скачанных bytes в обычном HEAD-проходе. SQL-счётчик включает явные запросы/locks, исключает BEGIN/COMMIT. Сложность O(objects+metadata), память ограничена страницей. Restore добавляет GET и O(size) checksum. Этот маленький локальный набор не доказывает скорость на миллионах объектов. + +Обычный dry-run, apply и наблюдение sweeper: + +```sh +docker compose --env-file .env --profile tools run --rm --no-deps reconciler +docker compose --env-file .env --profile tools run --rm --no-deps reconciler python -m src.reconcile --apply --grace-seconds 3600 +docker compose --env-file .env logs --tail 20 sweeper +``` + +Если bytes потеряны навсегда, из SHA-256 восстановить их нельзя. Оператор возвращает оригинал из backup под прежним ключом с правильными MIME/SHA metadata; apply проверяет фактические bytes и возвращает доступ. Без backup остаются metadata и честный 503, а не фиктивное исправление. + +## Нагрузка и ресурсы + +Intel Core i7-12700H, 14 физических/20 логических ядер, host RAM 16788811776 bytes. Docker Desktop 4.91.0 / Engine 29.8.0, WSL2 x86_64; VM: 20 CPU / 8129257472 bytes RAM. Каждая API — 1 CPU/1 GiB, DB — 1 CPU/384 MiB, MinIO — 1 CPU/512 MiB, LB — 0.5 CPU/128 MiB; sweeper — верхний предел 1 CPU/1 GiB. Raw hardware — `docker-environment.json`, `host-memory.json`, `final-containers.json`. + +Upload workload: синтетический несжимаемый RGB PNG, seed=42, barrier на **12** потоках, 3 запроса на поток, по одному предварительному upload каждого размера для прогрева. Клиент отдельный, его RAM не приписывается API. Samples читают RSS, CPU и cgroup memory.current; **memory.peak** ядра учитывает tmpfs и короткие пики между samples. Kernel peak накопительный со старта контейнера; сохранены значения до/после профиля, после нагрузки они выросли. Таблица — последний запуск с cgroup измерениями, после успешного полного verify. + +| Показатель | PNG 512×512 | PNG 1536×1536 | +| --- | ---: | ---: | +| Bytes | 788087 | 7089444 | +| Успех / ошибки | 36 / 0 | 36 / 0 | +| Длительность | 1.016 сек | 3.666 сек | +| Успешные RPS | 35.449 | 9.819 | +| p50 / p95 / p99 | 0.2585 / 0.5316 / 0.5417 сек | 1.1656 / 1.5725 / 1.5807 сек | +| Пиковый RSS api1 / api2 | 121.5 / 118.2 MiB | 156.3 / 161.7 MiB | +| Kernel peak api1 / api2 | 107.3 / 104.3 MiB | 177.6 / 184.5 MiB | +| CPU API суммарно | 1.74 сек | 6.78 сек | +| Средняя CPU двух API | 171.34% | 184.93% | +| Samples | 22 | 49 | + +100% CPU — одно ядро, две реплики могут суммарно достичь 200%. RSS и cgroup charge по-разному учитывают общие страницы, совпадать не должны. Второй профиль увеличил время decode, tmpfs и PUT; peak остался ниже 1 GiB. Подтверждён bounded расход на этих профилях, не production maximum и не worst case всех файлов до 40M pixels. Полный decode требует RAM пропорционально pixels; при перегрузке клиент обрабатывает 429. Прозрачного retry upload нет. + +Публичный load сообщений: 4 клиента, 15.062 сек, 840 подтверждений, 0 ошибок, **55.769 RPS**, p50/p95/p99=78/109/110 ms (`message-load.json`). Тесты добавляют данные, volumes сохранялись между запусками. Cost сверки измерен на 28 объектах до upload-профилей; один запуск upload-load добавляет 72 измеренных originals и 2 прогревочных. Closed-loop, 36 успешных запросов на размер и короткие серии ограничивают точность tail latency; coordinated omission, клиент и фоновые процессы влияют на цифры. Для production нужны длительные повторы и open-loop arrival rate. + +## Ограничения, источники и помощь ИИ + +Один host, LB, PostgreSQL и MinIO не дают HA при потере host/диска. Проверены отказы API, временная недоступность DB/S3, restart с volumes и рассогласование хранилищ. Backup, распределённый S3, TLS для удалённого deployment и HA БД — следующие шаги. Стенд локальный, данные синтетические. + +Retention невостребованных готовых Attachment не реализован: metadata защищает объект, его можно прикрепить позже. Истёкшие сессии/ключи не действуют, их фоновая очистка не исследовалась. Для здорового оригинала HEAD проверяет size и доверенную SHA metadata; полный bit-rot audit не выполняется. Restore всегда сверяет bytes. Presigned flow не выбран, обязательный proxy endpoint сохранён. + +Использованы Python, FastAPI/Starlette, Uvicorn, psycopg, boto3/botocore, Pillow, argon2-cffi, PostgreSQL, HAProxy и MinIO/mc. Приложение и свои сценарии написаны для лабораторной при помощи **OpenAI Codex**; публичные проверки взяты из skeleton. MinIO/mc — GNU AGPLv3, исходники доступны ниже. Памятка для понимания решения и защиты — `DEFENSE.md`. + +Первичные источники: [MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-04-22T22-12-26Z), [mc release](https://github.com/minio/mc/releases/tag/RELEASE.2025-04-16T18-13-26Z), [Pillow Image / pixel limits](https://pillow.readthedocs.io/en/stable/reference/Image.html), [botocore Config / таймауты](https://docs.aws.amazon.com/botocore/latest/reference/config.html), [S3 PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html), [PostgreSQL advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS). diff --git a/compose.test.yaml b/compose.test.yaml new file mode 100644 index 00000000..cae8780e --- /dev/null +++ b/compose.test.yaml @@ -0,0 +1,24 @@ +services: + api1: + environment: + ENABLE_TEST_METRICS: "1" + api2: + environment: + ENABLE_TEST_METRICS: "1" + test-runner: + image: mtusi-messenger:lab4 + init: true + profiles: [tests] + networks: [private] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [database_url, s3_access_key_id, s3_secret_access_key, minio_root_user, minio_root_password, reconcile_access_key_id, reconcile_secret_access_key] + command: [python, student-tests/acceptance.py] + depends_on: + lb: {condition: service_started} + fault-api: {condition: service_healthy} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e0d6dd71 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,158 @@ +name: mtusi-lab4 +x-app: &app + image: mtusi-messenger:lab4 + build: . + init: true + restart: unless-stopped + read_only: true + tmpfs: + - /tmp:size=67108864,mode=1777 + mem_limit: 1g + cpus: 1.0 + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + - cursor_secret + networks: [private] + depends_on: + migrate: {condition: service_completed_successfully} + bootstrap: {condition: service_completed_successfully} + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready',timeout=5)"] + interval: 5s + timeout: 6s + retries: 10 +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres-data:/var/lib/postgresql/data] + networks: [private] + mem_limit: 384m + cpus: 1.0 + restart: unless-stopped + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + s3: + image: mtusi-minio:RELEASE.2025-04-22T22-12-26Z + build: + context: . + dockerfile: infra/Dockerfile.minio + target: server + entrypoint: [/bin/sh, /config/minio-start.sh] + secrets: [minio_root_user, minio_root_password] + volumes: + - s3-data:/data + - ./infra/minio-start.sh:/config/minio-start.sh:ro + networks: [private] + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"] + interval: 3s + timeout: 3s + retries: 30 + bootstrap: + image: mtusi-mc:RELEASE.2025-04-16T18-13-26Z + build: + context: . + dockerfile: infra/Dockerfile.minio + target: client + entrypoint: [/bin/sh, /config/bootstrap.sh] + secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key, reconcile_access_key_id, reconcile_secret_access_key] + volumes: [./infra:/config:ro] + networks: [private] + depends_on: + s3: {condition: service_healthy} + migrate: + <<: *app + restart: "no" + command: [python, -m, scripts.migrate] + depends_on: + postgres: {condition: service_healthy} + healthcheck: {disable: true} + api1: + <<: *app + environment: + <<: *environment + INSTANCE_ID: api1 + api2: + <<: *app + environment: + <<: *environment + INSTANCE_ID: api2 + lb: + image: haproxy:3.0.11-alpine + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + ports: ["127.0.0.1:${LAB4_PORT:-8084}:8080"] + networks: [private, edge] + mem_limit: 128m + cpus: 0.5 + restart: unless-stopped + depends_on: + api1: {condition: service_healthy} + api2: {condition: service_healthy} + reconciler: + <<: *app + restart: "no" + command: [python, -m, src.reconcile] + profiles: [tools] + healthcheck: {disable: true} + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/reconcile_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/reconcile_secret_access_key + secrets: [database_url, reconcile_access_key_id, reconcile_secret_access_key] + sweeper: + <<: *app + command: [python, -m, src.sweep] + healthcheck: {disable: true} + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/reconcile_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/reconcile_secret_access_key + secrets: [database_url, reconcile_access_key_id, reconcile_secret_access_key] + fault-api: + <<: *app + restart: "no" + profiles: [tests] + environment: + <<: *environment + INSTANCE_ID: fault-api + FAULT_AFTER_PUT: "1" +networks: + private: + internal: true + edge: {} +volumes: + postgres-data: + s3-data: +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + minio_root_user: {file: ./secrets/minio_root_user} + minio_root_password: {file: ./secrets/minio_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + reconcile_access_key_id: {file: ./secrets/reconcile_access_key_id} + reconcile_secret_access_key: {file: ./secrets/reconcile_secret_access_key} + cursor_secret: {file: ./secrets/cursor_secret} diff --git a/evidence/acceptance.txt b/evidence/acceptance.txt new file mode 100644 index 00000000..d796ad35 --- /dev/null +++ b/evidence/acceptance.txt @@ -0,0 +1,3 @@ +{"status": "PASS", "negative_cases": 21, "jpeg_roundtrip": true, "idempotency_concurrency": 12, "idempotency_instances": 2, "private_bucket": true, "api_delete_denied": true, "prefix_write_denied": true, "interrupted_upload": true} + Container mtusi-lab4-test-runner-run-5cde5bcb0288 Creating + Container mtusi-lab4-test-runner-run-5cde5bcb0288 Created diff --git a/evidence/after-restart.json b/evidence/after-restart.json new file mode 100644 index 00000000..6e17228a --- /dev/null +++ b/evidence/after-restart.json @@ -0,0 +1,44 @@ +[ + { + "name": "/mtusi-lab4-api1-1", + "started_at": "2026-10-10T17:51:37.389990546Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-api2-1", + "started_at": "2026-10-10T17:51:37.270761082Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-lb-1", + "started_at": "2026-10-10T17:51:37.451731553Z", + "status": "running", + "memory_limit_bytes": "134217728", + "nano_cpus": "500000000" + }, + { + "name": "/mtusi-lab4-postgres-1", + "started_at": "2026-10-10T17:51:37.188764735Z", + "status": "running", + "memory_limit_bytes": "402653184", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-s3-1", + "started_at": "2026-10-10T17:51:37.059778931Z", + "status": "running", + "memory_limit_bytes": "536870912", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-sweeper-1", + "started_at": "2026-10-10T17:51:36.898129622Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + } +] \ No newline at end of file diff --git a/evidence/before-restart.json b/evidence/before-restart.json new file mode 100644 index 00000000..79848aff --- /dev/null +++ b/evidence/before-restart.json @@ -0,0 +1,44 @@ +[ + { + "name": "/mtusi-lab4-api1-1", + "started_at": "2026-10-10T17:51:23.809385841Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-api2-1", + "started_at": "2026-10-10T17:51:23.662770255Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-lb-1", + "started_at": "2026-10-10T17:50:54.716026892Z", + "status": "running", + "memory_limit_bytes": "134217728", + "nano_cpus": "500000000" + }, + { + "name": "/mtusi-lab4-postgres-1", + "started_at": "2026-10-10T17:39:45.444855934Z", + "status": "running", + "memory_limit_bytes": "402653184", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-s3-1", + "started_at": "2026-10-10T17:39:45.834387723Z", + "status": "running", + "memory_limit_bytes": "536870912", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-sweeper-1", + "started_at": "2026-10-10T17:51:23.96412061Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + } +] \ No newline at end of file diff --git a/evidence/bootstrap-repeat.txt b/evidence/bootstrap-repeat.txt new file mode 100644 index 00000000..83806fd6 --- /dev/null +++ b/evidence/bootstrap-repeat.txt @@ -0,0 +1,3 @@ +Private bucket and scoped users ready (idempotent bootstrap). + Container mtusi-lab4-bootstrap-run-2c835a1901b6 Creating + Container mtusi-lab4-bootstrap-run-2c835a1901b6 Created diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/compose-config.txt b/evidence/compose-config.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/docker-environment.json b/evidence/docker-environment.json new file mode 100644 index 00000000..ced0944f --- /dev/null +++ b/evidence/docker-environment.json @@ -0,0 +1,8 @@ +{ + "NCPU": 20, + "MemTotal": 8129257472, + "OperatingSystem": "Docker Desktop", + "ServerVersion": "29.8.0", + "Architecture": "x86_64", + "KernelVersion": "6.18.33.2-microsoft-standard-WSL2" +} \ No newline at end of file diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..a48a0ea0 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 71, "failed_attempts": 2, "first_recovery_seconds": 2.625} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-containers.json b/evidence/final-containers.json new file mode 100644 index 00000000..592e8761 --- /dev/null +++ b/evidence/final-containers.json @@ -0,0 +1,44 @@ +[ + { + "name": "/mtusi-lab4-api1-1", + "started_at": "2026-10-10T17:52:09.391507295Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-api2-1", + "started_at": "2026-10-10T17:51:59.67463385Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-lb-1", + "started_at": "2026-10-10T17:51:37.451731553Z", + "status": "running", + "memory_limit_bytes": "134217728", + "nano_cpus": "500000000" + }, + { + "name": "/mtusi-lab4-postgres-1", + "started_at": "2026-10-10T17:52:18.465074502Z", + "status": "running", + "memory_limit_bytes": "402653184", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-s3-1", + "started_at": "2026-10-10T17:52:13.959848687Z", + "status": "running", + "memory_limit_bytes": "536870912", + "nano_cpus": "1000000000" + }, + { + "name": "/mtusi-lab4-sweeper-1", + "started_at": "2026-10-10T17:51:36.898129622Z", + "status": "running", + "memory_limit_bytes": "1073741824", + "nano_cpus": "1000000000" + } +] \ No newline at end of file diff --git a/evidence/final-smoke.txt b/evidence/final-smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/final-smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/host-memory.json b/evidence/host-memory.json new file mode 100644 index 00000000..f55ed906 --- /dev/null +++ b/evidence/host-memory.json @@ -0,0 +1,3 @@ +{ + "TotalPhysicalMemory": 16788811776 +} diff --git a/evidence/integrity-and-secrets.txt b/evidence/integrity-and-secrets.txt new file mode 100644 index 00000000..0a464ce9 --- /dev/null +++ b/evidence/integrity-and-secrets.txt @@ -0,0 +1,2 @@ +PASS: mandatory README, COURSE, contracts and public tests unchanged (Git line endings normalized) +PASS: no local credential values in deliverables diff --git a/evidence/mc-version.txt b/evidence/mc-version.txt new file mode 100644 index 00000000..e16ab009 --- /dev/null +++ b/evidence/mc-version.txt @@ -0,0 +1,4 @@ +mc version DEVELOPMENT.2025-04-16T18-13-26Z (commit-id=b00526b153a31b36767991a4f5ce2cced435ee8e) +Runtime: go1.24.2 linux/amd64 +Copyright (c) 2015-2025 MinIO, Inc. +License GNU AGPLv3 diff --git a/evidence/message-load.json b/evidence/message-load.json new file mode 100644 index 00000000..7aae98a4 --- /dev/null +++ b/evidence/message-load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.062, + "concurrency": 4, + "successful_requests": 840, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 55.769, + "successful_latency_ms": { + "p50": 78.0, + "p95": 109.0, + "p99": 110.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/message-load.txt b/evidence/message-load.txt new file mode 100644 index 00000000..7aae98a4 --- /dev/null +++ b/evidence/message-load.txt @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.062, + "concurrency": 4, + "successful_requests": 840, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 55.769, + "successful_latency_ms": { + "p50": 78.0, + "p95": 109.0, + "p99": 110.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/minio-version.txt b/evidence/minio-version.txt new file mode 100644 index 00000000..b91ec8ba --- /dev/null +++ b/evidence/minio-version.txt @@ -0,0 +1,4 @@ +minio version DEVELOPMENT.2025-04-22T22-12-26Z (commit-id=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e) +Runtime: go1.24.2 linux/amd64 +License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html +Copyright: 2015-2025 MinIO, Inc. diff --git a/evidence/openapi-validation.txt b/evidence/openapi-validation.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi-validation.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/postgres-outage.txt b/evidence/postgres-outage.txt new file mode 100644 index 00000000..a5d3f454 --- /dev/null +++ b/evidence/postgres-outage.txt @@ -0,0 +1,3 @@ +{"status": "PASS", "dependency": "postgres", "live": 200, "ready": 503, "mutation": 503} + Container mtusi-lab4-test-runner-run-03d928cfa7aa Creating + Container mtusi-lab4-test-runner-run-03d928cfa7aa Created diff --git a/evidence/postgres-reconcile-abort.txt b/evidence/postgres-reconcile-abort.txt new file mode 100644 index 00000000..e72deb66 --- /dev/null +++ b/evidence/postgres-reconcile-abort.txt @@ -0,0 +1,4 @@ +{"status": "aborted", "error_type": "OperationalError"} + Container mtusi-lab4-reconciler-run-37dbe59d2e7b Creating + Container mtusi-lab4-reconciler-run-37dbe59d2e7b Created + diff --git a/evidence/production-containers.txt b/evidence/production-containers.txt new file mode 100644 index 00000000..7166b7a2 --- /dev/null +++ b/evidence/production-containers.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api1-1 mtusi-messenger:lab4 "python -m uvicorn s…" api1 12 seconds ago Up 9 seconds (healthy) +mtusi-lab4-api2-1 mtusi-messenger:lab4 "python -m uvicorn s…" api2 12 seconds ago Up 9 seconds (healthy) +mtusi-lab4-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 16 minutes ago Up 10 minutes 127.0.0.1:8084->8080/tcp +mtusi-lab4-postgres-1 postgres:17.6-alpine3.22 "docker-entrypoint.s…" postgres 22 minutes ago Up 9 minutes (healthy) 5432/tcp +mtusi-lab4-s3-1 mtusi-minio:RELEASE.2025-04-22T22-12-26Z "/bin/sh /config/min…" s3 22 minutes ago Up 9 minutes (healthy) +mtusi-lab4-sweeper-1 mtusi-messenger:lab4 "python -m src.sweep" sweeper 7 minutes ago Up 7 minutes diff --git a/evidence/production-smoke.txt b/evidence/production-smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/production-smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/reconciliation.txt b/evidence/reconciliation.txt new file mode 100644 index 00000000..c20e7c75 --- /dev/null +++ b/evidence/reconciliation.txt @@ -0,0 +1,3 @@ +{"apply": {"db_queries": 107, "deleted": 5, "duration_seconds": 0.6236, "fresh_orphans": 1, "grace_seconds": 2, "metadata_scanned": 18, "missing": 0, "mode": "apply", "objects_scanned": 24, "observation_started": 0, "orphan_candidates": 5, "quarantined": 0, "recovered": 0, "s3_requests": 33, "verified_bytes": 0}, "concurrent_scan": {"db_queries": 138, "deleted": 0, "duration_seconds": 0.9545, "fresh_orphans": 0, "grace_seconds": 2, "metadata_scanned": 28, "missing": 0, "mode": "apply", "objects_scanned": 24, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 33, "verified_bytes": 0}, "concurrent_uploads_preserved": 10, "cost_repeats": [{"db_queries": 146, "deleted": 0, "duration_seconds": 1.0823, "fresh_orphans": 0, "grace_seconds": 3600, "metadata_scanned": 28, "missing": 0, "mode": "dry-run", "objects_scanned": 28, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 34, "verified_bytes": 0}, {"db_queries": 146, "deleted": 0, "duration_seconds": 0.8517, "fresh_orphans": 0, "grace_seconds": 3600, "metadata_scanned": 28, "missing": 0, "mode": "dry-run", "objects_scanned": 28, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 34, "verified_bytes": 0}, {"db_queries": 146, "deleted": 0, "duration_seconds": 0.8582, "fresh_orphans": 0, "grace_seconds": 3600, "metadata_scanned": 28, "missing": 0, "mode": "dry-run", "objects_scanned": 28, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 34, "verified_bytes": 0}], "dry_run": {"db_queries": 107, "deleted": 0, "duration_seconds": 0.5808, "fresh_orphans": 1, "grace_seconds": 2, "metadata_scanned": 18, "missing": 0, "mode": "dry-run", "objects_scanned": 24, "observation_started": 0, "orphan_candidates": 5, "quarantined": 0, "recovered": 0, "s3_requests": 23, "verified_bytes": 0}, "first_missing_observation": {"db_queries": 96, "deleted": 1, "duration_seconds": 0.5301, "fresh_orphans": 0, "grace_seconds": 2, "metadata_scanned": 18, "missing": 1, "mode": "apply", "objects_scanned": 18, "observation_started": 1, "orphan_candidates": 1, "quarantined": 0, "recovered": 0, "s3_requests": 24, "verified_bytes": 0}, "grace_for_test_seconds": 2, "missing_dry_run": {"db_queries": 95, "deleted": 0, "duration_seconds": 0.5196, "fresh_orphans": 1, "grace_seconds": 2, "metadata_scanned": 18, "missing": 1, "mode": "dry-run", "objects_scanned": 18, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 22, "verified_bytes": 0}, "quarantine": {"db_queries": 94, "deleted": 0, "duration_seconds": 0.5602, "fresh_orphans": 0, "grace_seconds": 2, "metadata_scanned": 18, "missing": 1, "mode": "apply", "objects_scanned": 17, "observation_started": 0, "orphan_candidates": 0, "quarantined": 1, "recovered": 0, "s3_requests": 22, "verified_bytes": 0}, "repeat": {"db_queries": 97, "deleted": 0, "duration_seconds": 0.5334, "fresh_orphans": 1, "grace_seconds": 2, "metadata_scanned": 18, "missing": 0, "mode": "apply", "objects_scanned": 19, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 0, "s3_requests": 22, "verified_bytes": 0}, "restore": {"db_queries": 97, "deleted": 0, "duration_seconds": 0.5205, "fresh_orphans": 0, "grace_seconds": 2, "metadata_scanned": 18, "missing": 0, "mode": "apply", "objects_scanned": 18, "observation_started": 0, "orphan_candidates": 0, "quarantined": 0, "recovered": 1, "s3_requests": 23, "verified_bytes": 111817}, "status": "PASS"} + Container mtusi-lab4-test-runner-run-d9df868338b3 Creating + Container mtusi-lab4-test-runner-run-d9df868338b3 Created diff --git a/evidence/restore-api.txt b/evidence/restore-api.txt new file mode 100644 index 00000000..ec1140e3 --- /dev/null +++ b/evidence/restore-api.txt @@ -0,0 +1,18 @@ + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-bootstrap-1 Starting + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-bootstrap-1 Started + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-api2-1 Starting + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-api2-1 Started diff --git a/evidence/restore-postgres.txt b/evidence/restore-postgres.txt new file mode 100644 index 00000000..bd47b9e8 --- /dev/null +++ b/evidence/restore-postgres.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-postgres-1 Starting + Container mtusi-lab4-postgres-1 Started diff --git a/evidence/restore-s3.txt b/evidence/restore-s3.txt new file mode 100644 index 00000000..510bc876 --- /dev/null +++ b/evidence/restore-s3.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-s3-1 Starting + Container mtusi-lab4-s3-1 Started diff --git a/evidence/restore-upload-replica.txt b/evidence/restore-upload-replica.txt new file mode 100644 index 00000000..82a21b3e --- /dev/null +++ b/evidence/restore-upload-replica.txt @@ -0,0 +1,14 @@ + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-bootstrap-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-bootstrap-1 Started + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-api1-1 Starting + Container mtusi-lab4-api1-1 Started diff --git a/evidence/s3-outage.txt b/evidence/s3-outage.txt new file mode 100644 index 00000000..9184a548 --- /dev/null +++ b/evidence/s3-outage.txt @@ -0,0 +1,3 @@ +{"status": "PASS", "dependency": "s3", "live": 200, "ready": 503, "mutation": 503} + Container mtusi-lab4-test-runner-run-12654d5a1c8a Creating + Container mtusi-lab4-test-runner-run-12654d5a1c8a Created diff --git a/evidence/s3-reconcile-abort.txt b/evidence/s3-reconcile-abort.txt new file mode 100644 index 00000000..e02cab93 --- /dev/null +++ b/evidence/s3-reconcile-abort.txt @@ -0,0 +1,4 @@ +{"status": "aborted", "error_type": "EndpointConnectionError"} + Container mtusi-lab4-reconciler-run-4d5c0f90543f Creating + Container mtusi-lab4-reconciler-run-4d5c0f90543f Created + diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stop-fault-api.txt b/evidence/stop-fault-api.txt new file mode 100644 index 00000000..1d836a3c --- /dev/null +++ b/evidence/stop-fault-api.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-fault-api-1 Stopping + Container mtusi-lab4-fault-api-1 Stopped diff --git a/evidence/stop-postgres.txt b/evidence/stop-postgres.txt new file mode 100644 index 00000000..b818c494 --- /dev/null +++ b/evidence/stop-postgres.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-postgres-1 Stopping + Container mtusi-lab4-postgres-1 Stopped diff --git a/evidence/stop-s3.txt b/evidence/stop-s3.txt new file mode 100644 index 00000000..957bce2f --- /dev/null +++ b/evidence/stop-s3.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-s3-1 Stopping + Container mtusi-lab4-s3-1 Stopped diff --git a/evidence/stop-upload-replica.txt b/evidence/stop-upload-replica.txt new file mode 100644 index 00000000..1be27d0d --- /dev/null +++ b/evidence/stop-upload-replica.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-api1-1 Stopping + Container mtusi-lab4-api1-1 Stopped diff --git a/evidence/test-services.txt b/evidence/test-services.txt new file mode 100644 index 00000000..ce53d779 --- /dev/null +++ b/evidence/test-services.txt @@ -0,0 +1,49 @@ + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-s3-1 Running + Container mtusi-lab4-lb-1 Running + Container mtusi-lab4-migrate-1 Recreate + Container mtusi-lab4-migrate-1 Recreated + Container mtusi-lab4-api2-1 Recreate + Container mtusi-lab4-fault-api-1 Recreate + Container mtusi-lab4-api1-1 Recreate + Container mtusi-lab4-sweeper-1 Recreate + Container mtusi-lab4-sweeper-1 Recreated + Container mtusi-lab4-api1-1 Recreated + Container mtusi-lab4-api2-1 Recreated + Container mtusi-lab4-fault-api-1 Recreated + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-bootstrap-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-bootstrap-1 Started + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-fault-api-1 Starting + Container mtusi-lab4-api2-1 Starting + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-api1-1 Starting + Container mtusi-lab4-bootstrap-1 Exited + Container mtusi-lab4-sweeper-1 Starting + Container mtusi-lab4-fault-api-1 Started + Container mtusi-lab4-api2-1 Started + Container mtusi-lab4-api1-1 Started + Container mtusi-lab4-api1-1 Waiting + Container mtusi-lab4-api2-1 Waiting + Container mtusi-lab4-sweeper-1 Started + Container mtusi-lab4-api2-1 Healthy + Container mtusi-lab4-api1-1 Healthy diff --git a/evidence/upload-failover.json b/evidence/upload-failover.json new file mode 100644 index 00000000..89e609e4 --- /dev/null +++ b/evidence/upload-failover.json @@ -0,0 +1 @@ +{"status": "PASS", "upload_instance": "api1", "download_instance": "api2", "checksum_matches": true} \ No newline at end of file diff --git a/evidence/upload-load-runner.txt b/evidence/upload-load-runner.txt new file mode 100644 index 00000000..89f560df --- /dev/null +++ b/evidence/upload-load-runner.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-test-runner-run-3e3c9450a44e Creating + Container mtusi-lab4-test-runner-run-3e3c9450a44e Created diff --git a/evidence/upload-load.txt b/evidence/upload-load.txt new file mode 100644 index 00000000..83e7de67 --- /dev/null +++ b/evidence/upload-load.txt @@ -0,0 +1 @@ +{"status": "PASS", "profiles": [{"size_bytes": 788087, "dimensions": [512, 512], "concurrency": 12, "uploads": 36, "successful_rps": 35.449, "duration_seconds": 1.016, "errors": 0, "p50_seconds": 0.2585, "p95_seconds": 0.5316, "p99_seconds": 0.5417, "baseline_rss_bytes": [112431104, 113086464], "peak_rss_bytes": [127414272, 123965440], "cpu_seconds_total": 1.74, "baseline_container_memory_bytes": [92622848, 93675520], "sampled_peak_container_memory_bytes": [111976448, 108802048], "kernel_container_peak_before_bytes": [93327360, 94830592], "kernel_container_peak_after_bytes": [112476160, 109318144], "cpu_average_percent_total": 171.34, "memory_samples": 22}, {"size_bytes": 7089444, "dimensions": [1536, 1536], "concurrency": 12, "uploads": 36, "successful_rps": 9.819, "duration_seconds": 3.666, "errors": 0, "p50_seconds": 1.1656, "p95_seconds": 1.5725, "p99_seconds": 1.5807, "baseline_rss_bytes": [135786496, 123965440], "peak_rss_bytes": [163864576, 169558016], "cpu_seconds_total": 6.78, "baseline_container_memory_bytes": [116375552, 105029632], "sampled_peak_container_memory_bytes": [185982976, 193187840], "kernel_container_peak_before_bytes": [124903424, 109318144], "kernel_container_peak_after_bytes": [186245120, 193449984], "cpu_average_percent_total": 184.93, "memory_samples": 49}]} diff --git a/evidence/verification.json b/evidence/verification.json new file mode 100644 index 00000000..04812bab --- /dev/null +++ b/evidence/verification.json @@ -0,0 +1,7 @@ +{ + "status": "PASS", + "completed_at": "2026-10-10T17:53:03.311454+00:00", + "load_experiments": true, + "project": "mtusi-lab4", + "base_url": "http://127.0.0.1:8084" +} \ No newline at end of file diff --git a/infra/Dockerfile.minio b/infra/Dockerfile.minio new file mode 100644 index 00000000..3800cae0 --- /dev/null +++ b/infra/Dockerfile.minio @@ -0,0 +1,19 @@ +FROM golang:1.24.2-bookworm AS server-build +RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /source +WORKDIR /source +RUN CGO_ENABLED=0 go build -trimpath -ldflags="$(go run buildscripts/gen-ldflags.go)" -o /out/minio . + +FROM golang:1.24.2-bookworm AS client-build +RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /source +WORKDIR /source +RUN CGO_ENABLED=0 go build -trimpath -ldflags="$(go run buildscripts/gen-ldflags.go)" -o /out/mc . + +FROM python:3.12.11-slim-bookworm AS server +COPY --from=server-build /out/minio /usr/local/bin/minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z" +ENTRYPOINT ["minio"] + +FROM python:3.12.11-slim-bookworm AS client +COPY --from=client-build /out/mc /usr/local/bin/mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z" +ENTRYPOINT ["mc"] diff --git a/infra/api-policy.json b/infra/api-policy.json new file mode 100644 index 00000000..b9ea5a6d --- /dev/null +++ b/infra/api-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]}]} diff --git a/infra/bootstrap.sh b/infra/bootstrap.sh new file mode 100644 index 00000000..bf071890 --- /dev/null +++ b/infra/bootstrap.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +mc alias set local http://s3:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-api /config/api-policy.json >/dev/null +mc admin policy create local messenger-reconciler /config/reconcile-policy.json >/dev/null +mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null +mc admin user add local "$(cat /run/secrets/reconcile_access_key_id)" "$(cat /run/secrets/reconcile_secret_access_key)" >/dev/null +mc admin policy attach local messenger-reconciler --user "$(cat /run/secrets/reconcile_access_key_id)" >/dev/null +echo 'Private bucket and scoped users ready (idempotent bootstrap).' diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..812a17a7 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + maxconn 256 +defaults + mode http + timeout connect 3s + timeout client 35s + timeout server 35s +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s +frontend public + bind *:8080 + http-request deny if { path_beg /internal-test/ } + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + retries 2 + option redispatch + server api1 api1:8080 check inter 1s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api2 api2:8080 check inter 1s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/infra/minio-start.sh b/infra/minio-start.sh new file mode 100644 index 00000000..a8dd079a --- /dev/null +++ b/infra/minio-start.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/infra/reconcile-policy.json b/infra/reconcile-policy.json new file mode 100644 index 00000000..a476051d --- /dev/null +++ b/infra/reconcile-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]}]} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..fdbabc37 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, username text UNIQUE NOT NULL, display_name text NOT NULL, + password_hash text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, user_id uuid NOT NULL REFERENCES users(id), expires_at timestamptz NOT NULL +); +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, title text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS members ( + chat_id uuid REFERENCES chats(id), user_id uuid REFERENCES users(id), PRIMARY KEY(chat_id,user_id) +); +CREATE INDEX IF NOT EXISTS members_user_idx ON members(user_id,chat_id); +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id), owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, object_key text UNIQUE NOT NULL, content_type text NOT NULL, + size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), sha256 text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + availability text NOT NULL DEFAULT 'present' CHECK(availability IN ('present','missing')) +); +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id), sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS messages_order_idx ON messages(chat_id,created_at,id); +CREATE INDEX IF NOT EXISTS chats_order_idx ON chats(created_at,id); +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid REFERENCES messages(id), attachment_id uuid REFERENCES attachments(id), + position integer NOT NULL, PRIMARY KEY(message_id,attachment_id), UNIQUE(message_id,position) +); +CREATE TABLE IF NOT EXISTS idempotency ( + user_id uuid REFERENCES users(id), chat_id uuid REFERENCES chats(id), key text NOT NULL, + body_hash text NOT NULL, message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, PRIMARY KEY(user_id,chat_id,key) +); +CREATE TABLE IF NOT EXISTS missing_observations ( + attachment_id uuid PRIMARY KEY REFERENCES attachments(id), first_seen timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..91e5b881 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,26 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +boto3==1.38.46 +botocore==1.38.46 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +pillow==11.3.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +pycparser==3.11 +pydantic==2.14.0 +pydantic_core==2.50.0 +python-dateutil==2.9.0.post0 +s3transfer==0.13.1 +six==1.17.0 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7c96b6a2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +boto3==1.38.46 +Pillow==11.3.0 +argon2-cffi==25.1.0 diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..f182d25e --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,7 @@ +from pathlib import Path +from src.storage import db + +with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(440001)') + conn.execute(Path('migrations/001_initial.sql').read_text()) +print('Schema ready (idempotent migration).') diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..32720afb --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose --env-file .env restart postgres s3 api1 api2 lb sweeper diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..a59b824e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres s3 api1 api2 lb sweeper diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..c7d50dc2 --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,21 @@ +"""Generate local random secrets inside lab4 only; preserve existing files.""" +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(exist_ok=True) +values = {} +for name in ('postgres_password','minio_root_user','minio_root_password','s3_access_key_id', + 's3_secret_access_key','reconcile_access_key_id','reconcile_secret_access_key','cursor_secret'): + target = folder / name + if not target.exists(): + target.write_text(secrets.token_hex(24),encoding='utf-8') + values[name] = target.read_text().strip() +target = folder / 'database_url' +if not target.exists(): + target.write_text('postgresql://messenger:'+values['postgres_password']+'@postgres:5432/messenger',encoding='utf-8') +env = root / '.env' +if not env.exists(): + env.write_text('LAB4_PORT=8084\nSESSION_TTL_SECONDS=3600\n',encoding='utf-8') +print('Local lab4 secrets prepared; existing files preserved. No secret values displayed.') diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..ca080859 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api1" goto stop +if "%TARGET_INSTANCE_ID%"=="api2" goto stop +exit /b 2 +:stop +docker compose --env-file .env stop -t 2 %TARGET_INSTANCE_ID% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..e81ed6fd --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:-}" in api1|api2) ;; *) exit 2 ;; esac +docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID" diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..50ab2912 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,120 @@ +"""Run the complete lab4 demonstration. Only Compose project mtusi-lab4 is mutated. + +Requires the built stack. Results contain no session tokens or credential values. +Use --quick to omit load experiments. Never removes volumes. +""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, check + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--quick',action='store_true') + args = parser.parse_args() + os.chdir(ROOT) + os.environ['PYTHONUTF8'] = '1' + env_text = (ROOT/'.env').read_text() + port = next((line.split('=',1)[1] for line in env_text.splitlines() if line.startswith('LAB4_PORT=')),'8084') + base = 'http://127.0.0.1:'+port + compose = ['docker','compose','--env-file','.env','-f','compose.yaml','-f','compose.test.yaml','--profile','tests'] + evidence = ROOT/'evidence' + evidence.mkdir(exist_ok=True) + def run(name, command, expected=0): + print('Running '+name+' ...',flush=True) + result = subprocess.run(command,capture_output=True,text=True,encoding='utf-8',errors='replace') + (evidence/(name+'.txt')).write_text(result.stdout+result.stderr,encoding='utf-8') + if result.returncode != expected: + print(result.stdout+result.stderr) + raise RuntimeError(name+' failed, exit='+str(result.returncode)) + return result.stdout.strip() + def ready(): + deadline = time.monotonic()+90 + while time.monotonic()clock_timestamp()', (token_hash,)).fetchone() + if not row: + fail(401) + return row, token_hash + + +def access(conn, uid, cid): + row = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s', (cid,uid)).fetchone() + if not row: + fail(404) + return row + + +def authorize_upload(authorization, cid): + with db() as conn: + actor, _ = authenticate(conn, authorization) + cid = ident(cid) + access(conn, actor['id'], cid) + return str(actor['id']), cid + + +def query(request, allowed): + if set(request.query_params) - set(allowed) or any(len(request.query_params.getlist(k)) != 1 for k in request.query_params): + fail(400) + return dict(request.query_params) + + +def page(conn, kind, resource, actor, params): + raw = params.get('limit','50') + if len(raw)>3 or not re.fullmatch(r'[0-9]+', raw) or not 1 <= int(raw) <= 100: + fail(400) + limit = int(raw) + boundary = dt.datetime.min.replace(tzinfo=dt.timezone.utc) + bid = '00000000-0000-0000-0000-000000000000' + scope = kind + ':' + resource + if 'cursor' in params: + if len(params['cursor'])>2048: + fail(400) + try: + encoded, signature = params['cursor'].split('.') + expected = hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + fail(400) + payload = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4))) + if payload['scope'] != scope: + fail(400) + boundary = dt.datetime.fromisoformat(payload['time']) + bid = ident(payload['id']) + except (ValueError, KeyError, TypeError): + fail(400) + if kind == 'chats': + rows = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s::uuid) ORDER BY c.created_at,c.id LIMIT %s', (actor,boundary,bid,limit+1)).fetchall() + items = [chat(conn,x) for x in rows[:limit]] + else: + rows = conn.execute('SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s::uuid) ORDER BY created_at,id LIMIT %s', (resource,boundary,bid,limit+1)).fetchall() + items = [message(conn,x) for x in rows[:limit]] + cursor = None + if len(rows) > limit: + last = rows[limit-1] + encoded = base64.urlsafe_b64encode(json.dumps({'scope':scope,'time':stamp(last['created_at']),'id':str(last['id'])}, sort_keys=True).encode()).decode().rstrip('=') + cursor = encoded + '.' + hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest() + return {'items':items, 'next_cursor':cursor} + + +@asynccontextmanager +async def lifespan(app): + app.state.upload_slots = asyncio.Semaphore(4) + app.state.json_slots = asyncio.Semaphore(16) + yield + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +def error_response(request, status): + headers = {} + if status == 401: + headers['WWW-Authenticate'] = ('Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + return JSONResponse({'error':{'code':CODES[status], 'message':MESSAGES[status], + 'request_id':request.state.request_id}}, status, headers=headers) + + +@app.middleware('http') +async def errors_and_identity(request, call_next): + request.state.request_id = uuid.uuid4().hex + try: + result = await call_next(request) + except APIError as exc: + result = error_response(request, exc.status) + except (psycopg.Error, BotoCoreError, ClientError) as exc: + code = exc.response.get('Error',{}).get('Code','') if isinstance(exc,ClientError) else getattr(exc,'sqlstate','') + logging.warning('Storage failure: %s code=%s',type(exc).__name__,code) + result = error_response(request, 503) + except Exception: + result = error_response(request, 500) + result.headers['X-Request-Id'] = request.state.request_id + result.headers['X-Instance-Id'] = INSTANCE + return result + + +@app.exception_handler(APIError) +async def api_error(request, exc): + return error_response(request, exc.status) + + +@app.exception_handler(HTTPException) +async def http_error(request, exc): + return error_response(request, 404 if exc.status_code == 404 else 400) + + +@app.get('/health/live') +def live(): + return {'status':'ok'} + + +@app.get('/internal-test/memory') +def test_memory(): + if os.getenv('ENABLE_TEST_METRICS') != '1': + fail(404) + with open('/proc/self/status') as status: + rss = next(int(line.split()[1])*1024 for line in status if line.startswith('VmRSS:')) + usage = os.times() + with open('/sys/fs/cgroup/memory.current') as current: + container_memory = int(current.read()) + with open('/sys/fs/cgroup/memory.peak') as peak: + container_peak = int(peak.read()) + return {'rss_bytes':rss, 'cpu_seconds':usage.user+usage.system, + 'container_memory_bytes':container_memory, 'container_peak_bytes':container_peak} + + +@app.get('/health/ready') +def ready(): + with db() as conn: + conn.execute('SELECT 1 FROM attachments LIMIT 1') + s3().head_bucket(Bucket=BUCKET) + return {'status':'ok'} + + +def validate_image(file, declared): + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + file.seek(0) + with Image.open(file) as im: + if im.format not in ('PNG','JPEG') or Image.MIME[im.format] != declared: + fail(415) + if im.width * im.height > MAX_PIXELS: + fail(413) + if getattr(im,'n_frames',1) != 1: + fail(415) + im.verify() + file.seek(0) + with Image.open(file) as im: + im.load() # JPEG verify() alone does not detect truncation. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + fail(413) + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + fail(415) + file.seek(0) + + +def persist_upload(file, authorization, cid, filename, declared, size, sha): + validate_image(file, declared) + client = s3() + aid = str(uuid.uuid4()) + key = PREFIX + aid + # Shared transaction lock covers PUT -> COMMIT; reconciler takes the exclusive lock. + # If COMMIT outcome is ambiguous, never delete here: the reconciler checks the committed DB. + with db() as conn: + actor, _ = authenticate(conn, authorization) + access(conn, actor['id'], cid) + conn.execute('SELECT pg_advisory_xact_lock_shared(%s)', (STORAGE_LOCK,)) + md5 = base64.b64encode(hashlib.file_digest(file, 'md5').digest()).decode() + file.seek(0) + client.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size, + ContentType=declared, Metadata={'sha256':sha}, + ContentMD5=md5) + # Test-only fault injection is disabled in the regular Compose configuration. + if os.getenv('FAULT_AFTER_PUT') == '1': + fail(503) + row = conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (aid,cid,actor['id'],filename,key,declared,size,sha)).fetchone() + result = attachment(row) + return result + + +@app.post('/api/v1/chats/{cid}/attachments') +async def upload(request: Request, cid: str): + authorization = request.headers.get('authorization','') + _, cid = await run_in_threadpool(authorize_upload, authorization, cid) + params = query(request, ('filename',)) + filename = string(params.get('filename'),1,255) + if any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename): + fail(400) + declared = request.headers.get('content-type','').lower() + if declared not in ('image/png','image/jpeg'): + fail(415) + length = request.headers.get('content-length') + if length is not None: + if len(length)>10 or not re.fullmatch('[0-9]+', length): + fail(400) + if int(length) > MAX_BYTES: + fail(413) + try: + await asyncio.wait_for(request.app.state.upload_slots.acquire(), timeout=1) + except TimeoutError: + fail(429) + try: + with tempfile.TemporaryFile() as file: # ephemeral bounded staging, never required after 201 + digest = hashlib.sha256() + size = 0 + async def receive(): + nonlocal size + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + fail(413) + digest.update(chunk) + file.write(chunk) + try: + await asyncio.wait_for(receive(), timeout=30) + except (TimeoutError, ClientDisconnect): + fail(400) + if not size or (length is not None and size != int(length)): + fail(415 if not size else 400) + file.seek(0) + # file_digest leaves position at EOF; compute MD5 without moving the PUT body. + result = await run_in_threadpool(persist_upload, file, authorization, cid, filename, declared, size, digest.hexdigest()) + return JSONResponse(result,201) + finally: + request.app.state.upload_slots.release() + + +def dispatch(method, path, authorization, data, params, headers): + with db() as conn: + if path == '/api/v1/users' and method == 'POST': + value = strict_json(data, ('username','display_name','password')) + name = string(value['username'],3,32) + if not re.fullmatch('[a-z0-9_]{3,32}',name): + fail(400) + display = string(value['display_name'],1,100) + password = string(value['password'],12,128,False) + try: + row = conn.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING *', + (str(uuid.uuid4()),name,display,PH.hash(password))).fetchone() + except psycopg.errors.UniqueViolation: + fail(409) + return 201, user(row) + if path == '/api/v1/auth/sessions' and method == 'POST': + if data or not authorization.startswith('Basic '): + fail(401) + try: + name,password = base64.b64decode(authorization[6:],validate=True).decode('utf-8').split(':',1) + except (ValueError, UnicodeError): + fail(401) + row = conn.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone() + try: + PH.verify(row['password_hash'] if row else DUMMY_HASH, password) + except VerificationError: + fail(401) + if not row: + fail(401) + token = secrets.token_urlsafe(32) + expires = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=int(os.getenv('SESSION_TTL_SECONDS','3600'))) + conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)',(hashlib.sha256(token.encode()).hexdigest(),row['id'],expires)) + return 201, {'token':token,'token_type':'Bearer','expires_at':stamp(expires),'user':user(row)} + actor, token_hash = authenticate(conn, authorization) + uid = actor['id'] + if path == '/api/v1/users/me' and method == 'GET': + return 200, user(actor) + if path == '/api/v1/auth/sessions/current' and method == 'DELETE': + conn.execute('DELETE FROM sessions WHERE token_hash=%s',(token_hash,)) + return 204, None + if path == '/api/v1/chats': + if method == 'POST': + value = strict_json(data,('title','member_ids')) + title = string(value['title'],1,100) + members = ids(value['member_ids'],100,1) + if str(uid) not in members or conn.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(members,)).fetchone()['n'] != len(members): + fail(400) + row = conn.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *',(str(uuid.uuid4()),title)).fetchone() + for member in members: + conn.execute('INSERT INTO members VALUES(%s,%s)',(row['id'],member)) + return 201, chat(conn,row) + if method == 'GET': + return 200, page(conn,'chats',str(uid),uid,params) + match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages)?', path) + if match: + cid = ident(match[1]) + current_chat = access(conn,uid,cid) + if not match[2] and method == 'GET': + return 200, chat(conn,current_chat) + if match[2] and method == 'GET': + return 200, page(conn,'messages',cid,uid,params) + if match[2] and method == 'POST': + value = strict_json(data,('text',),('attachment_ids',)) + text = string(value['text'],0,4000,False) + attachments = ids(value.get('attachment_ids',[]),10) + if not text.strip() and not attachments: + fail(400) + key = headers.get('idempotency-key','') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key): + fail(400) + canonical = json.dumps({'text':text,'attachment_ids':attachments},sort_keys=True,ensure_ascii=False) + body_hash = hashlib.sha256(canonical.encode()).hexdigest() + # One transaction lock per user/chat/key, shared across API replicas. + lock = int.from_bytes(hashlib.sha256((str(uid)+cid+key).encode()).digest()[:8],'big',signed=True) + conn.execute('SELECT pg_advisory_xact_lock(%s)',(lock,)) + for aid in attachments: + a = conn.execute('SELECT * FROM attachments WHERE id=%s AND chat_id=%s',(aid,cid)).fetchone() + if not a: + fail(404) + if a['availability'] != 'present': + fail(503) + old = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()', (uid,cid,key)).fetchone() + if old: + if old['body_hash'] != body_hash: + fail(409) + return 201, message(conn,conn.execute('SELECT * FROM messages WHERE id=%s',(old['message_id'],)).fetchone()) + conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s',(uid,cid,key)) + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES(%s,%s,%s,%s) RETURNING *',(str(uuid.uuid4()),cid,uid,text)).fetchone() + for pos,aid in enumerate(attachments): + conn.execute('INSERT INTO message_attachments VALUES(%s,%s,%s)',(row['id'],aid,pos)) + conn.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(uid,cid,key,body_hash,row['id'])) + return 201, message(conn,row) + match = re.fullmatch(r'/api/v1/attachments/([^/]+)(/content)?',path) + if match and method == 'GET': + aid = ident(match[1]) + row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,uid)).fetchone() + if not row: + fail(404) + if not match[2]: + return 200, attachment(row) + if params.get('variant','original') != 'original': + fail(404) + if row['availability'] != 'present': + fail(503) + content = s3().get_object(Bucket=BUCKET,Key=row['object_key']) + body = content['Body'] + def stream(): + try: + while chunk := body.read(65536): + yield chunk + finally: + body.close() + return 200, StreamingResponse(stream(),media_type=row['content_type'], + headers={'Content-Length':str(row['size_bytes']), 'X-Content-Type-Options':'nosniff'}) + fail(404) + + +@app.api_route('/api/v1/{path:path}', methods=['GET','POST','DELETE','PUT','PATCH']) +async def api(request: Request, path: str): + if not (path == 'users' and request.method == 'POST') and not (path == 'auth/sessions' and request.method == 'POST'): + def check_auth(): + with db() as conn: + authenticate(conn,request.headers.get('authorization','')) + await run_in_threadpool(check_auth) + params = query(request, ('limit','cursor') if request.method == 'GET' and (path == 'chats' or path.endswith('/messages')) else ('variant',) if path.endswith('/content') else ()) + try: + await asyncio.wait_for(request.app.state.json_slots.acquire(),timeout=1) + except TimeoutError: + fail(429) + try: + data = bytearray() + async def receive(): + async for chunk in request.stream(): + data.extend(chunk) + if len(data) > 65536: + fail(400) + try: + await asyncio.wait_for(receive(),timeout=10) + except (TimeoutError, ClientDisconnect): + fail(400) + # Authentication for protected routes is performed before parsing JSON in dispatch. + status, result = await run_in_threadpool(dispatch, request.method, '/api/v1/'+path, + request.headers.get('authorization',''),bytes(data),params,dict(request.headers)) + if isinstance(result, Response): + return result + return Response(status_code=204) if status == 204 else JSONResponse(result,status) + finally: + request.app.state.json_slots.release() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..6543b834 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,146 @@ +"""Bounded, repeatable S3/DB reconciliation; dry-run is the default. + +Never deletes Attachment rows, linked or unlinked. Missing/corrupt originals are +quarantined after two observations separated by grace; recovery requires exact +SHA-256. Orphans can be removed only after grace and a DB recheck under the same +exclusive advisory lock used by upload. Storage failures abort rather than being +misclassified as absence. Each decision commits separately and is idempotent. +""" +import argparse +import datetime as dt +import hashlib +import json +import time +from botocore.exceptions import ClientError +from .storage import BUCKET, PREFIX, STORAGE_LOCK, db, s3 + + +def missing(exc): + return exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound') + + +def reconcile(apply=False, grace=3600, page_size=100): + started = time.monotonic() + result = dict(mode='apply' if apply else 'dry-run', grace_seconds=grace, + objects_scanned=0, metadata_scanned=0, s3_requests=0, db_queries=0, + orphan_candidates=0, deleted=0, fresh_orphans=0, missing=0, + quarantined=0, observation_started=0, recovered=0, verified_bytes=0) + client = s3() + # Complete both dependencies' readiness before any mutation. + client.head_bucket(Bucket=BUCKET) + result['s3_requests'] += 1 + with db() as conn: + conn.execute('SELECT 1 FROM attachments LIMIT 1') + result['db_queries'] += 1 + continuation = None + while True: + args = dict(Bucket=BUCKET, Prefix=PREFIX, MaxKeys=page_size) + if continuation: + args['ContinuationToken'] = continuation + response = client.list_objects_v2(**args) + result['s3_requests'] += 1 + for obj in response.get('Contents',[]): + result['objects_scanned'] += 1 + with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(%s)',(STORAGE_LOCK,)) + exists = conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(obj['Key'],)).fetchone() + result['db_queries'] += 2 + if exists: + continue + age = (dt.datetime.now(dt.timezone.utc)-obj['LastModified']).total_seconds() + if age < grace: + result['fresh_orphans'] += 1 + continue + result['orphan_candidates'] += 1 + if apply: + # HEAD recheck prevents deleting an object replaced since LIST. + head = client.head_object(Bucket=BUCKET,Key=obj['Key']) + result['s3_requests'] += 1 + if (dt.datetime.now(dt.timezone.utc)-head['LastModified']).total_seconds() < grace: + continue + client.delete_object(Bucket=BUCKET,Key=obj['Key']) + result['s3_requests'] += 1 + result['deleted'] += 1 + continuation = response.get('NextContinuationToken') + if not continuation: + break + last_id = '00000000-0000-0000-0000-000000000000' + while True: + with db() as conn: + rows = conn.execute('SELECT id FROM attachments WHERE id>%s::uuid ORDER BY id LIMIT %s',(last_id,page_size)).fetchall() + result['db_queries'] += 1 + if not rows: + break + for item in rows: + result['metadata_scanned'] += 1 + with db() as conn: + conn.execute('SELECT pg_advisory_xact_lock(%s)',(STORAGE_LOCK,)) + row = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(item['id'],)).fetchone() + result['db_queries'] += 2 + present = True + try: + head = client.head_object(Bucket=BUCKET,Key=row['object_key']) + result['s3_requests'] += 1 + present = head['ContentLength'] == row['size_bytes'] and head.get('Metadata',{}).get('sha256') == row['sha256'] + except ClientError as exc: + result['s3_requests'] += 1 + if not missing(exc): + raise + present = False + if present: + observation = conn.execute('SELECT 1 FROM missing_observations WHERE attachment_id=%s',(row['id'],)).fetchone() + result['db_queries'] += 1 + if row['availability'] == 'missing' or observation: + content = client.get_object(Bucket=BUCKET,Key=row['object_key'])['Body'] + result['s3_requests'] += 1 + digest = hashlib.sha256() + try: + while chunk := content.read(65536): + result['verified_bytes'] += len(chunk) + digest.update(chunk) + finally: + content.close() + present = digest.hexdigest() == row['sha256'] + if present: + result['recovered'] += 1 + if apply: + conn.execute("UPDATE attachments SET availability='present',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.execute('DELETE FROM missing_observations WHERE attachment_id=%s',(row['id'],)) + result['db_queries'] += 2 + if not present: + result['missing'] += 1 + observation = conn.execute('SELECT *,clock_timestamp()-first_seen AS age FROM missing_observations WHERE attachment_id=%s',(row['id'],)).fetchone() + result['db_queries'] += 1 + if not observation: + if apply: + conn.execute('INSERT INTO missing_observations(attachment_id) VALUES(%s) ON CONFLICT DO NOTHING',(row['id'],)) + result['db_queries'] += 1 + result['observation_started'] += 1 + elif observation['age'].total_seconds() >= grace and row['availability'] != 'missing': + result['quarantined'] += 1 + if apply: + conn.execute("UPDATE attachments SET availability='missing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + result['db_queries'] += 1 + last_id = str(rows[-1]['id']) + result['duration_seconds'] = round(time.monotonic()-started,4) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--grace-seconds',type=int,default=3600) + parser.add_argument('--page-size',type=int,default=100) + args = parser.parse_args() + if args.grace_seconds < 1 or not 1 <= args.page_size <= 1000: + parser.error('grace >=1, page size 1..1000 required') + try: + print(json.dumps(reconcile(args.apply,args.grace_seconds,args.page_size),sort_keys=True)) + except Exception as exc: + # Do not print endpoint, SQL, credentials, object keys or request parameters. + print(json.dumps({'status':'aborted','error_type':type(exc).__name__})) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..a881eeb3 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,30 @@ +"""Shared dependencies. No raw credentials are printed or exposed through HTTP.""" +import os +from pathlib import Path +import boto3 +from botocore.config import Config +import psycopg +from psycopg.rows import dict_row + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +PREFIX = 'originals/' +STORAGE_LOCK = 440004 + + +def setting(name, default=''): + path = os.getenv(name + '_FILE') + return Path(path).read_text().strip() if path else os.getenv(name, default) + + +def db(): + return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, + connect_timeout=3, options='-c statement_timeout=15000 -c lock_timeout=10000') + + +def s3(): + return boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=setting('S3_ACCESS_KEY_ID'), + aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), + config=Config(connect_timeout=3, read_timeout=10, max_pool_connections=16, + retries={'total_max_attempts': 1}, s3={'addressing_style': 'path'})) diff --git a/src/sweep.py b/src/sweep.py new file mode 100644 index 00000000..16424e5b --- /dev/null +++ b/src/sweep.py @@ -0,0 +1,11 @@ +"""Periodic reconciliation; operational failures abort the pass and retry next minute.""" +import json +import time +from .reconcile import reconcile + +while True: + try: + print(json.dumps(reconcile(apply=True,grace=3600),sort_keys=True),flush=True) + except Exception as exc: + print(json.dumps({'status':'aborted','error_type':type(exc).__name__}),flush=True) + time.sleep(60) diff --git a/student-tests/acceptance.py b/student-tests/acceptance.py new file mode 100644 index 00000000..58be3c31 --- /dev/null +++ b/student-tests/acceptance.py @@ -0,0 +1,119 @@ +"""Extra black-box, failure and reconciliation checks; no public tests modified.""" +import concurrent.futures as cf +import hashlib +import io +import json +import os +from pathlib import Path +import socket +import struct +import sys +import time +import urllib.error +import urllib.request +import uuid +import zlib + +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from client import Client, check +from PIL import Image +from botocore.exceptions import ClientError +from src.storage import BUCKET, PREFIX, STORAGE_LOCK, db, s3, setting +from src.reconcile import reconcile + + +def png(width,height): + def chunk(kind,content): + return struct.pack('!I',len(content))+kind+content+struct.pack('!I',zlib.crc32(kind+content)) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!IIBBBBB',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(b'\0\0\0\0'))+chunk(b'IEND',b'') + + +def main(): + c = Client('http://lb:8080',timeout=40) + a,b,e,chat = c.bootstrap() + content = (Path('tests/fixtures/sample.png')).read_bytes() + path = '/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png' + def upload(raw=content,mime='image/png',expected=201,actor=a,url=path,client=c): + return client.call('POST',url,expected,actor=actor,raw=raw, + headers={'Content-Type':mime},schema='Attachment' if expected==201 else None) + for label,raw,mime,expected in ( + ('oversize',b'x'*(10485760+1),'image/png',413), + ('corrupt',b'not an image','image/png',415), + ('truncated_png',content[:70],'image/png',415), + ('mime_mismatch',content,'image/jpeg',415), + ('pixel_limit',png(10000,5000),'image/png',413)): + upload(raw,mime,expected) + for name in ('..%2Fx.png','..%5Cx.png','a%00.png','a%0A.png'): + upload(expected=400,url=path.split('?')[0]+'?filename='+name) + # No Content-Length: the limit is enforced while receiving chunked transfer. + upload(raw=(b'x'*65536 for _ in range(161)),expected=413) + jpeg = io.BytesIO() + Image.new('RGB',(64,48),(10,30,80)).save(jpeg,format='JPEG') + jpeg_bytes = jpeg.getvalue() + jpeg_att = upload(jpeg_bytes,'image/jpeg')['json'] + check(c.call('GET','/api/v1/attachments/'+jpeg_att['id']+'/content',actor=b)['bytes']==jpeg_bytes,'JPEG byte mismatch') + upload(jpeg_bytes[:-20],'image/jpeg',415) + att = upload()['json'] + upload(actor=e,expected=404) + other = c.call('POST','/api/v1/chats',201,actor=a,body={'title':'Other','member_ids':[a['id']]},schema='Chat')['json'] + c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=a, + body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=a, + body={'text':'','attachment_ids':[str(uuid.uuid4())]},headers={'Idempotency-Key':uuid.uuid4().hex}) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=e, + body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + # Concurrent same-key requests distributed across two real API processes. + key = uuid.uuid4().hex + with cf.ThreadPoolExecutor(max_workers=12) as executor: + replies = list(executor.map(lambda _: c.message(a,chat,'race',key=key),range(12))) + check(len({r['json']['id'] for r in replies})==1,'idempotency race duplicated message') + check(len({r['headers']['X-Instance-Id'] for r in replies})==2,'race did not cross replicas') + cid = replies[0]['json']['id'] + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=a, + body={'attachment_ids':[],'text':'race'},headers={'Idempotency-Key':key},schema='Message') + for value in ({'text':123},{'text':'a','sender_id':a['id']}, + {'text':'a','attachment_ids':[att['id'],att['id']]}): + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',400,actor=a, + body=value,headers={'Idempotency-Key':uuid.uuid4().hex}) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',400,actor=a, + raw=b'{"text":"\\ud800"}',headers={'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}) + c.call('GET','/api/v1/attachments/invalid-uuid',400,actor=a) + c.call('GET','/api/v1/attachments/invalid-uuid',401) + # Private bucket and application IAM tested from inside isolated network. + with db() as conn: + row = conn.execute('SELECT * FROM attachments WHERE id=%s',(att['id'],)).fetchone() + url = 'http://s3:9000/'+BUCKET+'/'+row['object_key'] + try: + urllib.request.urlopen(url) + raise AssertionError('Anonymous bucket access succeeded') + except urllib.error.HTTPError as exc: + check(exc.code==403,'private bucket must deny anonymous GET') + try: + s3().delete_object(Bucket=BUCKET,Key=row['object_key']) + raise AssertionError('API user can delete objects') + except ClientError as exc: + check(exc.response['Error']['Code']=='AccessDenied','unexpected IAM error') + try: + s3().put_object(Bucket=BUCKET,Key='outside-prefix',Body=b'no') + raise AssertionError('API user can write outside prefix') + except ClientError as exc: + check(exc.response['Error']['Code']=='AccessDenied','prefix IAM denial expected') + # Disconnect before completing body: no object or metadata is persisted. + with db() as conn: + before = conn.execute('SELECT count(*) AS n FROM attachments').fetchone()['n'] + sock = socket.create_connection(('api1',8080)) + req = ('POST '+path+' HTTP/1.1\r\nHost: api1\r\nAuthorization: Bearer '+a['token']+ + '\r\nContent-Type: image/png\r\nContent-Length: 100000\r\n\r\n').encode()+content[:20] + sock.sendall(req) + sock.close() + time.sleep(.5) + with db() as conn: + check(conn.execute('SELECT count(*) AS n FROM attachments').fetchone()['n']==before,'disconnect persisted attachment') + print(json.dumps({'status':'PASS','negative_cases':21,'jpeg_roundtrip':True, + 'idempotency_concurrency':12,'idempotency_instances':2,'private_bucket':True, + 'api_delete_denied':True,'prefix_write_denied':True,'interrupted_upload':True})) + + +if __name__ == '__main__': + main() diff --git a/student-tests/outage.py b/student-tests/outage.py new file mode 100644 index 00000000..6ba0a7d2 --- /dev/null +++ b/student-tests/outage.py @@ -0,0 +1,18 @@ +"""Called while the operator has explicitly stopped one dependency.""" +import json +from pathlib import Path +import sys +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client + +c = Client('http://api1:8080',timeout=40) +c.call('GET','/health/live',schema='Health') +c.call('GET','/health/ready',503) +if sys.argv[1]=='postgres': + c.call('POST','/api/v1/users',503,body={'username':'outage','display_name':'Test','password':'test-password-123'}) +else: + a = c.user('outage') + chat = c.call('POST','/api/v1/chats',201,actor=a,body={'title':'S3 outage','member_ids':[a['id']]},schema='Chat')['json'] + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=a, + raw=Path('tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'}) +print(json.dumps({'status':'PASS','dependency':sys.argv[1],'live':200,'ready':503,'mutation':503})) diff --git a/student-tests/reconciliation.py b/student-tests/reconciliation.py new file mode 100644 index 00000000..259f8470 --- /dev/null +++ b/student-tests/reconciliation.py @@ -0,0 +1,101 @@ +"""Inject real orphan/missing failures using a dedicated operator-only test container.""" +import concurrent.futures as cf +import hashlib +import json +import os +from pathlib import Path +import sys +import time +import uuid +import boto3 +from botocore.exceptions import ClientError + +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from client import Client, check +from src.storage import BUCKET, PREFIX, db, setting +from src.reconcile import reconcile + + +def main(): + os.environ['S3_ACCESS_KEY_ID_FILE'] = '/run/secrets/reconcile_access_key_id' + os.environ['S3_SECRET_ACCESS_KEY_FILE'] = '/run/secrets/reconcile_secret_access_key' + c = Client('http://lb:8080',timeout=40) + a,b,e,chat = c.bootstrap() + content = Path('tests/fixtures/sample.png').read_bytes() + att,_ = c.attachment(a,chat) + c.message(a,chat,'used attachment',attachments=[att['id']]) + with db() as conn: + used = conn.execute('SELECT * FROM attachments WHERE id=%s',(att['id'],)).fetchone() + # Only the tests profile has root credentials; normal API/reconciler do not. + admin = boto3.client('s3',endpoint_url='http://s3:9000',region_name='us-east-1', + aws_access_key_id=setting('MINIO_ROOT_USER'),aws_secret_access_key=setting('MINIO_ROOT_PASSWORD')) + old_key = PREFIX+str(uuid.uuid4()) + admin.put_object(Bucket=BUCKET,Key=old_key,Body=content) + # Real API error after PUT and before INSERT/COMMIT, through a test-only process. + fault = Client('http://fault-api:8080') + fault.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=fault.png',503, + actor=a,raw=content,headers={'Content-Type':'image/png'}) + # A deferred PostgreSQL trigger raises a real error at COMMIT, after S3 PUT. + with db() as conn: + conn.execute("""CREATE OR REPLACE FUNCTION test_commit_fault() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.filename='commit-fault.png' THEN RAISE EXCEPTION 'test commit failure'; END IF; RETURN NEW; END $$""") + conn.execute('DROP TRIGGER IF EXISTS test_commit_fault ON attachments') + conn.execute('CREATE CONSTRAINT TRIGGER test_commit_fault AFTER INSERT ON attachments DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION test_commit_fault()') + try: + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=commit-fault.png',503, + actor=a,raw=content,headers={'Content-Type':'image/png'}) + finally: + with db() as conn: + conn.execute('DROP TRIGGER IF EXISTS test_commit_fault ON attachments') + conn.execute('DROP FUNCTION IF EXISTS test_commit_fault()') + time.sleep(3) + fresh_key = PREFIX+str(uuid.uuid4()) + admin.put_object(Bucket=BUCKET,Key=fresh_key,Body=content) + dry = reconcile(False,2,7) + check(dry['orphan_candidates']>=3 and dry['deleted']==0,'dry-run did not find failures safely') + admin.head_object(Bucket=BUCKET,Key=old_key) + applied = reconcile(True,2,7) + check(applied['deleted']>=3,'old orphans not removed') + admin.head_object(Bucket=BUCKET,Key=fresh_key) + admin.head_object(Bucket=BUCKET,Key=used['object_key']) + repeat = reconcile(True,2,7) + check(repeat['deleted']==0,'repeat immediately deleted fresh object') + # S3 loss: metadata + message link are retained, grace observations precede quarantine. + admin.delete_object(Bucket=BUCKET,Key=used['object_key']) + missing_dry = reconcile(False,2,7) + check(missing_dry['missing']>=1 and missing_dry['observation_started']==0,'dry-run changed DB') + first = reconcile(True,2,7) + check(first['observation_started']>=1 and first['quarantined']==0,'fresh absence quarantined too early') + time.sleep(3) + second = reconcile(True,2,7) + check(second['quarantined']>=1,'persistent missing object not quarantined') + c.call('GET','/api/v1/attachments/'+att['id']+'/content',503,actor=b) + with db() as conn: + check(conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s',(att['id'],)).fetchone(),'used metadata lost') + admin.put_object(Bucket=BUCKET,Key=used['object_key'],Body=content, + ContentType='image/png',Metadata={'sha256':used['sha256']}) + restored = reconcile(True,2,7) + check(restored['recovered']>=1,'restored bytes not verified/recovered') + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==content,'restored content mismatch') + # Reconciliation concurrent with PUT/COMMIT cannot collect valid attachments. + def uploading(_): + return c.attachment(a,chat)[0] + with cf.ThreadPoolExecutor(max_workers=11) as executor: + futures = [executor.submit(uploading,i) for i in range(10)] + scan = executor.submit(reconcile,True,2,7) + uploaded = [f.result() for f in futures] + concurrent = scan.result() + for item in uploaded: + check(c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=b)['bytes']==content,'concurrent sweep lost fresh upload') + # A cost sweep over > one page of objects/rows. + cost = [reconcile(False,3600,7) for _ in range(3)] + print(json.dumps({'status':'PASS','grace_for_test_seconds':2,'dry_run':dry, + 'apply':applied,'repeat':repeat,'missing_dry_run':missing_dry, + 'first_missing_observation':first,'quarantine':second,'restore':restored, + 'concurrent_scan':concurrent,'concurrent_uploads_preserved':len(uploaded), + 'cost_repeats':cost},sort_keys=True)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/upload_load.py b/student-tests/upload_load.py new file mode 100644 index 00000000..64fe7a82 --- /dev/null +++ b/student-tests/upload_load.py @@ -0,0 +1,88 @@ +"""Barrier-started >=10 upload clients and sampled RSS of both API processes.""" +import concurrent.futures as cf +import io +import json +from pathlib import Path +import statistics +import sys +import threading +import time +import urllib.request +from PIL import Image + +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client, check + + +def rss(): + return [json.load(urllib.request.urlopen('http://'+host+':8080/internal-test/memory',timeout=2)) + for host in ('api1','api2')] + + +def main(): + c = Client('http://lb:8080',timeout=40) + a,b,e,chat = c.bootstrap() + summaries = [] + for edge in (512,1536): + # Uncompressible RGB is a genuine PNG; ~0.75 and ~6.76 MiB. + import random + image = Image.frombytes('RGB',(edge,edge),random.Random(42).randbytes(edge*edge*3)) + output = io.BytesIO() + image.save(output,format='PNG') + content = output.getvalue() + del image,output + path = '/api/v1/chats/'+chat['id']+'/attachments?filename=load.png' + c.call('POST',path,201,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') + baseline = rss() + barrier = threading.Barrier(12) + latencies = [] + errors = [] + samples = [] + done = threading.Event() + def observe(): + while not done.is_set(): + samples.append(rss()) + done.wait(.025) + observer = threading.Thread(target=observe) + observer.start() + def upload(_): + barrier.wait() + for __ in range(3): + started = time.monotonic() + try: + result = c.call('POST',path,201,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') + latencies.append(time.monotonic()-started) + except Exception as exc: + errors.append(type(exc).__name__) + started = time.monotonic() + try: + with cf.ThreadPoolExecutor(max_workers=12) as pool: + list(pool.map(upload,range(12))) + finally: + done.set() + observer.join() + elapsed = time.monotonic()-started + check(not errors and len(latencies)==36,'concurrent uploads failed') + ordered = sorted(latencies) + peaks = [max(sample[i]['rss_bytes'] for sample in samples) for i in (0,1)] + cpu_end = rss() + cpu_delta = sum(cpu_end[i]['cpu_seconds']-baseline[i]['cpu_seconds'] for i in (0,1)) + summaries.append({'size_bytes':len(content),'dimensions':[edge,edge], + 'concurrency':12,'uploads':36,'successful_rps':round(36/elapsed,3), + 'duration_seconds':round(elapsed,3),'errors':0, + 'p50_seconds':round(statistics.median(ordered),4), + 'p95_seconds':round(ordered[int((len(ordered)-1)*.95)],4), + 'p99_seconds':round(ordered[int((len(ordered)-1)*.99)],4), + 'baseline_rss_bytes':[x['rss_bytes'] for x in baseline], + 'peak_rss_bytes':peaks,'cpu_seconds_total':round(cpu_delta,3), + 'baseline_container_memory_bytes':[x['container_memory_bytes'] for x in baseline], + 'sampled_peak_container_memory_bytes':[max(s[i]['container_memory_bytes'] for s in samples) for i in (0,1)], + 'kernel_container_peak_before_bytes':[x['container_peak_bytes'] for x in baseline], + 'kernel_container_peak_after_bytes':[x['container_peak_bytes'] for x in cpu_end], + 'cpu_average_percent_total':round(100*cpu_delta/elapsed,2), + 'memory_samples':len(samples)}) + print(json.dumps({'status':'PASS','profiles':summaries})) + + +if __name__ == '__main__': + main() -- 2.54.0