diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..22679bdd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..5f06ec8d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,20 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: заполняет владелец перед публикацией. +Целевая оценка (3/4/5): 5. +Трек 5: 2 — согласованность PostgreSQL и S3. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Из собственной lab3 перенесены API, PostgreSQL-сессии, ACL, пагинация, идемпотентность и балансировка. Добавлены приватный SeaweedFS, синхронные PNG/JPEG-вложения, потоковое скачивание, валидация, компенсация, reconciler и исследование отказов. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `python -X utf8 scripts/bootstrap.py`, `docker compose up -d --build --wait --wait-timeout 180`, `python -X utf8 scripts/verify.py --full`. + +Результаты: [REPORT.md](../REPORT.md), [evidence/RESULTS.md](../evidence/RESULTS.md), raw JSON и вывод проверок в evidence. Smoke, persistence, оба failover, негативные сценарии, 10 загрузок и исследование согласованности пройдены. + +Ограничения: один Docker-host, одиночные БД/LB/S3; lost original требует backup; сверка блокирует публикацию на время прохода. Источники: собственная lab3, официальная документация SeaweedFS/Boto3; использован OpenAI Codex. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Выполнены Python-эквиваленты make check / make test, приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..0af5bb34 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: application +on: [push, pull_request] +permissions: + contents: read +jobs: + integration: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: python scripts/bootstrap.py + - run: chmod +x scripts/restart.sh scripts/stop-one.sh + - run: docker compose up -d --build --wait --wait-timeout 180 + - run: python scripts/verify.py --full + - uses: actions/upload-artifact@v4 + if: always() + with: + name: evidence + path: evidence/ + - run: docker compose down + if: always() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..905a4610 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.13.2-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONUTF8=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src src +COPY migrations migrations +USER app +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..c5015dfc 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,81 @@ -# Отчёт — лабораторная 4 +# Лабораторная работа № 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +## Цель -## Запуск из чистого clone +Добавить к сообщениям изображения и проверить, что файлы не теряются при остановке одной копии приложения или перезапуске стенда. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Работа выполнена на оценку 5. Я выбрала второй трек — проверку согласованности базы данных и файлового хранилища. -## Архитектура и решения +## Что сделано -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +За основу я взяла свою третью лабораторную. Из неё остались регистрация, вход, чаты, сообщения и две копии API с балансировщиком HAProxy. Повторная отправка сообщения с тем же ключом не создаёт дубликат. -## Доказательства критериев +Для файлов я добавила SeaweedFS — хранилище. В PostgreSQL записываются название файла, размер, контрольная сумма и связь с чатом. Сам файл хранится отдельно в хранилище. Данные обоих хранилищ находятся в Docker volumes и сохраняются после перезапуска. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Можно загружать PNG и JPEG размером до 10 MiB и до 40 миллионов пикселей. Сервер проверяет содержимое изображения, а не только расширение. Повреждённые файлы и опасные имена отклоняются. Файл принимается частями, чтобы не держать его целиком в оперативной памяти. В одной копии API одновременно обрабатываются не больше двух загрузок, время приёма ограничено 30 секундами. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Название файла не используется как путь в хранилище: сервер сам создаёт случайный ключ. Скачать изображение можно через API после входа и проверки участия в чате. Само хранилище закрыто для посторонних. У API отдельные права только на нужный bucket, административные пароли ему не передаются. Наружу открыт только порт балансировщика. -## Эксперименты и отказы +Ответ 201 возвращается после сохранения файла и записи в базе. Если файл сохранился, а запись в базу не удалась, приложение пытается удалить лишний файл. Если результат сохранения в базу неизвестен, файл остаётся до отдельной проверки, чтобы случайно не удалить нужное вложение. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Запуск -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Из папки проекта нужно выполнить: -## Ограничения и следующие шаги +``` +python -X utf8 scripts/bootstrap.py +docker compose up -d --build --wait --wait-timeout 180 +python -X utf8 scripts/verify.py --full +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Первая команда создаёт локальные пароли и настройки. Повторный запуск сохраняет существующие пароли. Таблицы и закрытый bucket создаются автоматически. Файлы из `secrets/` и `.env` не добавляются в Git. + +API работает по адресу `http://localhost:8080`. Проверка готовности — `http://localhost:8080/health/ready`. + +Команда `docker compose down` останавливает стенд с сохранением данных. Команда `docker compose down --volumes` удаляет и контейнеры, и учебные данные. Для обычной остановки она не нужна. + +## Проверки + +Для полной проверки я использовала `python -X utf8 scripts/verify.py --full`. Скрипт сам запускает тесты, перезапускает стенд и по очереди останавливает копии API. + +| Что проверялось | Результат | +| --- | --- | +| Регистрация, вход, чаты и сообщения | Работают, повторная отправка не создаёт дубликат | +| Загрузка PNG и JPEG | После скачивания байты и контрольная сумма совпадают | +| Доступ другого участника чата | Файл доступен | +| Доступ постороннего | API возвращает 404, прямое обращение к S3 — 403 | +| Файл больше 10 MiB или 40 млн пикселей | Ошибка 413 | +| Повреждённое изображение или неверный MIME | Ошибка 415 | +| Опасное имя файла | Ошибка 400 | +| Вложение из другого чата | Ошибка 404 | +| Обрыв загрузки | Неполный файл не публикуется | +| Остановка загрузившей копии API | Другая копия скачивает тот же файл | +| Перезапуск API, PostgreSQL и SeaweedFS | Сообщения, файлы и действующая сессия сохраняются | +| Остановка S3 | Проверка готовности возвращает 503, после запуска файл снова доступен | + + +Полные числа, время ответов и вывод тестов находятся в [evidence/RESULTS.md] + +## Задание на 5: проверка двух хранилищ + +Моя гипотеза: отдельная проверка сможет найти лишние файлы и потерянные оригиналы, при этом не удаляя новые и используемые вложения. + +Для этого я добавила скрипт сверки. Сначала он только показывает найденные проблемы. С параметром `--apply` исправляет их: + +``` +docker compose exec -T api python -m src.reconcile +docker compose exec -T api python -m src.reconcile --apply +``` + +Если у файла нет записи в базе, скрипт удаляет его только после защитного периода. Если запись есть, но самого файла нет, запись и ссылки из сообщений сохраняются, а вложение отмечается как недоступное. Вернуть потерянные байты скрипт не может — для этого нужна резервная копия. + +Во время сверки используется блокировка PostgreSQL. Она не позволяет удалить файл, который другая копия API сейчас сохраняет. Новые файлы и файлы с записью в базе не удаляются. Сетевые ошибки не считаются доказательством потери файла. + +Я специально создала файл без записи в базе и запись без файла. Первый проход нашёл обе проблемы и ничего не изменил. Исправление удалило лишний файл и отметило потерянный. Повторный запуск уже ничего не изменил. Свежий файл и старое вложение, используемое в сообщении, сохранились. + + +Эти проверки находятся в `student-tests/acceptance.py` и `student-tests/consistency.py`.Она запускается вручную, автоматически при старте API не выполняется. + +## Ограничения + +Всё запущено на одном компьютере. Проверки показали, что остановка одной API и перезапуск стенда не теряют данные. Но при потере диска или Docker volumes данные без резервной копии восстановить не получится. База, балансировщик и S3 работают в одном экземпляре. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..9c73e4b7 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,115 @@ +name: mtusi-lab4 +services: + postgres: + image: postgres:17.2-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: + - db_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 15s + migrate: + image: mtusi-lab4-api:1.1.0 + pull_policy: never + command: [python, -m, src.migrate] + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + api: + build: . + image: mtusi-lab4-api:1.1.0 + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_SIZE: ${DB_POOL_SIZE:-4} + S3_ENDPOINT: http://s3:9000 + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS:-0.5} + mem_limit: ${API_MEMORY:-512m} + stop_grace_period: 8s + healthcheck: + test: [CMD, python, -S, src/healthcheck.py] + interval: 2s + timeout: 3s + retries: 2 + start_period: 10s + s3: + image: chrislusf/seaweedfs:3.85 + command: [server, -dir=/data, -ip=s3, -ip.bind=0.0.0.0, -s3, -s3.port=9000, -s3.config=/run/secrets/s3_config, -volume.max=8, -master.volumeSizeLimitMB=128] + secrets: + - s3_config + volumes: + - s3_data:/data + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://localhost:9333/cluster/status] + interval: 2s + timeout: 2s + retries: 30 + s3-bootstrap: + image: mtusi-lab4-api:1.1.0 + pull_policy: never + command: [python, -m, src.bootstrap_storage] + secrets: + - minio_root_user + - minio_root_password + depends_on: + s3: + condition: service_healthy + lb: + image: haproxy:3.0.7-alpine + ports: + - '127.0.0.1:${PUBLIC_PORT:-8080}:8080' + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + stop_grace_period: 8s +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_config: + file: ./secrets/s3_config +volumes: + s3_data: + postgres_data: diff --git a/contracts/reconciliation.md b/contracts/reconciliation.md new file mode 100644 index 00000000..0a4c7284 --- /dev/null +++ b/contracts/reconciliation.md @@ -0,0 +1,15 @@ +# Эксплуатационное расширение: сверка S3 и PostgreSQL + +Обязательный HTTP API не изменён. Команда `python -m src.reconcile` работает внутри контейнера API с его ограниченными S3 credentials. По умолчанию выполняется dry-run; `--apply` разрешает изменения, `--grace-seconds` задаёт возраст кандидата. Значение по умолчанию 3600 секунд, минимально допустимое — 60. + +Команда получает эксклюзивную транзакционную advisory lock 202604. Загрузчик получает shared lock с тем же номером перед S3 PUT и держит её до COMMIT. Загрузка тела и проверка изображения проходят до блокировки. Это исключает удаление объекта между PUT и фиксацией его метаданных. Другой reconciler сериализуется той же блокировкой. + +Постраничный ListObjectsV2 просматривает только серверный prefix `originals/`. Объекты с метаданными сохраняются независимо от возраста и использования в сообщениях. Объекты без метаданных моложе grace сохраняются. Для старого orphan дополнительно проверяется HEAD непосредственно перед DELETE. Серверные ключи уникальны и не переиспользуются. Внешние записи с административными credentials находятся за пределами этого протокола. + +Ready-метаданные старше grace проверяются HEAD. Только достоверный 404 переводит внутреннее состояние в `missing` с `original_missing`. БД и ссылки сообщений сохраняются. Сетевая ошибка, 403 или 5xx не считаются отсутствием объекта. Изменения метаданных транзакционны; повторный проход не обрабатывает уже отмеченный missing. + +Внутренний missing не возвращается как Attachment, поскольку обязательный контракт lab4 допускает только ready. Авторизованные GET метаданных и содержимого возвращают 503; POST нового сообщения с таким ID возвращает 409. Посторонний всегда получает 404. Восстановление оригинала из резервной копии и возврат ready выполняются оператором после проверки размера и SHA-256. Reconciler не восстанавливает потерянные байты и не объявляет их восстановленными. + +Сверка не удаляет метаданные и не удаляет объекты, для которых есть строка attachments. DELETE идемпотентен; сбой после DELETE до COMMIT безопасен: следующий проход больше не увидит orphan. При недоступности БД сверка не начинает S3 DELETE, при недоступности S3 завершается с ошибкой. Полной атомарности между S3 и БД нет. + +Стоимость: ceil(N/1000) LIST, N индексных SQL lookup, до A HEAD для старых ready-строк плюс HEAD старых orphan и DELETE каждого исправленного orphan. Память ограничена одной страницей S3 и 100 строками SQL cursor. Эксклюзивная блокировка временно задерживает публикацию вложений; запускать вручную в окно обслуживания. Для большого хранилища необходимы секционирование и ограничение времени прохода. В учебном стенде безопасный отказ загрузки при ожидании блокировки предпочтительнее неограниченного ожидания. diff --git a/evidence/RESULTS.md b/evidence/RESULTS.md new file mode 100644 index 00000000..2cae09d8 --- /dev/null +++ b/evidence/RESULTS.md @@ -0,0 +1,82 @@ +# Фактические результаты + +## Загрузка: 10 клиентов одновременно + +| Байт | Успех / ошибки | Время, с | RPS | p50, с | p95/p99, с | Наблюдаемый максимум RAM, MiB | Cgroup peak за жизнь API, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 787247 | 10 / 0 | 4.349 | 2.30 | 0.939 | 1.552 | 79.20 | 148.36 | +| 3147775 | 10 / 0 | 4.808 | 2.08 | 1.516 | 1.895 | 101.30 | 148.36 | + +Профиль: одна волна на размер, без повторов HTTP. Docker stats опрашивается параллельно; время ожидания мониторинга исключено из RPS. p95 и p99 при n=10 совпадают с максимумом (nearest rank). Cgroup peak включает прогрев и все предыдущие запросы процесса; это верхняя граница памяти волны, а не её изолированный расход. + +## Reconciler: внедрённые orphan и missing + +| Режим | Объекты | Метаданные ready | LIST | HEAD | Удалено | Помечено missing | Время, с | +| --- | --- | --- | --- | --- | --- | --- | --- | +| dry_run | 54 | 53 | 1 | 44 | 0 | 0 | 0.811 | +| apply | 54 | 53 | 1 | 49 | 1 | 1 | 0.861 | +| repeat | 53 | 52 | 1 | 52 | 0 | 0 | 1.316 | + +## Стоимость обхода: три повтора на размер + +| Дополнительные объекты | Всего объектов | LIST | HEAD | Медиана, с | Минимум / максимум, с | +| --- | --- | --- | --- | --- | --- | +| 10 | 63 | 1 | 1 | 0.998 | 0.554 / 2.476 | +| 100 | 153 | 1 | 1 | 0.839 | 0.808 / 0.880 | + +Сбой INSERT после успешного PUT: число объектов до и после компенсации — 53 и 53. Старое вложение, используемое сообщением, сохранено; SHA-256/байты и ссылка сообщения проверены. + +## Проверки + +### contract.txt + +```text +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +``` + +### smoke.txt + +```text +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +``` + +### acceptance.txt + +```text +PASS: negative images, ACL, private S3, interrupted upload, concurrent uploads, reconciliation, S3 outage +``` + +### consistency.txt + +```text +PASS: failure after S3 PUT compensates, old used attachment survives, scan cost measured twice with three repeats +``` + +### persistence.txt + +```text +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +### failover-1.txt + +```text +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 96, "failed_attempts": 6, "first_recovery_seconds": 4.274} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +### failover-2.txt + +```text +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 3, "first_recovery_seconds": 3.328} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +### smoke-final.txt + +```text +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +``` diff --git a/evidence/acceptance.json b/evidence/acceptance.json new file mode 100644 index 00000000..39c41ce1 --- /dev/null +++ b/evidence/acceptance.json @@ -0,0 +1,251 @@ +{ + "negative_tests": "PASS: byte/pixel limits, corrupt/truncated images, MIME mismatch, filename, ACL, foreign and unknown attachment", + "private_bucket": { + "anonymous_get": 403 + }, + "uploads": [ + { + "size_bytes": 787247, + "concurrency": 10, + "success": 10, + "errors": 0, + "seconds": 4.3491024999530055, + "successful_rps": 2.299324975695113, + "p50": 0.9388057499891147, + "p95": 1.552345400035847, + "p99": 1.552345400035847, + "baseline": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "31.36%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "14.98%", + "MemUsage": "76.69MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "795kB / 281kB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.67%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "13.87%", + "MemUsage": "71.03MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "11.7MB / 650kB", + "PIDs": "8" + } + ], + "samples": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "22.72%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "14.97%", + "MemUsage": "76.65MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "799kB / 289kB", + "PIDs": "7" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.94%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "13.82%", + "MemUsage": "70.77MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "11.7MB / 654kB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.40%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "15.47%", + "MemUsage": "79.2MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "4.79MB / 4.28MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.99%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "14.72%", + "MemUsage": "75.34MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "15.6MB / 4.64MB", + "PIDs": "10" + } + ], + "lifetime_cgroup_peak_bytes": { + "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e": 155566080, + "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd": 150491136 + } + }, + { + "size_bytes": 3147775, + "concurrency": 10, + "success": 10, + "errors": 0, + "seconds": 4.808497400023043, + "successful_rps": 2.079651743172842, + "p50": 1.5156683000095654, + "p95": 1.8952780999825336, + "p99": 1.8952780999825336, + "baseline": [ + { + "BlockIO": "69.6kB / 0B", + "CPUPerc": "1.63%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "15.46%", + "MemUsage": "79.15MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "4.79MB / 4.29MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.70%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "14.72%", + "MemUsage": "75.36MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "15.7MB / 4.65MB", + "PIDs": "10" + } + ], + "samples": [ + { + "BlockIO": "69.6kB / 0B", + "CPUPerc": "15.45%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "15.56%", + "MemUsage": "79.66MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "4.79MB / 4.3MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.82%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "14.74%", + "MemUsage": "75.46MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "15.7MB / 4.65MB", + "PIDs": "10" + }, + { + "BlockIO": "69.6kB / 0B", + "CPUPerc": "51.64%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "19.12%", + "MemUsage": "97.88MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "20.7MB / 17MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "52.52%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "19.79%", + "MemUsage": "101.3MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "31.1MB / 14.2MB", + "PIDs": "11" + }, + { + "BlockIO": "69.6kB / 0B", + "CPUPerc": "2.01%", + "Container": "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e", + "ID": "4eba80629d64", + "MemPerc": "17.12%", + "MemUsage": "87.64MiB / 512MiB", + "Name": "mtusi-lab4-api-1", + "NetIO": "20.7MB / 20.1MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.95%", + "Container": "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd", + "ID": "0fe6963e2e3f", + "MemPerc": "16.87%", + "MemUsage": "86.39MiB / 512MiB", + "Name": "mtusi-lab4-api-2", + "NetIO": "31.5MB / 20.5MB", + "PIDs": "9" + } + ], + "lifetime_cgroup_peak_bytes": { + "4eba80629d64309da396f81516e404de8b9b38544e565800710f50cb6c77572e": 155566080, + "0fe6963e2e3f298ea2bc81dd0c50713412627b76bb0871dc97321c0311e250cd": 150491136 + } + } + ], + "interrupted_upload": "Connection closed after 100 bytes of declared 100000; no S3 PUT before validation", + "reconciler": { + "dry_run": { + "apply": false, + "grace_seconds": 60, + "objects": 54, + "metadata": 53, + "list_requests": 1, + "head_requests": 44, + "orphan_candidates": 1, + "missing_candidates": 1, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 1, + "referenced_preserved": 52, + "elapsed_seconds": 0.811033 + }, + "apply": { + "apply": true, + "grace_seconds": 60, + "objects": 54, + "metadata": 53, + "list_requests": 1, + "head_requests": 49, + "orphan_candidates": 1, + "missing_candidates": 1, + "deleted": 1, + "marked_failed": 1, + "fresh_preserved": 1, + "referenced_preserved": 52, + "elapsed_seconds": 0.860571 + }, + "repeat": { + "apply": true, + "grace_seconds": 60, + "objects": 53, + "metadata": 52, + "list_requests": 1, + "head_requests": 52, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 1, + "referenced_preserved": 52, + "elapsed_seconds": 1.31619 + } + }, + "s3_outage": { + "ready": 503 + } +} \ No newline at end of file diff --git a/evidence/acceptance.txt b/evidence/acceptance.txt new file mode 100644 index 00000000..11df8a42 --- /dev/null +++ b/evidence/acceptance.txt @@ -0,0 +1 @@ +PASS: negative images, ACL, private S3, interrupted upload, concurrent uploads, reconciliation, S3 outage diff --git a/evidence/after-restart.json b/evidence/after-restart.json new file mode 100644 index 00000000..f24a5f4a --- /dev/null +++ b/evidence/after-restart.json @@ -0,0 +1,42 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "4eba80629d64", + "started": "2026-11-16T14:50:45.918611054Z", + "image": "mtusi-lab4-api:1.1.0", + "memory_limit": 536870912, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-api-2", + "id": "0fe6963e2e3f", + "started": "2026-11-16T14:50:45.756873756Z", + "image": "mtusi-lab4-api:1.1.0", + "memory_limit": 536870912, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "b037c886a390", + "started": "2026-11-16T14:50:44.691408938Z", + "image": "haproxy:3.0.7-alpine", + "memory_limit": 134217728, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-postgres-1", + "id": "01a11c4599ef", + "started": "2026-11-16T14:50:44.737552905Z", + "image": "postgres:17.2-alpine", + "memory_limit": 536870912, + "cpus": 1.0 + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "1e4d8a357ae6", + "started": "2026-11-16T14:50:45.849082671Z", + "image": "chrislusf/seaweedfs:3.85", + "memory_limit": 536870912, + "cpus": 1.0 + } +] \ No newline at end of file diff --git a/evidence/before-restart.json b/evidence/before-restart.json new file mode 100644 index 00000000..25b9c2ba --- /dev/null +++ b/evidence/before-restart.json @@ -0,0 +1,42 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "4eba80629d64", + "started": "2026-11-16T14:45:51.305158139Z", + "image": "mtusi-lab4-api:1.1.0", + "memory_limit": 536870912, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-api-2", + "id": "0fe6963e2e3f", + "started": "2026-11-16T14:45:51.57620223Z", + "image": "mtusi-lab4-api:1.1.0", + "memory_limit": 536870912, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "b037c886a390", + "started": "2026-11-16T14:32:24.804548928Z", + "image": "haproxy:3.0.7-alpine", + "memory_limit": 134217728, + "cpus": 0.5 + }, + { + "name": "/mtusi-lab4-postgres-1", + "id": "01a11c4599ef", + "started": "2026-11-16T14:32:24.878985233Z", + "image": "postgres:17.2-alpine", + "memory_limit": 536870912, + "cpus": 1.0 + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "1e4d8a357ae6", + "started": "2026-11-16T14:48:26.751569712Z", + "image": "chrislusf/seaweedfs:3.85", + "memory_limit": 536870912, + "cpus": 1.0 + } +] \ No newline at end of file diff --git a/evidence/consistency.json b/evidence/consistency.json new file mode 100644 index 00000000..bc154f72 --- /dev/null +++ b/evidence/consistency.json @@ -0,0 +1,128 @@ +{ + "database_failure_after_put": { + "before_objects": 53, + "after_objects": 53, + "http_status": 503 + }, + "scan_cost": [ + { + "extra_objects": 10, + "repeats": [ + { + "apply": false, + "grace_seconds": 3600, + "objects": 63, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 10, + "referenced_preserved": 53, + "elapsed_seconds": 0.554496 + }, + { + "apply": false, + "grace_seconds": 3600, + "objects": 63, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 10, + "referenced_preserved": 53, + "elapsed_seconds": 2.475882 + }, + { + "apply": false, + "grace_seconds": 3600, + "objects": 63, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 10, + "referenced_preserved": 53, + "elapsed_seconds": 0.997638 + } + ] + }, + { + "extra_objects": 100, + "repeats": [ + { + "apply": false, + "grace_seconds": 3600, + "objects": 153, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 100, + "referenced_preserved": 53, + "elapsed_seconds": 0.83912 + }, + { + "apply": false, + "grace_seconds": 3600, + "objects": 153, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 100, + "referenced_preserved": 53, + "elapsed_seconds": 0.807927 + }, + { + "apply": false, + "grace_seconds": 3600, + "objects": 153, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 100, + "referenced_preserved": 53, + "elapsed_seconds": 0.879804 + } + ] + } + ], + "old_used_attachment": { + "checksum_matches": true, + "message_reference_preserved": true, + "apply": { + "apply": true, + "grace_seconds": 3600, + "objects": 153, + "metadata": 53, + "list_requests": 1, + "head_requests": 1, + "orphan_candidates": 0, + "missing_candidates": 0, + "deleted": 0, + "marked_failed": 0, + "fresh_preserved": 100, + "referenced_preserved": 53, + "elapsed_seconds": 0.793124 + } + } +} \ No newline at end of file diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 00000000..10219f6c --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1 @@ +PASS: failure after S3 PUT compensates, old used attachment survives, scan cost measured twice with three repeats diff --git a/evidence/contract.txt b/evidence/contract.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/contract.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..4820923b --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,8 @@ +{ + "ServerVersion": "29.1.3", + "OperatingSystem": "Docker Desktop (containerized)", + "Architecture": "x86_64", + "NCPU": 8, + "MemTotal": 8214183936, + "KernelVersion": "6.18.58.1-microsoft-standard-WSL2" +} \ No newline at end of file diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..13766bf9 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 96, "failed_attempts": 6, "first_recovery_seconds": 4.274} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..6d63eaae --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 3, "first_recovery_seconds": 3.328} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/restore-1.txt b/evidence/restore-1.txt new file mode 100644 index 00000000..b9d40513 --- /dev/null +++ b/evidence/restore-1.txt @@ -0,0 +1,3 @@ + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started diff --git a/evidence/restore-2.txt b/evidence/restore-2.txt new file mode 100644 index 00000000..7d4a6893 --- /dev/null +++ b/evidence/restore-2.txt @@ -0,0 +1,3 @@ + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started diff --git a/evidence/runtime-packages.txt b/evidence/runtime-packages.txt new file mode 100644 index 00000000..3c22ae9a --- /dev/null +++ b/evidence/runtime-packages.txt @@ -0,0 +1,27 @@ +annotated-types==0.8.0 +anyio==4.16.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.2.0 +boto3==1.36.26 +botocore==1.36.26 +cffi==2.1.1 +click==8.5.1 +fastapi==0.115.6 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +pillow==11.1.0 +psycopg==3.2.3 +psycopg-binary==3.2.3 +psycopg-pool==3.3.3 +pycparser==3.12 +pydantic==2.14.3 +pydantic_core==2.50.4 +python-dateutil==2.9.0.post0 +s3transfer==0.11.3 +six==1.17.0 +starlette==0.41.3 +typing-inspection==0.4.5 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.0 diff --git a/evidence/smoke-final.txt b/evidence/smoke-final.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke-final.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/source-manifest.json b/evidence/source-manifest.json new file mode 100644 index 00000000..f2ac2b94 --- /dev/null +++ b/evidence/source-manifest.json @@ -0,0 +1,24 @@ +{ + "sha256": { + "compose.yaml": "6901392374b4ac92f183782886168bac9b42b1bd2bae3a432f31d443872781b9", + "Dockerfile": "190e39b1ce03433d0010ddf063a46ab82e539dc16c9be2ce8704bc39d89015ce", + "infra/filer.toml": "ae3c01bf7708f5d4a319fdb454f753be0f137fe7c31269f1c413cbccc860f5c7", + "infra/haproxy.cfg": "3dbce285792a5347d5588eea9f0d8558295202a5bccf42ec05bc027376871f41", + "infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278", + "migrations/001_initial.sql": "6e4c5ff62e1fd04f0c3cdeaf129709a8942d40baf2515db9d1c0f8d8491173e3", + "migrations/002_attachments.sql": "e3a5e05767086a5121706cc4c341eed2d906366aba70d03ce6ca16bb7cca34c3", + "migrations/003_attachment_fields.sql": "78fe93590db8508dec1bae48f5a074d4dfa081ec4e0c33558011a7082dc085a2", + "requirements.txt": "602bd0923251ac8874a32b5b9a102692f34fb6a486744bec6af25766ea0a8e1e", + "src/__init__.py": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "src/attachments.py": "6cb081938be6101886cd438240935a413e6ed0be203cb95d2b9489b0ac090a53", + "src/bootstrap_storage.py": "9f10a9cc4accac2e9c0323b319ef89677f90cdcb6b950ef43c62f98349ba7ba9", + "src/healthcheck.py": "403007af34767db13e501295a849a5187e8cffbfdfafc4890fa022442a8c27d3", + "src/main.py": "5bf0610186532d856e96b3014a2c1c3ac8ed63937c655036e1460a7de2521db1", + "src/migrate.py": "dedbaa3e5810b9cca9b70347a6489bfc2286aa572f612e5c9079ca027a72f44a", + "src/reconcile.py": "89118be83b3b213713862e7409c3e9ccbf792a55590ce1295bfc4e769c6a016f", + "src/storage.py": "090ada62b5c5c537d7af99df5b245982bb688bd0b33af644eec98bf9fe0f5834", + "student-tests/acceptance.py": "5f6264dc9f65742d7787682aa19f96d9c9db3f087a116f1617814d8914cf9cdf", + "student-tests/consistency.py": "ce32d6c89b0030b44d08751cc5fde53df855142e1058698c620ef65edff3293d" + }, + "container_sources_match": true +} \ No newline at end of file diff --git a/evidence/stop-events.jsonl b/evidence/stop-events.jsonl new file mode 100644 index 00000000..089efd61 --- /dev/null +++ b/evidence/stop-events.jsonl @@ -0,0 +1,2 @@ +{"target_instance": "4eba80629d64", "container_id": "4eba80629d64", "name": "/mtusi-lab4-api-1", "status_after": "exited", "finished_at": "2026-11-16T14:51:20.539870025Z"} +{"target_instance": "0fe6963e2e3f", "container_id": "0fe6963e2e3f", "name": "/mtusi-lab4-api-2", "status_after": "exited", "finished_at": "2026-11-16T14:51:44.938007996Z"} diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 00000000..85b2149c --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filer" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..6aeab6e4 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + maxconn 2048 + log stdout format raw local0 + hard-stop-after 5s + +defaults + mode http + timeout connect 1s + timeout client 40s + timeout server 40s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend messenger + +backend messenger + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 8 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..67f620fb --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text UNIQUE NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX IF NOT EXISTS sessions_expiry ON sessions(expires_at); +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY(chat_id, user_id) +); +CREATE INDEX IF NOT EXISTS members_user ON members(user_id, chat_id); +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS messages_history ON messages(chat_id, created_at, id); +CREATE TABLE IF NOT EXISTS idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + body text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY(user_id, chat_id, key) +); diff --git a/migrations/002_attachments.sql b/migrations/002_attachments.sql new file mode 100644 index 00000000..a9f83599 --- /dev/null +++ b/migrations/002_attachments.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text UNIQUE NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + status text NOT NULL DEFAULT 'ready', + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids uuid[] NOT NULL DEFAULT '{}'; +CREATE INDEX IF NOT EXISTS attachments_chat ON attachments(chat_id); diff --git a/migrations/003_attachment_fields.sql b/migrations/003_attachment_fields.sql new file mode 100644 index 00000000..1d94f7e5 --- /dev/null +++ b/migrations/003_attachment_fields.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS owner_id uuid REFERENCES users(id); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); +ALTER TABLE attachments ALTER COLUMN owner_id SET NOT NULL; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4376ba61 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +psycopg[binary,pool]==3.2.3 +argon2-cffi==23.1.0 +boto3==1.36.26 +Pillow==11.1.0 diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 00000000..90d02fc1 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,32 @@ +import ast +import hashlib +import io +import json +from pathlib import Path +import subprocess +import tokenize + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + paths = [] + for directory in ('src', 'migrations', 'infra', 'student-tests'): + paths.extend(path for path in (ROOT / directory).rglob('*') if path.is_file() and '__pycache__' not in path.parts) + paths.extend(ROOT / name for name in ('Dockerfile', 'compose.yaml', 'requirements.txt')) + for directory in ('src', 'student-tests'): + for path in (ROOT / directory).glob('*.py'): + code = path.read_text(encoding='utf-8') + ast.parse(code) + assert not any(token.type == tokenize.COMMENT for token in tokenize.generate_tokens(io.StringIO(code).readline)), str(path) + manifest = {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(paths)} + code = "import hashlib,json; from pathlib import Path; print(json.dumps({p.as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in Path('src').glob('*.py')}))" + result = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code], cwd=ROOT, text=True) + container = json.loads(result) + assert container and all(manifest[name] == digest for name, digest in container.items()), 'Container sources differ' + (ROOT / 'evidence/source-manifest.json').write_text(json.dumps({'sha256': manifest, 'container_sources_match': True}, indent=2), encoding='utf-8') + print('PASS: application Python has no comments; running container matches final source hashes') + + +if __name__ == '__main__': + main() diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..513727b2 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,33 @@ +from pathlib import Path +import secrets +import json + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(exist_ok=True) +password_file = directory / 'db_password' +if not password_file.exists(): + password_file.write_text(secrets.token_hex(32), encoding='utf-8') +url_file = directory / 'database_url' +if not url_file.exists(): + url_file.write_text('postgresql://messenger:' + password_file.read_text().strip() + '@postgres:5432/messenger', encoding='utf-8') +environment = root / '.env' +if not environment.exists(): + environment.write_text('API_REPLICAS=2\nAPI_CPUS=0.5\nAPI_MEMORY=512m\nDB_POOL_SIZE=4\nSESSION_TTL_SECONDS=3600\n', encoding='utf-8') +print('Local configuration prepared; existing credentials preserved.') +for name, value in { + 'minio_root_user': 'root-' + secrets.token_hex(8), + 'minio_root_password': secrets.token_hex(32), + 's3_access_key_id': 'api-' + secrets.token_hex(8), + 's3_secret_access_key': secrets.token_hex(32), +}.items(): + target = directory / name + if not target.exists(): + target.write_text(value, encoding='utf-8') +identities = [] +for name, key, password, actions in ( + ('bootstrap', 'minio_root_user', 'minio_root_password', ['Admin', 'Read', 'Write', 'List']), + ('api', 's3_access_key_id', 's3_secret_access_key', ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']), +): + identities.append({'name': name, 'credentials': [{'accessKey': (directory / key).read_text().strip(), 'secretKey': (directory / password).read_text().strip()}], 'actions': actions}) +(directory / 's3_config').write_text(json.dumps({'identities': identities}), encoding='utf-8') diff --git a/scripts/operate.py b/scripts/operate.py new file mode 100644 index 00000000..4855fce4 --- /dev/null +++ b/scripts/operate.py @@ -0,0 +1,74 @@ +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args): + return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip() + + +def containers(): + ids = docker('compose', 'ps', '-q', 'api').split() + return json.loads(docker('inspect', *ids)) if ids else [] + + +def main(): + mode = sys.argv[1] + if mode == 'restart': + subprocess.run(['docker', 'compose', 'restart', 'postgres', 's3', 'api', 'lb'], cwd=ROOT, check=True) + elif mode == 'stop': + target = os.environ.get('TARGET_INSTANCE_ID') + matches = [item for item in containers() if item['Config']['Hostname'] == target] + if len(matches) != 1: + raise SystemExit('Target must identify exactly one running API container') + print('Stopping API container ' + matches[0]['Id'][:12]) + docker('stop', '-t', '2', matches[0]['Id']) + state = json.loads(docker('inspect', matches[0]['Id']))[0] + event = {'target_instance': target, 'container_id': state['Id'][:12], + 'name': state['Name'], 'status_after': state['State']['Status'], + 'finished_at': state['State']['FinishedAt']} + with (ROOT / 'evidence/stop-events.jsonl').open('a', encoding='utf-8') as output: + output.write(json.dumps(event) + '\n') + elif mode == 'rolling': + import time + import urllib.request + for item in containers(): + name = item['Name'].lstrip('/') + started = time.time() + stages = {} + docker('stop', '-t', '6', item['Id']) + stages['stopped_seconds'] = round(time.time() - started, 3) + docker('rm', item['Id']) + stages['removed_seconds'] = round(time.time() - started, 3) + docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api') + stages['created_seconds'] = round(time.time() - started, 3) + deadline = time.monotonic() + 45 + observations = [] + previous = None + while time.monotonic() < deadline: + active = containers() + state = [(c['Id'][:12], c.get('State', {}).get('Health', {}).get('Status')) for c in active] + if state != previous: + observations.append({'elapsed_seconds': round(time.time()-started, 3), 'containers': [{'id': c['Id'][:12], 'health': c.get('State', {}).get('Health', {})} for c in active]}) + previous = state + if len(active) == 2 and all(c.get('State', {}).get('Health', {}).get('Status') == 'healthy' for c in active): + break + time.sleep(0.5) + else: + raise SystemExit('Replacement did not become healthy: ' + name) + stages['healthy_seconds'] = round(time.time() - started, 3) + with urllib.request.urlopen(os.getenv('BASE_URL', 'http://localhost:8080').rstrip('/') + '/health/ready', timeout=3) as response: + if response.status != 200: + raise SystemExit('LB unavailable') + time.sleep(3) + print(json.dumps({'event': 'rolling_replacement', 'old_name': name, 'old_container': item['Id'][:12], 'started_epoch': started, 'finished_epoch': time.time(), 'stages': stages, 'health_observations': observations, 'new_containers': [c['Id'][:12] for c in active]}), flush=True) + else: + raise SystemExit('Unknown operation') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..685915d7 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0operate.py" restart diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..645eef9f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/operate.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8afeaa4f --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0operate.py" stop diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..1490a02a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/operate.py stop diff --git a/scripts/summarize.py b/scripts/summarize.py new file mode 100644 index 00000000..cd37012c --- /dev/null +++ b/scripts/summarize.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path +import statistics + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + data = json.loads((ROOT / 'evidence/acceptance.json').read_text(encoding='utf-8')) + consistency = json.loads((ROOT / 'evidence/consistency.json').read_text(encoding='utf-8')) + lines = ['# Фактические результаты', '', '## Загрузка: 10 клиентов одновременно', '', + '| Байт | Успех / ошибки | Время, с | RPS | p50, с | p95/p99, с | Наблюдаемый максимум RAM, MiB | Cgroup peak за жизнь API, MiB |', + '| --- | --- | --- | --- | --- | --- | --- | --- |'] + for row in data['uploads']: + ram = max(float(r['MemUsage'].split('MiB')[0]) for r in row['samples']) + high = max(row.get('lifetime_cgroup_peak_bytes', {}).values(), default=0) / 1048576 + lines.append(f"| {row['size_bytes']} | {row['success']} / {row['errors']} | {row['seconds']:.3f} | {row['successful_rps']:.2f} | {row['p50']:.3f} | {row['p95']:.3f} | {ram:.2f} | {high:.2f} |") + lines += ['', 'Профиль: одна волна на размер, без повторов HTTP. Docker stats опрашивается параллельно; время ожидания мониторинга исключено из RPS. p95 и p99 при n=10 совпадают с максимумом (nearest rank). Cgroup peak включает прогрев и все предыдущие запросы процесса; это верхняя граница памяти волны, а не её изолированный расход.', '', + '## Reconciler: внедрённые orphan и missing', '', '| Режим | Объекты | Метаданные ready | LIST | HEAD | Удалено | Помечено missing | Время, с |', '| --- | --- | --- | --- | --- | --- | --- | --- |'] + for name, row in data['reconciler'].items(): + lines.append(f"| {name} | {row['objects']} | {row['metadata']} | {row['list_requests']} | {row['head_requests']} | {row['deleted']} | {row['marked_failed']} | {row['elapsed_seconds']:.3f} |") + lines += ['', '## Стоимость обхода: три повтора на размер', '', '| Дополнительные объекты | Всего объектов | LIST | HEAD | Медиана, с | Минимум / максимум, с |', '| --- | --- | --- | --- | --- | --- |'] + for group in consistency['scan_cost']: + repeats = group['repeats'] + durations = [r['elapsed_seconds'] for r in repeats] + row = repeats[0] + lines.append(f"| {group['extra_objects']} | {row['objects']} | {row['list_requests']} | {row['head_requests']} | {statistics.median(durations):.3f} | {min(durations):.3f} / {max(durations):.3f} |") + lines += ['', 'Сбой INSERT после успешного PUT: число объектов до и после компенсации — ' + str(consistency['database_failure_after_put']['before_objects']) + ' и ' + str(consistency['database_failure_after_put']['after_objects']) + '. Старое вложение, используемое сообщением, сохранено; SHA-256/байты и ссылка сообщения проверены.', '', + '## Проверки', ''] + for name in ('contract.txt', 'smoke.txt', 'acceptance.txt', 'consistency.txt', 'persistence.txt', 'failover-1.txt', 'failover-2.txt', 'smoke-final.txt'): + lines += ['### ' + name, '', '```text', (ROOT / 'evidence' / name).read_text(encoding='utf-8').strip(), '```', ''] + (ROOT / 'evidence/RESULTS.md').write_text('\n'.join(lines), encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..1e3a8fd2 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,68 @@ +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] + + +def call(name, args): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', env=os.environ | {'PYTHONUTF8': '1'}) + (ROOT / 'evidence' / name).write_text(result.stdout + result.stderr, encoding='utf-8') + print(name + ': ' + str(result.returncode), flush=True) + if result.returncode: + print(result.stdout + result.stderr) + raise SystemExit(result.returncode) + + +def snapshot(name): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split() + rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + safe = [{'name': r['Name'], 'id': r['Id'][:12], 'started': r['State']['StartedAt'], + 'image': r['Config']['Image'], 'memory_limit': r['HostConfig']['Memory'], + 'cpus': r['HostConfig']['NanoCpus']/1e9} for r in rows] + (ROOT / 'evidence' / name).write_text(json.dumps(safe, indent=2), encoding='utf-8') + + +def ready(): + for _ in range(90): + try: + with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3) as response: + if response.status == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError('Readiness timeout') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--full', action='store_true') + args = parser.parse_args() + call('contract.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py']) + call('smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py']) + if not args.full: + return + call('acceptance.txt', [sys.executable, '-X', 'utf8', 'student-tests/acceptance.py']) + call('consistency.txt', [sys.executable, '-X', 'utf8', 'student-tests/consistency.py']) + call('audit.txt', [sys.executable, '-X', 'utf8', 'scripts/audit.py']) + call('runtime-packages.txt', ['docker', 'compose', 'exec', '-T', 'api', 'pip', 'freeze']) + snapshot('before-restart.json') + suffix = 'cmd' if os.name == 'nt' else 'sh' + call('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart.' + suffix]) + snapshot('after-restart.json') + for repeat in (1, 2): + call('failover-' + str(repeat) + '.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one.' + suffix]) + call('restore-' + str(repeat) + '.txt', ['docker', 'compose', 'up', '-d', '--no-deps', 'api']) + ready() + call('smoke-final.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py']) + call('summarize.txt', [sys.executable, '-X', 'utf8', 'scripts/summarize.py']) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..083a7970 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,143 @@ +import asyncio +from contextlib import suppress +import hashlib +import tempfile +import unicodedata +import uuid +import warnings +from urllib.parse import quote + +from botocore.exceptions import ClientError +from fastapi import Request +from fastapi.responses import JSONResponse, StreamingResponse +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool + +from src.storage import client, BUCKET, PREFIX + + +def install(app, api): + def view(row): + return {key: value for key, value in row.items() if key != 'object_key'} | {'variants': []} + + def authorize(request, chat_id): + with api.POOL.connection() as db: + user, _ = api.authenticate(request, db) + api.access(db, api.identifier(chat_id), user['id']) + return user['id'] + + def inspect_image(file, content_type): + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + file.seek(0) + with Image.open(file) as image: + if image.width * image.height > api.MAX_PIXELS: + raise api.Problem(413, 'payload_too_large') + if {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) != content_type: + raise api.Problem(415, 'unsupported_media_type') + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise api.Problem(413, 'payload_too_large') + except (UnidentifiedImageError, OSError, SyntaxError, ValueError): + raise api.Problem(415, 'unsupported_media_type') + finally: + file.seek(0) + + @app.post('/api/v1/chats/{chat_id}/attachments') + async def upload(request: Request, chat_id: str): + owner_id = await run_in_threadpool(authorize, request, chat_id) + name = request.query_params.get('filename', '') + if set(request.query_params) != {'filename'} or not 1 <= len(name) <= 255 or not name.strip() or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in name): + api.invalid() + mime = request.headers.get('content-type', '').split(';')[0].lower().strip() + if mime not in ('image/png', 'image/jpeg'): + raise api.Problem(415, 'unsupported_media_type') + declared = request.headers.get('content-length') + if declared is not None: + if not declared.isdigit(): + api.invalid() + if int(declared) > api.MAX_BYTES: + raise api.Problem(413, 'payload_too_large') + try: + await asyncio.wait_for(api.UPLOAD_SLOTS.acquire(), timeout=2) + except TimeoutError: + raise api.Problem(503, 'unavailable') + try: + with tempfile.TemporaryFile() as file: + digest = hashlib.sha256() + size = 0 + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > api.MAX_BYTES: + raise api.Problem(413, 'payload_too_large') + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + await run_in_threadpool(inspect_image, file, mime) + attachment_id = uuid.uuid4() + key = PREFIX + str(attachment_id) + def save(): + s3 = client() + with api.POOL.connection() as db: + db.execute('SELECT pg_advisory_xact_lock_shared(202604)') + s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size, ContentType=mime) + try: + row = db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', (attachment_id, api.identifier(chat_id), owner_id, name, key, mime, size, digest.hexdigest())).fetchone() + except Exception: + with suppress(Exception): + s3.delete_object(Bucket=BUCKET, Key=key) + raise + db.commit() + return row + row = await run_in_threadpool(save) + return JSONResponse(api.clean(view(row)), status_code=201) + except TimeoutError: + raise api.Problem(400, 'invalid_request', 'Upload timeout') + finally: + api.UPLOAD_SLOTS.release() + + def find(request, attachment_id): + with api.POOL.connection() as db: + user, _ = api.authenticate(request, db) + row = db.execute('SELECT * FROM attachments WHERE id=%s', (api.identifier(attachment_id),)).fetchone() + if not row: + raise api.Problem(404, 'not_found') + api.access(db, row['chat_id'], user['id']) + if row['status'] != 'ready': + raise api.Problem(503, 'unavailable') + return row + + @app.get('/api/v1/attachments/{attachment_id}') + async def metadata(request: Request, attachment_id: str): + return JSONResponse(api.clean(view(await run_in_threadpool(find, request, attachment_id)))) + + @app.get('/api/v1/attachments/{attachment_id}/content') + async def content(request: Request, attachment_id: str): + row = await run_in_threadpool(find, request, attachment_id) + if row['status'] != 'ready': + raise api.Problem(409, 'conflict') + if set(request.query_params) - {'variant'}: + api.invalid() + if request.query_params.get('variant', 'original') != 'original': + raise api.Problem(404, 'not_found') + try: + result = await run_in_threadpool(lambda: client().get_object(Bucket=BUCKET, Key=row['object_key'])) + except ClientError as exc: + if exc.response['Error']['Code'] in ('NoSuchKey', '404'): + raise api.Problem(503, 'unavailable') + raise + stream = result['Body'] + def chunks(): + try: + while chunk := stream.read(65536): + yield chunk + finally: + stream.close() + return StreamingResponse(chunks(), media_type=row['content_type'], headers={ + 'Content-Length': str(row['size_bytes']), + 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(row['filename'], safe=''), + 'X-Content-Type-Options': 'nosniff'}) diff --git a/src/bootstrap_storage.py b/src/bootstrap_storage.py new file mode 100644 index 00000000..84c6c799 --- /dev/null +++ b/src/bootstrap_storage.py @@ -0,0 +1,32 @@ +import time +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from src.storage import BUCKET + + +def main(): + s3 = boto3.client('s3', endpoint_url='http://s3:9000', region_name='us-east-1', + aws_access_key_id=Path('/run/secrets/minio_root_user').read_text().strip(), + aws_secret_access_key=Path('/run/secrets/minio_root_password').read_text().strip(), + config=Config(connect_timeout=2, read_timeout=3, retries={'total_max_attempts': 1})) + for _ in range(45): + try: + try: + s3.head_bucket(Bucket=BUCKET) + except ClientError as exc: + if exc.response['Error']['Code'] not in ('404', 'NoSuchBucket', 'NotFound'): + raise + s3.create_bucket(Bucket=BUCKET) + print('Private bucket prepared; identities loaded from S3 configuration') + return + except (BotoCoreError, ClientError): + time.sleep(1) + raise RuntimeError('S3 bootstrap timeout') + + +if __name__ == '__main__': + main() diff --git a/src/healthcheck.py b/src/healthcheck.py new file mode 100644 index 00000000..e0bc86b4 --- /dev/null +++ b/src/healthcheck.py @@ -0,0 +1,9 @@ +import socket + + +with socket.create_connection(('127.0.0.1', 8080), timeout=1) as connection: + connection.sendall(b'GET /health/ready HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n') + with connection.makefile('rb') as stream: + status = stream.readline(128) + if status.split(b' ', 2)[1:2] != [b'200']: + raise SystemExit(1) diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..2eda67c3 --- /dev/null +++ b/src/main.py @@ -0,0 +1,367 @@ +import base64 +import asyncio +from contextlib import asynccontextmanager, suppress +from datetime import datetime, timezone +import hashlib +import json +import logging +import os +import re +import secrets +import uuid + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from PIL import Image +from botocore.exceptions import BotoCoreError, ClientError +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, ConnectionPool, PoolTimeout +from starlette.concurrency import run_in_threadpool + +from src.migrate import database_url +from src.storage import client, BUCKET + + +INSTANCE = os.environ.get('HOSTNAME', str(uuid.uuid4())) +PASSWORDS = PasswordHasher() +DUMMY_HASH = PASSWORDS.hash(secrets.token_hex(32)) +POOL = ConnectionPool(database_url(), min_size=1, max_size=int(os.getenv('DB_POOL_SIZE', '1')), open=False, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}) +HEALTH_POOL = AsyncConnectionPool(database_url(), min_size=1, max_size=1, open=False, timeout=0.5, kwargs={'connect_timeout': 1, 'options': '-c statement_timeout=500 -c lock_timeout=500'}) +MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) +MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) +UPLOAD_SLOTS = asyncio.Semaphore(2) +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +@asynccontextmanager +async def lifespan(app): + POOL.open() + POOL.wait(timeout=20) + await HEALTH_POOL.open() + await HEALTH_POOL.wait(timeout=20) + async def record_pool(): + while True: + await asyncio.sleep(5) + print(json.dumps({'event': 'pool_metrics', 'instance': INSTANCE, 'at': datetime.now(timezone.utc).isoformat(), 'stats': POOL.get_stats()}), flush=True) + recorder = asyncio.create_task(record_pool()) + yield + recorder.cancel() + with suppress(asyncio.CancelledError): + await recorder + await HEALTH_POOL.close() + POOL.close() + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +class Problem(Exception): + def __init__(self, status, code, message='Request cannot be completed'): + self.status, self.code, self.message = status, code, message + + +def invalid(): + raise Problem(400, 'invalid_request') + + +def clean(value): + if isinstance(value, dict): + return {key: clean(item) for key, item in value.items()} + if isinstance(value, list): + return [clean(item) for item in value] + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z') + return value + + +@app.middleware('http') +async def headers(request, call_next): + request.state.request_id = str(uuid.uuid4()) + try: + response = await call_next(request) + except Exception: + logging.exception('Unhandled request failure') + response = error(request, Problem(500, 'internal_error')) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = INSTANCE + return response + + +def error(request, exc): + headers = {} + if exc.status == 401: + headers['WWW-Authenticate'] = 'Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"' + return JSONResponse({'error': {'code': exc.code, 'message': exc.message, 'request_id': request.state.request_id}}, status_code=exc.status, headers=headers) + + +@app.exception_handler(Problem) +async def problem_handler(request, exc): + return error(request, exc) + + +@app.exception_handler(psycopg.Error) +@app.exception_handler(PoolTimeout) +@app.exception_handler(BotoCoreError) +@app.exception_handler(ClientError) +async def database_handler(request, exc): + return error(request, Problem(503, 'unavailable')) + + +@app.exception_handler(404) +@app.exception_handler(405) +async def missing_handler(request, exc): + return error(request, Problem(404, 'not_found')) + + +async def body(request, fields, empty_attachments=False): + if request.headers.get('content-type', '').split(';')[0].strip().lower() != 'application/json': + raise Problem(415, 'unsupported_media_type') + raw = bytearray() + async for chunk in request.stream(): + if len(raw) + len(chunk) > 65536: + raise Problem(413, 'payload_too_large') + raw.extend(chunk) + try: + value = json.loads(raw) + except (ValueError, UnicodeError): + invalid() + if empty_attachments and isinstance(value, dict) and 'attachment_ids' in value: + fields = (*fields, 'attachment_ids') + if not isinstance(value, dict) or set(value) != set(fields): + invalid() + if empty_attachments: + value.setdefault('attachment_ids', []) + return value + + +def text(value, maximum, minimum=1): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + return value + + +def identifier(value): + try: + return uuid.UUID(value) + except (ValueError, TypeError, AttributeError): + invalid() + + +USER_COLUMNS = 'id, username, display_name, created_at' + + +def authenticate(request, db): + authorization = request.headers.get('authorization', '') + if not authorization.startswith('Bearer ') or not authorization[7:]: + raise Problem(401, 'unauthorized') + digest = hashlib.sha256(authorization[7:].encode()).hexdigest() + user = db.execute('SELECT u.* FROM users u JOIN sessions s ON s.user_id=u.id WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()', (digest,)).fetchone() + if not user: + raise Problem(401, 'unauthorized') + return user, digest + + +def user_view(user): + return {key: user[key] for key in ('id', 'username', 'display_name', 'created_at')} + + +def access(db, chat_id, user_id): + if not db.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone(): + raise Problem(404, 'not_found') + + +def chat_view(db, chat): + chat['member_ids'] = [row['user_id'] for row in db.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))] + return chat + + +def pagination(request, scope): + if set(request.query_params) - {'limit', 'cursor'}: + invalid() + raw_limit = request.query_params.get('limit', '50') + if not re.fullmatch('[0-9]+', raw_limit) or not 1 <= int(raw_limit) <= 100: + invalid() + cursor = request.query_params.get('cursor') + position = None + if cursor is not None: + try: + resource, when, item = json.loads(base64.urlsafe_b64decode(cursor + '=' * (-len(cursor) % 4))) + if resource != scope: + invalid() + position = (datetime.fromisoformat(when.replace('Z', '+00:00')), identifier(item)) + if position[0].tzinfo is None: + invalid() + except (ValueError, TypeError, UnicodeError, OverflowError): + invalid() + return int(raw_limit), position + + +def page(rows, limit, scope): + selected = rows[:limit] + cursor = None + if len(rows) > limit: + last = clean(selected[-1]) + cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=') + return {'items': selected, 'next_cursor': cursor} + + +@app.get('/health/live') +def live(): + return {'status': 'ok'} + + +@app.get('/health/ready') +async def ready(): + async with HEALTH_POOL.connection() as db: + await db.execute('SELECT 1 FROM users LIMIT 1') + await run_in_threadpool(lambda: client().head_bucket(Bucket=BUCKET)) + return {'status': 'ok'} + + +@app.post('/api/v1/users') +async def register(request: Request): + data = await body(request, ('username', 'display_name', 'password')) + if not isinstance(data['username'], str) or not re.fullmatch('[a-z0-9_]{3,32}', data['username']): + invalid() + text(data['display_name'], 100) + text(data['password'], 128, 12) + def create(): + hashed = PASSWORDS.hash(data['password']) + try: + with POOL.connection() as db: + return db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING ' + USER_COLUMNS, (uuid.uuid4(), data['username'], data['display_name'], hashed)).fetchone() + except psycopg.errors.UniqueViolation: + raise Problem(409, 'conflict') + return JSONResponse(clean(await run_in_threadpool(create)), status_code=201) + + +@app.post('/api/v1/auth/sessions') +async def login(request: Request): + authorization = request.headers.get('authorization', '') + try: + if not authorization.startswith('Basic '): + raise ValueError() + username, password = base64.b64decode(authorization[6:], validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError): + raise Problem(401, 'unauthorized') + if await request.body(): + invalid() + def create(): + with POOL.connection() as db: + user = db.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone() + try: + PASSWORDS.verify(user['password_hash'] if user else DUMMY_HASH, password) + except VerificationError: + raise Problem(401, 'unauthorized') + if not user: + raise Problem(401, 'unauthorized') + token = secrets.token_urlsafe(32) + expiry = db.execute("INSERT INTO sessions VALUES(%s,%s,clock_timestamp()+(%s * interval '1 second')) RETURNING expires_at", (hashlib.sha256(token.encode()).hexdigest(), user['id'], int(os.getenv('SESSION_TTL_SECONDS', '3600')))).fetchone()['expires_at'] + return {'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user_view(user)} + return JSONResponse(clean(await run_in_threadpool(create)), status_code=201) + + +from src.attachments import install +import sys + +install(app, sys.modules[__name__]) + + +@app.api_route('/api/v1/{path:path}', methods=['GET', 'POST', 'DELETE']) +async def business(request: Request, path: str): + data = None + if request.method == 'POST': + def verify_identity(): + with POOL.connection() as db: + authenticate(request, db) + await run_in_threadpool(verify_identity) + data = await body(request, ('title', 'member_ids') if path == 'chats' else ('text',), empty_attachments=path.endswith('/messages')) + def execute(): + with POOL.connection() as db: + user, digest = authenticate(request, db) + if path == 'users/me' and request.method == 'GET': + return user_view(user) + if path == 'auth/sessions/current' and request.method == 'DELETE': + db.execute('DELETE FROM sessions WHERE token_hash=%s', (digest,)) + return None + if path == 'chats' and request.method == 'POST': + text(data['title'], 100) + ids = data['member_ids'] + if not isinstance(ids, list) or not 1 <= len(ids) <= 100: + invalid() + ids = [identifier(value) for value in ids] + if len(set(ids)) != len(ids) or user['id'] not in ids: + invalid() + if db.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s)', (ids,)).fetchone()['n'] != len(ids): + invalid() + chat = db.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *', (uuid.uuid4(), data['title'])).fetchone() + for member in ids: + db.execute('INSERT INTO members VALUES(%s,%s)', (chat['id'], member)) + return chat_view(db, chat) + if path == 'chats' and request.method == 'GET': + scope = 'chats:' + str(user['id']) + limit, position = pagination(request, scope) + query = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + values = [user['id']] + if position: + query += ' AND (c.created_at,c.id)>(%s,%s)' + values.extend(position) + rows = db.execute(query + ' ORDER BY c.created_at,c.id LIMIT %s', values + [limit + 1]).fetchall() + return page([chat_view(db, row) for row in rows], limit, scope) + parts = path.split('/') + if len(parts) not in (2, 3) or parts[0] != 'chats': + raise Problem(404, 'not_found') + chat_id = identifier(parts[1]) + access(db, chat_id, user['id']) + if len(parts) == 2 and request.method == 'GET': + return chat_view(db, db.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()) + if len(parts) != 3 or parts[2] != 'messages': + raise Problem(404, 'not_found') + if request.method == 'GET': + scope = 'messages:' + str(chat_id) + limit, position = pagination(request, scope) + query = 'SELECT * FROM messages WHERE chat_id=%s' + values = [chat_id] + if position: + query += ' AND (created_at,id)>(%s,%s)' + values.extend(position) + rows = db.execute(query + ' ORDER BY created_at,id LIMIT %s', values + [limit + 1]).fetchall() + return page(rows, limit, scope) + if request.method != 'POST': + raise Problem(404, 'not_found') + attachments = data['attachment_ids'] + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [identifier(item) for item in attachments] + if len(set(attachments)) != len(attachments): + invalid() + if not isinstance(data['text'], str) or len(data['text']) > 4000 or (not data['text'].strip() and not attachments): + invalid() + for item in attachments: + row = db.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (item, chat_id)).fetchone() + if not row: + raise Problem(404, 'not_found') + if row['status'] != 'ready': + raise Problem(409, 'conflict') + canonical = json.dumps({'text': data['text'], 'attachment_ids': [str(item) for item in attachments]}, sort_keys=True) + key = request.headers.get('idempotency-key', '') + if not re.fullmatch('[A-Za-z0-9._:-]{1,128}', key): + invalid() + lock = str(user['id']) + ':' + str(chat_id) + ':' + key + db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (lock,)) + db.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()', (user['id'], chat_id, key)) + previous = db.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', (user['id'], chat_id, key)).fetchone() + if previous: + if previous['body'] != canonical: + raise Problem(409, 'conflict') + return db.execute('SELECT * FROM messages WHERE id=%s', (previous['message_id'],)).fetchone() + message = db.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES(%s,%s,%s,%s,%s) RETURNING *', (uuid.uuid4(), chat_id, user['id'], data['text'], attachments)).fetchone() + db.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", (user['id'], chat_id, key, canonical, message['id'])) + return message + result = await run_in_threadpool(execute) + return Response(status_code=204) if result is None else JSONResponse(clean(result), status_code=201 if request.method == 'POST' else 200) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..67d373ab --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,18 @@ +from pathlib import Path +import os +import psycopg + + +def database_url(): + return Path(os.environ.get('DATABASE_URL_FILE', '/run/secrets/database_url')).read_text().strip() + + +def main(): + with psycopg.connect(database_url()) as connection: + connection.execute('SELECT pg_advisory_xact_lock(202603)') + for migration in sorted(Path('migrations').glob('*.sql')): + connection.execute(migration.read_text()) + + +if __name__ == '__main__': + main() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..50f5d775 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,71 @@ +import argparse +from datetime import datetime, timedelta, timezone +import json +import time + +from botocore.exceptions import ClientError +import psycopg +from psycopg.rows import dict_row + +from src.migrate import database_url +from src.storage import client, BUCKET, PREFIX + + +def run(apply=False, grace_seconds=3600): + if grace_seconds < 60: + raise ValueError('Grace period must be at least 60 seconds') + started = time.monotonic() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + report = dict(apply=apply, grace_seconds=grace_seconds, objects=0, metadata=0, + list_requests=0, head_requests=0, orphan_candidates=0, missing_candidates=0, + deleted=0, marked_failed=0, fresh_preserved=0, referenced_preserved=0) + s3 = client() + with psycopg.connect(database_url(), row_factory=dict_row) as db: + db.execute("SET lock_timeout='5s'") + db.execute('SELECT pg_advisory_xact_lock(202604)') + for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX): + report['list_requests'] += 1 + for obj in page.get('Contents', []): + report['objects'] += 1 + if db.execute('SELECT 1 FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone(): + report['referenced_preserved'] += 1 + continue + if obj['LastModified'] >= cutoff: + report['fresh_preserved'] += 1 + continue + report['head_requests'] += 1 + current = s3.head_object(Bucket=BUCKET, Key=obj['Key']) + if current['LastModified'] >= cutoff: + report['fresh_preserved'] += 1 + continue + report['orphan_candidates'] += 1 + if apply: + s3.delete_object(Bucket=BUCKET, Key=obj['Key']) + report['deleted'] += 1 + with db.cursor(name='attachments_scan') as cursor: + cursor.execute("SELECT id,object_key,created_at FROM attachments WHERE status='ready' ORDER BY id") + while rows := cursor.fetchmany(100): + for row in rows: + report['metadata'] += 1 + if row['created_at'] >= cutoff: + continue + report['head_requests'] += 1 + try: + s3.head_object(Bucket=BUCKET, Key=row['object_key']) + except ClientError as exc: + if exc.response['Error']['Code'] not in ('NoSuchKey', '404', 'NotFound'): + raise + report['missing_candidates'] += 1 + if apply: + db.execute("UPDATE attachments SET status='missing',error_code='original_missing',updated_at=clock_timestamp() WHERE id=%s AND status='ready'", (row['id'],)) + report['marked_failed'] += 1 + report['elapsed_seconds'] = round(time.monotonic() - started, 6) + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + args = parser.parse_args() + print(json.dumps(run(args.apply, args.grace_seconds))) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..86c99a69 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,21 @@ +import os +from pathlib import Path + +import boto3 +from botocore.config import Config + + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +PREFIX = 'originals/' + + +def secret(name): + return Path(os.getenv(name + '_FILE', '/run/secrets/' + name.lower())).read_text().strip() + + +def client(): + return boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'), + aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'), + config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required', + s3={'addressing_style': 'path'})) diff --git a/student-tests/acceptance.py b/student-tests/acceptance.py new file mode 100644 index 00000000..ed5b4aae --- /dev/null +++ b/student-tests/acceptance.py @@ -0,0 +1,176 @@ +import concurrent.futures +import hashlib +import http.client +import json +import os +from pathlib import Path +import statistics +import struct +import subprocess +import sys +import threading +import time +import urllib.parse +import uuid +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def docker(*args, input=None): + result = subprocess.run(['docker', *args], cwd=ROOT, input=input, text=True, capture_output=True, check=True) + return result.stdout.strip() + + +def internal(code): + return docker('compose', 'exec', '-T', 'api', 'python', '-', input=code) + + +def png(width, height, random=False): + def chunk(kind, data): + return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data)) + raw = b''.join(b'\0' + (os.urandom(width * 3) if random else b'\x80' * (width * 3)) for _ in range(height)) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!2I5B', width, height, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'') + + +def wait_ready(): + for _ in range(90): + try: + Client().call('GET', '/health/ready', schema='Health') + return + except Exception: + time.sleep(1) + raise AssertionError('Readiness timeout') + + +def main(): + evidence = {} + c = Client(timeout=40) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=fixture.png' + sample = (ROOT / 'tests/fixtures/sample.png').read_bytes() + def upload(data=sample, mime='image/png', expected=201, actor=alice, route=path): + return Client(timeout=40).call('POST', route, expected, actor=actor, raw=data, + headers={'Content-Type': mime}, schema='Attachment' if expected == 201 else None) + att = upload()['json'] + upload(b'x' * (10485760 + 1), expected=413) + upload(b'not an image', expected=415) + upload(sample[:-20], expected=415) + upload(sample, 'image/jpeg', 415) + upload(actor=eve, expected=404) + upload(actor=None, expected=401) + for name in ('../bad.png', 'bad\\x.png', 'bad\x00.png', 'bad\n.png', ' '): + upload(expected=400, route=path.split('?')[0] + '?filename=' + urllib.parse.quote(name)) + other = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'Other', 'member_ids': [eve['id']]}, schema='Chat')['json'] + foreign = upload(actor=eve, route='/api/v1/chats/' + other['id'] + '/attachments?filename=x.png')['json'] + for item in (foreign['id'], str(uuid.uuid4())): + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 404, actor=alice, + body={'text': '', 'attachment_ids': [item]}, headers={'Idempotency-Key': uuid.uuid4().hex}) + huge = bytearray(sample) + huge[16:24] = struct.pack('!II', 40000001, 1) + huge[29:33] = struct.pack('!I', zlib.crc32(huge[12:29])) + upload(bytes(huge), expected=413) + jpeg = internal("from PIL import Image\nimport io,base64\nf=io.BytesIO(); Image.new('RGB',(32,32)).save(f,'JPEG'); print(base64.b64encode(f.getvalue()).decode())") + import base64 + jpg = base64.b64decode(jpeg) + uploaded_jpeg = upload(jpg, 'image/jpeg')['json'] + check(c.call('GET', '/api/v1/attachments/' + uploaded_jpeg['id'] + '/content', actor=bob)['bytes'] == jpg, 'JPEG changed') + evidence['negative_tests'] = 'PASS: byte/pixel limits, corrupt/truncated images, MIME mismatch, filename, ACL, foreign and unknown attachment' + evidence['private_bucket'] = json.loads(internal("""import urllib.request, urllib.error, json +from src.storage import client,BUCKET +try: + urllib.request.urlopen('http://s3:9000/'+BUCKET+'/originals/""" + att['id'] + """') + raise AssertionError('Anonymous read allowed') +except urllib.error.HTTPError as e: + assert e.code==403 + print(json.dumps({'anonymous_get':e.code})) +try: + client().create_bucket(Bucket='forbidden-bucket') + raise AssertionError('API can create bucket') +except Exception as e: + assert getattr(e,'response',{}).get('Error',{}).get('Code')=='AccessDenied' +""")) + orphan_key = 'originals/' + str(uuid.uuid4()) + missing = upload()['json'] + internal("""from src.storage import client,BUCKET +from src.migrate import database_url +import psycopg +s=client() +s.put_object(Bucket=BUCKET,Key='""" + orphan_key + """',Body=b'orphan') +s.delete_object(Bucket=BUCKET,Key='originals/""" + missing['id'] + """') +with psycopg.connect(database_url()) as db: + db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s", ('""" + missing['id'] + """',)) +""") + injected = time.monotonic() + evidence['uploads'] = [] + for width in (512, 1024): + data = png(width, width, True) + baseline = docker('stats', '--no-stream', '--format', '{{json .}}').splitlines() + observations = [] + stop = threading.Event() + def monitor(): + while not stop.is_set(): + observations.extend(json.loads(line) for line in docker('stats', '--no-stream', '--format', '{{json .}}').splitlines() if 'lab4-api' in line) + watcher = threading.Thread(target=monitor) + watcher.start() + started = time.monotonic() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(lambda _: upload(data), range(10))) + elapsed = time.monotonic() - started + finally: + stop.set() + watcher.join() + peak = {container: int(docker('exec', container, 'cat', '/sys/fs/cgroup/memory.peak')) + for container in docker('compose', 'ps', '-q', 'api').split()} + durations = sorted(row['elapsed'] for row in results) + evidence['uploads'].append(dict(size_bytes=len(data), concurrency=10, success=len(results), errors=0, + seconds=elapsed, successful_rps=10/elapsed, p50=statistics.median(durations), p95=durations[-1], p99=durations[-1], + baseline=[json.loads(line) for line in baseline if 'lab4-api' in line], samples=observations, lifetime_cgroup_peak_bytes=peak)) + connection = http.client.HTTPConnection('localhost', 8080, timeout=5) + connection.putrequest('POST', path) + connection.putheader('Authorization', 'Bearer ' + alice['token']) + connection.putheader('Content-Type', 'image/png') + connection.putheader('Content-Length', '100000') + connection.endheaders() + connection.send(sample[:100]) + connection.close() + evidence['interrupted_upload'] = 'Connection closed after 100 bytes of declared 100000; no S3 PUT before validation' + remaining = 62 - (time.monotonic() - injected) + if remaining > 0: + while remaining > 0: + time.sleep(min(remaining, 20)) + remaining = 62 - (time.monotonic() - injected) + fresh_key = 'originals/' + str(uuid.uuid4()) + internal("from src.storage import client,BUCKET\nclient().put_object(Bucket=BUCKET,Key='" + fresh_key + "',Body=b'fresh')") + def reconcile(apply): + return json.loads(internal('from src.reconcile import run\nimport json\nprint(json.dumps(run(' + str(apply) + ',60)))')) + dry = reconcile(False) + check(dry['orphan_candidates'] >= 1 and dry['missing_candidates'] >= 1 and dry['deleted'] == 0, 'Dry-run failed') + applied = reconcile(True) + check(applied['deleted'] >= 1 and applied['marked_failed'] >= 1, 'Repair failed') + repeated = reconcile(True) + check(repeated['deleted'] == 0 and repeated['marked_failed'] == 0, 'Reconcile is not idempotent') + check(applied['fresh_preserved'] >= 1 and applied['referenced_preserved'] >= 1, 'Safety failed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == sample, 'Referenced file removed') + c.call('GET', '/api/v1/attachments/' + missing['id'], 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + missing['id'] + '/content', 503, actor=bob) + evidence['reconciler'] = {'dry_run': dry, 'apply': applied, 'repeat': repeated} + docker('compose', 'stop', 's3') + try: + response = c.call('GET', '/health/ready', 503) + evidence['s3_outage'] = {'ready': response['status']} + internal("from src.reconcile import run\ntry:\n run(True,60)\nexcept Exception:\n print('PASS: outage fails closed')\nelse:\n raise AssertionError('Expected failure')") + finally: + docker('compose', 'start', 's3') + wait_ready() + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == sample, 'S3 restart changed data') + internal("from src.storage import client,BUCKET\nclient().delete_object(Bucket=BUCKET,Key='" + fresh_key + "')") + (ROOT / 'evidence/acceptance.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: negative images, ACL, private S3, interrupted upload, concurrent uploads, reconciliation, S3 outage') + + +if __name__ == '__main__': + main() diff --git a/student-tests/consistency.py b/student-tests/consistency.py new file mode 100644 index 00000000..f465f54b --- /dev/null +++ b/student-tests/consistency.py @@ -0,0 +1,65 @@ +import concurrent.futures +import json +from pathlib import Path +import sys +import time +import uuid + +from acceptance import ROOT, docker, internal + +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def main(): + c = Client(timeout=40) + alice, bob, _, chat = c.bootstrap() + attachment, original = c.attachment(alice, chat) + message = c.message(alice, chat, '', attachments=[attachment['id']])['json'] + internal("""import psycopg +from src.migrate import database_url +with psycopg.connect(database_url()) as db: + db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s", ('""" + attachment['id'] + """',)) + db.execute("ALTER TABLE attachments ADD CONSTRAINT injected_database_failure CHECK(filename <> 'reject-db.png') NOT VALID") +""") + def count(): + return int(internal("from src.storage import client,BUCKET\nprint(sum(len(p.get('Contents',[])) for p in client().get_paginator('list_objects_v2').paginate(Bucket=BUCKET)))")) + before = count() + try: + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=reject-db.png', 503, + actor=alice, raw=original, headers={'Content-Type': 'image/png'}) + after = count() + check(before == after, 'S3 compensation left an orphan') + finally: + internal("import psycopg\nfrom src.migrate import database_url\nwith psycopg.connect(database_url()) as db:\n db.execute('ALTER TABLE attachments DROP CONSTRAINT injected_database_failure')") + results = {'database_failure_after_put': {'before_objects': before, 'after_objects': after, 'http_status': 503}, 'scan_cost': []} + prefix = 'originals/study-' + uuid.uuid4().hex + '-' + try: + for size in (10, 100): + internal("""import concurrent.futures +from src.storage import client,BUCKET +s=client() +with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + list(pool.map(lambda i:s.put_object(Bucket=BUCKET,Key='""" + prefix + """'+str(i),Body=b'fresh'),range(""" + str(size) + """))) +""") + observations = [] + for _ in range(3): + observations.append(json.loads(internal('from src.reconcile import run\nimport json\nprint(json.dumps(run(False,3600)))'))) + results['scan_cost'].append({'extra_objects': size, 'repeats': observations}) + applied = json.loads(internal('from src.reconcile import run\nimport json\nprint(json.dumps(run(True,3600)))')) + check(c.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob)['bytes'] == original, 'Used old attachment removed') + check(c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json']['items'][0] == message, 'Message reference changed') + results['old_used_attachment'] = {'checksum_matches': True, 'message_reference_preserved': True, 'apply': applied} + finally: + internal("""from src.storage import client,BUCKET +s=client() +for p in s.get_paginator('list_objects_v2').paginate(Bucket=BUCKET,Prefix='""" + prefix + """'): + for o in p.get('Contents',[]): + s.delete_object(Bucket=BUCKET,Key=o['Key']) +""") + (ROOT / 'evidence/consistency.json').write_text(json.dumps(results, indent=2), encoding='utf-8') + print('PASS: failure after S3 PUT compensates, old used attachment survives, scan cost measured twice with three repeats') + + +if __name__ == '__main__': + main()