diff --git a/.dockerignore b/.dockerignore index bc16e556..d58aec81 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +output .DS_Store diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c7af2262 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto +*.sh text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.sql text eol=lf +*.cmd text eol=crlf diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..5163db89 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application grade 3 +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: python scripts/bootstrap.py + - run: docker compose build api + - run: docker compose up -d --no-build + - run: python scripts/wait_ready.py + - run: python scripts/verify.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: grade3-evidence + path: evidence/ + - if: always() + run: docker compose down diff --git a/.gitignore b/.gitignore index b5a71896..9dd409a0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +/output/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bedb441d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.13.7-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src ./src +COPY migrations ./migrations +COPY contracts ./contracts +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY lab.json requirements-dev.txt ./ +USER app +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"] +CMD ["python", "-m", "messenger.server"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..a263ef03 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,134 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: **Осокин Глеб, БВТ2302**, Gitea `glebossokin`. +- Целевая оценка: **3**. Уровни 4/5 и исследовательские треки не заявлены. +- Ветка решения: [`solution/BVT2302-Osokin`](https://gitea.privcloud.tech/glebossokin/mtusi2026lab4/src/branch/solution/BVT2302-Osokin). Запрос на слияние направляется в `mtusi2026/mtusi2026lab4:main`. +- Дата полного прогона: **16.11.2026**. +- Перенос: собственная [ЛР3](https://gitea.privcloud.tech/glebossokin/mtusi2026lab3/src/branch/solution/BVT2302-Osokin), исходники приложения, миграции 001/002, Dockerfile, конфигурация HAProxy/Redis, проверка общей идемпотентности. Rolling update и исследования нагрузки предыдущей работы не переносились. +- Помощь ИИ: OpenAI Codex — реализация, подготовка отчёта и запуск проверок. Наблюдения получены на реально работающем Docker-стенде. Библиотеки: psycopg, Redis client, boto3, Pillow; первичные источники приведены ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Python 3.9+ для публичных проверок, Docker Engine с Compose v2+ и Linux containers. На Windows использованы Python 3.14.3 и Docker Desktop. Свободный порт **127.0.0.1:8084**; остальные порты не публикуются. При первой сборке нужен доступ к Docker Hub и PyPI. + +```powershell +git clone --branch solution/BVT2302-Osokin https://gitea.privcloud.tech/glebossokin/mtusi2026lab4.git +cd mtusi2026lab4 +python -X utf8 scripts/bootstrap.py +docker compose --env-file .env config --quiet +docker compose build api +docker compose --env-file .env up -d --no-build +python -X utf8 scripts/wait_ready.py +python -X utf8 scripts/verify.py +``` + +На Linux заменить `python -X utf8` на `python3`. У hooks `scripts/restart.sh` и `scripts/stop-one.sh` executable bit сохранён в Git; при необходимости выполнить `chmod +x scripts/restart.sh scripts/stop-one.sh`. Если Docker Desktop ещё не настроен на Linux engine, выбрать его командой `docker context use desktop-linux`. + +`bootstrap.py` создаёт локальные `.env` и `secrets/`: случайные пароль PostgreSQL, ключ HMAC cursor и credentials MinIO. Существующие значения сохраняются. Эти файлы исключены из Git и Docker build context; их содержимое не печатается. Не менять secrets при повторном использовании прежних volumes. + +`migrate` применяет SQL в одной транзакции с блокировкой и проверкой контрольных сумм. `bucket` создаёт приватный bucket перед API; anonymous/public policy не устанавливается. API запускаются после миграций, bootstrap bucket и готовности Redis. HAProxy проверяет readiness обеих реплик. После запуска LB нужно дождаться `wait_ready.py`. + +Отдельные публичные проверки на Windows: + +```powershell +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084 +python -X utf8 tests/resilience.py --mode persistence --base-url http://127.0.0.1:8084 --action-script scripts/restart.cmd +python -X utf8 tests/resilience.py --mode failover --base-url http://127.0.0.1:8084 --action-script scripts/stop-one.cmd +python -X utf8 scripts/labctl.py restore +``` + +Это прямые эквиваленты `make check`, `make test BASE_URL=http://127.0.0.1:8084` и `make test-persistence/test-failover` с соответствующим `ACTION_SCRIPT`. На Linux передавать `.sh` hooks. Контракт и публичные тесты не изменялись. Полный `verify.py` сохраняет результаты в `evidence/`, останавливает/восстанавливает API и MinIO, перезапускает работающий стенд. Все данные синтетические. После ручного failover реплика остаётся остановленной до `labctl.py restore`. + +Для другого host-порта изменить `APP_PORT` в `.env`, задать `BASE_URL` для `wait_ready.py`, `verify.py`, `grade3.py` и явно передать его публичным тестам. Безопасная остановка с сохранением данных: `docker compose down`. **Намеренное удаление всех учебных данных этого Compose-проекта:** `docker compose down --volumes`. При проверках отказов volumes не удаляются. У ЛР4 отдельное имя проекта `mtusi-lab4-osokin`. + +Закреплённые версии образов: Python `3.13.7-slim-bookworm`, PostgreSQL `17.6-bookworm`, Redis `7.4.5-bookworm`, HAProxy `3.0.16-alpine`, MinIO `RELEASE.2025-04-22T22-12-26Z`. Зависимости в `requirements.txt`, в том числе psycopg 3.2.10, psycopg-pool 3.2.6, Redis client 6.4.0, boto3 1.40.6, Pillow 11.3.0. Фактическое окружение и digest инфраструктурных образов: [environment.json](evidence/environment.json). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент → 127.0.0.1:8084 → HAProxy → API × 2 + ├─ PostgreSQL → pgdata + ├─ Redis → redisdata (сессии, AOF always) + └─ MinIO → s3data (приватный bucket) +``` + +PostgreSQL — источник истины для пользователей, чатов, участников, сообщений, результатов идемпотентных запросов и метаданных вложений. Общий Redis хранит серверные сессии с абсолютным expiry и AOF `appendfsync always`; restart не продлевает TTL. У API нет volume с оригиналами или обязательного локального состояния. Все зависимости находятся в общем `compose.yaml`, backend-сеть внутренняя. Публичный порт есть только у LB. Реальные процессы, порты, лимиты и mounts: [processes.txt](evidence/processes.txt). + +Сохранён минимум ЛР3: Bearer-сессии, scrypt, миграции PostgreSQL, keyset pagination по `(created_at,id)`, общие данные, `X-Instance-Id` и общий `Idempotency-Key` минимум на 24 часа. Advisory transaction lock защищает одновременные повторы: создание сообщения и сохранение результата атомарны. Отсутствующий `attachment_ids` и `[]` дают одинаковый fingerprint. Повтор заново проверяет сессию и участие в чате. + +Новое в ЛР4: + +- `003_attachments.sql`: Attachment с UUID, chat/owner, отображаемым filename, внутренним object_key, MIME, размером, SHA-256 и временем. В Message добавлен массив UUID вложений с пустым default, старые текстовые сообщения валидны. Приложение проверяет существование всех вложений именно этого чата внутри транзакции создания Message. Удаление вложений в обязательный API не входит. +- Binary POST принимает PNG/JPEG. До чтения body проверяются сессия, доступ к чату, filename и Content-Length ≤10 MiB. Transfer-Encoding и неправильная длина отклоняются. Чтение ограничено; временных файлов нет. Pillow открывает header лениво: формат и ≤40 млн пикселей проверяются до decode, MIME сверяется с содержимым; далее `verify()` и `load()` проверяют изображение. Filename 1–255 code points, без `/`, `\`, NUL и управляющих символов. +- Сервер независимо от filename генерирует случайный S3 key `originals/`, считает lowercase SHA-256 и отправляет неизменённые bytes в приватный bucket. После успешного PUT выполняются INSERT metadata и COMMIT. Только после этого возвращается **201 ready**, `variants=[]`, `error_code=null`. +- Metadata/content читаются только участником чата через JOIN с chat_members. Посторонний/неизвестный объект — 404. Download через авторизованный API, без redirect, с истинным MIME и проверкой размера/hash. `variant=original` поддерживается, других вариантов нет. Публичный API не раскрывает object key и credentials. + +Общей распределённой транзакции PostgreSQL/S3 нет. Если PUT не состоялся, metadata не создаются и 201 не выдаётся. После PUT до COMMIT возможен orphan. Потерянный HTTP-ответ может скрывать уже завершённую операцию; самостоятельный повтор upload может создать второй объект. Автоматическая компенсация, сверка и очистка с grace period не реализованы — это дополнительный уровень 4/5. + +`/health/live` проверяет процесс. `/health/ready` проверяет PostgreSQL, Redis и доступность bucket MinIO, поскольку S3 участвует в обязательном синхронном пути. Ошибки S3 преобразуются в 503 общего формата. При потере S3 HAProxy может исключить обе реплики; это консервативная readiness. В таком сценарии ответ самого API проверяется из контейнера без публикации порта в обход LB. + +| Переменные `.env.example` | Использование | +| --- | --- | +| APP_HOST / APP_PORT / APP_ENV | API внутри слушает 0.0.0.0:8080; APP_PORT задаёт host-порт LB (bootstrap: 8084); APP_ENV=lab | +| SESSION_TTL_SECONDS | TTL сессии 3600 s | +| SESSION_STORE / CACHE_URL_FILE | В Compose выбран redis вместо postgres из примера; адрес из secret-файла | +| DATABASE_URL_FILE | DSN из /run/secrets/database_url | +| API_REPLICAS | scale API, default 2 | +| IDEMPOTENCY_TTL_SECONDS | минимум 86400 s | +| S3_ENDPOINT / S3_REGION / S3_BUCKET | http://s3:9000 / us-east-1 / messenger-attachments | +| S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE | credentials через Compose secrets | +| MAX_UPLOAD_BYTES / MAX_IMAGE_PIXELS | 10485760 / 40000000; повышать сверх контракта нельзя | + +Дополнительно CURSOR_KEY_FILE задаёт общий ключ HMAC. У каждой API 4 workers и очередь до 32 соединений. Используются credentials учебного MinIO без отдельной ограниченной S3-роли уровня 4. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Целостность skeleton | `python -X utf8 scripts/check_contract.py` | PASS | [check.txt](evidence/check.txt) | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8084` | PASS: базовый API, upload ready, checksum и ACL | [smoke.txt](evidence/smoke.txt) | +| Минимум ЛР3: две реплики, общая сессия, идемпотентность | `docker compose exec -T api python student-tests/test_idempotency.py` | 16 одновременных повторов через 2 процесса создали 1 Message | [idempotency.txt](evidence/idempotency.txt) | +| PostgreSQL metadata, private S3 bytes, persistent volumes | `scripts/labctl.py state` и private probe в grade3.py | API без volume оригиналов, три data volumes; anonymous S3 GET=403 | [processes.txt](evidence/processes.txt), [private-and-formats.json](evidence/private-and-formats.json) | +| Реальные PNG/JPEG и byte-identical download | `python -X utf8 student-tests/grade3.py` | Оба формата приняты, оригиналы скачаны без изменения | [private-and-formats.json](evidence/private-and-formats.json) | +| Лимиты bytes/pixels до чтения/decode, filename | `docker compose exec -T api python student-tests/test_limits.py` | Точно 10 MiB и 40 млн пикселей приняты; превышение отклонено до read/decode; опасные имена отклонены | [limits.txt](evidence/limits.txt) | +| Upload → остановка именно загрузившей API → download другой | `python -X utf8 student-tests/grade3.py` | Разные X-Instance-Id, uploader exited, оригинал и Message доступны второму участнику | [upload-failover.json](evidence/upload-failover.json) | +| Чужой пользователь | Public smoke и grade3.py | Metadata/content возвращают 404 | [upload-failover.json](evidence/upload-failover.json), [smoke.txt](evidence/smoke.txt) | +| Restart всего стенда | Публичный persistence с `scripts/restart.cmd` | PASS с прежним токеном, metadata/bytes и сообщениями; StartedAt изменён, volumes прежние | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/restart.json) | +| Публичный failover с записью | Failover с `scripts/stop-one.cmd` | PASS, 99 записей, 4 переходные ошибки, восстановление 5.198 s, без потерь подтверждённых записей и дублей | [failover.txt](evidence/failover.txt), [stop-one.json](evidence/stop-one.json) | +| Недоступность S3 | `python -X utf8 student-tests/grade3.py` | live/metadata 200, ready/upload/content 503; после start прежний файл доступен | [s3-outage.json](evidence/s3-outage.json) | +| Стенд восстановлен | Финальный public smoke полного verify | PASS | [smoke-after.txt](evidence/smoke-after.txt) | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Публичные `tests/`, обязательные `contracts/`, README и COURSE сохранены. Проверки выполнены на реальных контейнерах. Уровни 4/5 не заявлены. Результаты полного прогона — в `evidence/`. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Исследования производительности уровня 4/5 не проводились. Функциональные проверки выполнялись с синтетическими данными. Граничные изображения проверены последовательно: PNG 10 MiB с сохранением trailing bytes и PNG 8000×5000. Это проверка допустимого входа, не исследование параллельной загрузки. + +Docker VM: 8 CPU, примерно 7.7 GiB RAM, Linux x86_64, Engine 29.8.2; host Windows. Лимиты: каждая API 1 CPU / 768 MiB; PostgreSQL 2 CPU / 512 MiB; Redis 0.5 CPU / 128 MiB; HAProxy 0.5 CPU / 128 MiB; MinIO 1 CPU / 512 MiB. Версии/digest/kernel — в environment.json. Перед отказами выполнены ожидание readiness и smoke, отдельного нагрузочного прогрева не было. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановить uploader API | Скачать оригинал через другую | X-Instance-Id различается, bytes совпадают, attached Message прочитан | Без потери подтверждённого файла; затем явный restore | +| Остановить MinIO | Не вернуть ложный 201, корректно сообщить недоступность | ready/upload/content=503; live/metadata=200; после start прежний токен скачал тот же файл | 20.312 s на весь сценарий с Docker CLI, проверками и восстановлением LB; original сохранён | +| Restart API, PostgreSQL, Redis, MinIO, LB | Сохранить сессию, metadata, bytes и сообщения | Public persistence PASS; StartedAt всех работающих сервисов изменён, volumes те же | Данные не потеряны, публичный тест дождался готовности в своём бюджете 90 s | +| Остановить API во время 15 s записи | Восстановить запись ≤10 s, сохранить подтверждения | Public failover PASS, 99 успешных записей, 4 переходные ошибки | 5.198 s; потерь подтверждённых сообщений/дублей нет | + +Public failover последовательно отправляет сообщения с паузой 0.1 s; неопределённый POST повторяет с прежним ключом. Эти результаты не выдаются за RPS-бенчмарк, p50/p95/p99 или пик производительности. 20.312 s для S3 включает команды Docker и проверки, а не только время восстановления S3. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host и одиночные PostgreSQL/Redis/LB/MinIO: потерю host или volume две API не перекрывают. +- Между S3 PUT и DB COMMIT возможен orphan. Автоматическая компенсация, reconciler и очистка с grace period не реализованы. +- Body буферизуется до 10 MiB, decode ограничен 40 млн пикселей. Нет эксперимента ≥10 одновременных uploads и измерения пика памяти; RAM-лимит контейнера не гарантирует успех любой комбинации больших конкурентных decode. +- API использует root credentials учебного MinIO. Отдельных ограниченных S3 credentials уровня 4 нет. S3/консоль не опубликованы на host в соответствии с общей изоляцией стенда. +- Presigned upload, thumbnails, worker, варианты изображений и исследовательские треки не реализованы. Дополнительная полная матрица негативных тестов уровня 4 не заявлена. +- HTTP допускается только для локального учебного стенда. Это обязательный минимум на 3 без заявления о production-ready/HA. + +## Источники + +- [Задание ЛР4](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4), локальные COURSE/CONTRIBUTING, OpenAPI, contracts/README и tests/README. +- [Собственная ЛР3](https://gitea.privcloud.tech/glebossokin/mtusi2026lab3/src/branch/solution/BVT2302-Osokin). +- [Pillow Image: open/verify/load и pixel limits](https://pillow.readthedocs.io/en/stable/reference/Image.html). +- [Boto3 S3 PutObject](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html). +- [MinIO](https://github.com/minio/minio). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..47b94fa6 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,130 @@ +name: mtusi-lab4-osokin +x-app: &app + image: mtusi-messenger:lab4 + build: . + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + CURSOR_KEY_FILE: /run/secrets/cursor_key + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKERS: 4 + secrets: [database_url, cache_url, cursor_key, s3_access_key_id, s3_secret_access_key] + mem_limit: 768m + cpus: 1 + dns_opt: ['timeout:1', 'attempts:1'] + networks: [backend] +services: + s3: + image: minio/minio:RELEASE.2025-04-22T22-12-26Z + command: [server, /data, --console-address, ':9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_access_key_id, s3_secret_access_key] + volumes: [s3data:/data] + healthcheck: + test: [CMD, curl, -f, http://localhost:9000/minio/health/live] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1 + networks: [backend] + db: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 2 + networks: [backend] + redis: + image: redis:7.4.5-bookworm + command: [redis-server, /usr/local/etc/redis/redis.conf] + volumes: + - redisdata:/data + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.5 + networks: [backend] + migrate: + <<: *app + command: [python, -m, messenger.migrate] + healthcheck: {disable: true} + depends_on: + db: {condition: service_healthy} + bucket: + <<: *app + command: [python, -m, messenger.bucket] + healthcheck: {disable: true} + depends_on: + s3: {condition: service_healthy} + api: + <<: *app + scale: ${API_REPLICAS:-2} + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + bucket: {condition: service_completed_successfully} + stop_grace_period: 10s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"] + interval: 2s + timeout: 5s + start_period: 5s + retries: 2 + lb: + image: haproxy:3.0.16-alpine + ports: ['127.0.0.1:${APP_PORT:-8080}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 3s + retries: 5 + stop_grace_period: 10s + mem_limit: 128m + cpus: 0.5 + networks: [frontend, backend] +volumes: + s3data: + pgdata: + redisdata: +networks: + frontend: + backend: + internal: true +secrets: + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + db_password: {file: ./secrets/db_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + cursor_key: {file: ./secrets/cursor_key} diff --git a/evidence/attachments-grade3.txt b/evidence/attachments-grade3.txt new file mode 100644 index 00000000..fdf90ade --- /dev/null +++ b/evidence/attachments-grade3.txt @@ -0,0 +1,7 @@ +$ python student-tests/grade3.py + Container mtusi-lab4-osokin-api-1 Running + Container mtusi-lab4-osokin-api-2 Starting + Container mtusi-lab4-osokin-api-2 Started +PASS: uploading API stopped; another API returned identical original and attached message. +PASS: real PNG/JPEG originals and anonymous S3 403. +PASS: unavailable S3 yields 503; restart restores the same original and existing session. diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9180d646 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/compose.txt b/evidence/compose.txt new file mode 100644 index 00000000..8b8dc967 --- /dev/null +++ b/evidence/compose.txt @@ -0,0 +1 @@ +$ docker compose --env-file .env config --quiet diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..03020b5f --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,28 @@ +{ + "verified_at_utc": "2026-11-16T12:17:45.593148+00:00", + "host_os": "Windows-10-10.0.19045-SP0", + "client_python": "3.14.3", + "docker": { + "ServerVersion": "29.8.2", + "OperatingSystem": "Docker Desktop", + "KernelVersion": "6.18.54.1-microsoft-standard-WSL2", + "NCPU": 8, + "MemTotal": 8275030016, + "Architecture": "x86_64" + }, + "images": { + "postgres:17.6-bookworm": [ + "postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3" + ], + "redis:7.4.5-bookworm": [ + "redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb" + ], + "haproxy:3.0.16-alpine": [ + "haproxy@sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf" + ], + "minio/minio:RELEASE.2025-04-22T22-12-26Z": [ + "minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e" + ] + }, + "app_python": "Python 3.13.7" +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..c07762af --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +$ python tests/resilience.py --mode failover --base-url http://127.0.0.1:8084 --action-script scripts/stop-one.cmd +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 4, "first_recovery_seconds": 5.198} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..bedaba5e --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,2 @@ +$ docker compose exec -T api python student-tests/test_idempotency.py +{"result": "PASS", "parallel_requests": 16, "api_instances": ["2026dc4e851d-ef8f1bfddac8", "63b31780a94f-17a31e0697e3"], "unique_messages": 1, "scope_conflict_logout": "PASS"} diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..d57dd523 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,3 @@ +$ docker compose exec -T api python student-tests/test_limits.py +PASS: 10 MiB and 40,000,000 pixels accepted; excess bytes rejected before read; +excess dimensions rejected before decode; unsafe filenames rejected; Unicode filename accepted. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..2ef21a95 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +$ python tests/resilience.py --mode persistence --base-url http://127.0.0.1:8084 --action-script scripts/restart.cmd +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private-and-formats.json b/evidence/private-and-formats.json new file mode 100644 index 00000000..b4a6a6f9 --- /dev/null +++ b/evidence/private-and-formats.json @@ -0,0 +1,15 @@ +{ + "anonymous_get": 403, + "real_images": [ + { + "format": "PNG", + "bytes": 95, + "sha256": "506fcf4116edf75bba71fe37e2317dbe2c2fed5c3ed12505f37b3d088d5572c5" + }, + { + "format": "JPEG", + "bytes": 645, + "sha256": "7cf106893f3c292cd0c385ed007e48e74919288e09e4674ab3c6efb1ff800ef7" + } + ] +} diff --git a/evidence/processes.txt b/evidence/processes.txt new file mode 100644 index 00000000..a8e0f075 --- /dev/null +++ b/evidence/processes.txt @@ -0,0 +1,155 @@ +$ python scripts/labctl.py state +[ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 12852, + "started_at": "2026-11-16T12:17:20.776941959Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 11211, + "started_at": "2026-11-16T12:16:40.901560588Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "6f7fa2489ec73b82edd15bd154a13fc5b6782a419b080588035554640799f0f5", + "name": "mtusi-lab4-osokin-bucket-1", + "service": "bucket", + "hostname": "6f7fa2489ec7", + "pid": 0, + "started_at": "2026-11-16T12:13:32.160735673Z", + "status": "exited", + "health": null, + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": {}, + "mounts": [] + }, + { + "id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1", + "name": "mtusi-lab4-osokin-db-1", + "service": "db", + "hostname": "7a3000f4e2e3", + "pid": 11171, + "started_at": "2026-11-16T12:16:40.543566533Z", + "status": "running", + "health": "healthy", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "memory_limit": 536870912, + "cpu_limit": 2.0, + "published_ports": { + "5432/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_pgdata", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688", + "name": "mtusi-lab4-osokin-lb-1", + "service": "lb", + "hostname": "f69eb31bce35", + "pid": 11295, + "started_at": "2026-11-16T12:16:41.324174766Z", + "status": "running", + "health": "healthy", + "image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "mounts": [] + }, + { + "id": "e1f68174df2ed177dc6a002b23228d3859bdcf83b0734d9e6f7bdc8fcce4c81e", + "name": "mtusi-lab4-osokin-migrate-1", + "service": "migrate", + "hostname": "e1f68174df2e", + "pid": 0, + "started_at": "2026-11-16T12:13:31.670346003Z", + "status": "exited", + "health": null, + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": {}, + "mounts": [] + }, + { + "id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922", + "name": "mtusi-lab4-osokin-redis-1", + "service": "redis", + "hostname": "ae03f444cf66", + "pid": 11162, + "started_at": "2026-11-16T12:16:40.390083828Z", + "status": "running", + "health": "healthy", + "image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "6379/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_redisdata", + "destination": "/data" + } + ] + }, + { + "id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900", + "name": "mtusi-lab4-osokin-s3-1", + "service": "s3", + "hostname": "c49b92fdbaae", + "pid": 11084, + "started_at": "2026-11-16T12:16:40.108761513Z", + "status": "running", + "health": "healthy", + "image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "memory_limit": 536870912, + "cpu_limit": 1.0, + "published_ports": { + "9000/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_s3data", + "destination": "/data" + } + ] + } +] diff --git a/evidence/restart.json b/evidence/restart.json new file mode 100644 index 00000000..ed0f2deb --- /dev/null +++ b/evidence/restart.json @@ -0,0 +1,250 @@ +{ + "before": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 3801, + "started_at": "2026-11-16T12:13:39.391673918Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 8365, + "started_at": "2026-11-16T12:15:31.242158531Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1", + "name": "mtusi-lab4-osokin-db-1", + "service": "db", + "hostname": "7a3000f4e2e3", + "pid": 811, + "started_at": "2026-11-16T12:12:04.793750358Z", + "status": "running", + "health": "healthy", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "memory_limit": 536870912, + "cpu_limit": 2.0, + "published_ports": { + "5432/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_pgdata", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688", + "name": "mtusi-lab4-osokin-lb-1", + "service": "lb", + "hostname": "f69eb31bce35", + "pid": 4289, + "started_at": "2026-11-16T12:13:52.556515025Z", + "status": "running", + "health": "healthy", + "image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "mounts": [] + }, + { + "id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922", + "name": "mtusi-lab4-osokin-redis-1", + "service": "redis", + "hostname": "ae03f444cf66", + "pid": 959, + "started_at": "2026-11-16T12:12:07.441668202Z", + "status": "running", + "health": "healthy", + "image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "6379/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_redisdata", + "destination": "/data" + } + ] + }, + { + "id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900", + "name": "mtusi-lab4-osokin-s3-1", + "service": "s3", + "hostname": "c49b92fdbaae", + "pid": 10343, + "started_at": "2026-11-16T12:16:24.37991011Z", + "status": "running", + "health": "healthy", + "image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "memory_limit": 536870912, + "cpu_limit": 1.0, + "published_ports": { + "9000/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_s3data", + "destination": "/data" + } + ] + } + ], + "after": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 11286, + "started_at": "2026-11-16T12:16:41.122280354Z", + "status": "running", + "health": "starting", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 11211, + "started_at": "2026-11-16T12:16:40.901560588Z", + "status": "running", + "health": "starting", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "7a3000f4e2e3053f9be9a1e14172ad606d5cca86c2c1b6e2f6c10720c01444e1", + "name": "mtusi-lab4-osokin-db-1", + "service": "db", + "hostname": "7a3000f4e2e3", + "pid": 11171, + "started_at": "2026-11-16T12:16:40.543566533Z", + "status": "running", + "health": "healthy", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "memory_limit": 536870912, + "cpu_limit": 2.0, + "published_ports": { + "5432/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_pgdata", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "id": "f69eb31bce350053a70a069732156ba5fea9360300676dbe567c017d6c1c9688", + "name": "mtusi-lab4-osokin-lb-1", + "service": "lb", + "hostname": "f69eb31bce35", + "pid": 11295, + "started_at": "2026-11-16T12:16:41.324174766Z", + "status": "running", + "health": "starting", + "image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "mounts": [] + }, + { + "id": "ae03f444cf66bca154f467aea2333362aeea77fed94f819369fb5176a00ae922", + "name": "mtusi-lab4-osokin-redis-1", + "service": "redis", + "hostname": "ae03f444cf66", + "pid": 11162, + "started_at": "2026-11-16T12:16:40.390083828Z", + "status": "running", + "health": "healthy", + "image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb", + "memory_limit": 134217728, + "cpu_limit": 0.5, + "published_ports": { + "6379/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_redisdata", + "destination": "/data" + } + ] + }, + { + "id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900", + "name": "mtusi-lab4-osokin-s3-1", + "service": "s3", + "hostname": "c49b92fdbaae", + "pid": 11084, + "started_at": "2026-11-16T12:16:40.108761513Z", + "status": "running", + "health": "starting", + "image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "memory_limit": 536870912, + "cpu_limit": 1.0, + "published_ports": { + "9000/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_s3data", + "destination": "/data" + } + ] + } + ] +} diff --git a/evidence/restore.txt b/evidence/restore.txt new file mode 100644 index 00000000..07c0884a --- /dev/null +++ b/evidence/restore.txt @@ -0,0 +1,4 @@ +$ python scripts/labctl.py restore + Container mtusi-lab4-osokin-api-2 Running + Container mtusi-lab4-osokin-api-1 Starting + Container mtusi-lab4-osokin-api-1 Started diff --git a/evidence/s3-outage.json b/evidence/s3-outage.json new file mode 100644 index 00000000..afb9a380 --- /dev/null +++ b/evidence/s3-outage.json @@ -0,0 +1,59 @@ +{ + "before": [ + { + "id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900", + "name": "mtusi-lab4-osokin-s3-1", + "service": "s3", + "hostname": "c49b92fdbaae", + "pid": 891, + "started_at": "2026-11-16T12:12:06.319676602Z", + "status": "running", + "health": "healthy", + "image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "memory_limit": 536870912, + "cpu_limit": 1.0, + "published_ports": { + "9000/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_s3data", + "destination": "/data" + } + ] + } + ], + "after": [ + { + "id": "c49b92fdbaae49470b7f69e4f973c6822b587b4c62067ee2a70db7b4d084f900", + "name": "mtusi-lab4-osokin-s3-1", + "service": "s3", + "hostname": "c49b92fdbaae", + "pid": 10343, + "started_at": "2026-11-16T12:16:24.37991011Z", + "status": "running", + "health": "starting", + "image": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "memory_limit": 536870912, + "cpu_limit": 1.0, + "published_ports": { + "9000/tcp": null + }, + "mounts": [ + { + "name": "mtusi-lab4-osokin_s3data", + "destination": "/data" + } + ] + } + ], + "observation": { + "live": 200, + "ready": 503, + "upload": 503, + "content": 503, + "metadata": 200 + }, + "recovered_within_seconds": 20.312, + "original_preserved": true +} diff --git a/evidence/smoke-after.txt b/evidence/smoke-after.txt new file mode 100644 index 00000000..72901c46 --- /dev/null +++ b/evidence/smoke-after.txt @@ -0,0 +1,2 @@ +$ python tests/smoke.py --base-url http://127.0.0.1:8084 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..72901c46 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +$ python tests/smoke.py --base-url http://127.0.0.1:8084 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stop-one.json b/evidence/stop-one.json new file mode 100644 index 00000000..ed2576f4 --- /dev/null +++ b/evidence/stop-one.json @@ -0,0 +1,74 @@ +{ + "target_instance": "2026dc4e851d-9723d8bbbf19", + "before": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 11286, + "started_at": "2026-11-16T12:16:41.122280354Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 11211, + "started_at": "2026-11-16T12:16:40.901560588Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + } + ], + "after": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 0, + "started_at": "2026-11-16T12:16:41.122280354Z", + "status": "exited", + "health": "unhealthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": {}, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 11211, + "started_at": "2026-11-16T12:16:40.901560588Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + } + ], + "hook_seconds": 2.982078799999954 +} diff --git a/evidence/upload-failover.json b/evidence/upload-failover.json new file mode 100644 index 00000000..311fbb1e --- /dev/null +++ b/evidence/upload-failover.json @@ -0,0 +1,80 @@ +{ + "uploader": "63b31780a94f-17a31e0697e3", + "downloader": "2026dc4e851d-ef8f1bfddac8", + "before": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 3801, + "started_at": "2026-11-16T12:13:39.391673918Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 3747, + "started_at": "2026-11-16T12:13:38.666975081Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + } + ], + "after": [ + { + "id": "2026dc4e851d17a53ca8abc9da19edbf1a896265ab9fbb29d30e307cc3b672c4", + "name": "mtusi-lab4-osokin-api-1", + "service": "api", + "hostname": "2026dc4e851d", + "pid": 3801, + "started_at": "2026-11-16T12:13:39.391673918Z", + "status": "running", + "health": "healthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": { + "8080/tcp": null + }, + "mounts": [] + }, + { + "id": "63b31780a94fc04de9631600e577c8a578fb715b26c6d672a25529f1dfcf8c8b", + "name": "mtusi-lab4-osokin-api-2", + "service": "api", + "hostname": "63b31780a94f", + "pid": 0, + "started_at": "2026-11-16T12:13:38.666975081Z", + "status": "exited", + "health": "unhealthy", + "image": "sha256:6166d1ba0c0b46f550fcdd39b773b6fe889f7ddbefa263188f5d734cb47af92d", + "memory_limit": 805306368, + "cpu_limit": 1.0, + "published_ports": {}, + "mounts": [] + } + ], + "bytes": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "same_original": true, + "attached_message_read": true, + "stranger_metadata": 404, + "stranger_content": 404 +} diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..c3c8b925 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,38 @@ +global + maxconn 256 + log stdout format raw local0 warning + +defaults + mode http + log global + timeout connect 500ms + timeout client 8s + timeout server 8s + timeout http-request 3s + timeout queue 1s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + hold other 1s + +frontend public + bind :8080 + default_backend messenger + +backend messenger + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 1s downinter 1s fall 2 rise 1 + server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 00000000..b3842842 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,10 @@ +bind 0.0.0.0 +protected-mode no +port 6379 +dir /data +appendonly yes +appendfsync always +save "" +maxmemory 64mb +maxmemory-policy noeviction +# Only the private Compose network can reach Redis; no host port is published. diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..edc61ab6 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_user ON chat_members(user_id, chat_id); +CREATE INDEX chats_order ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..a45f0eac --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +-- Additive migration: old lab2 tables and rows are unchanged. +-- This table is only for POST /chats/{chat_id}/messages; method/path scope is implicit. +CREATE TABLE message_requests ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + fingerprint text NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY (actor_id, chat_id, key) +); +CREATE INDEX message_requests_expiry ON message_requests(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..16a3dc05 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,19 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX attachments_chat ON attachments(chat_id); +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK ( + char_length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0) +); +ALTER TABLE messages ADD CONSTRAINT messages_attachment_count CHECK (cardinality(attachment_ids) <= 10); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e8ce75b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +psycopg==3.2.10 +psycopg-binary==3.2.10 +psycopg-pool==3.2.6 +redis==6.4.0 +typing_extensions==4.15.0 +boto3==1.40.6 +Pillow==11.3.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..aaa41fb9 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Create local synthetic credentials once; never print them or replace existing files.""" +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(exist_ok=True) +password_file = folder / 'db_password' +if not password_file.exists(): + password_file.write_text(secrets.token_hex(24), encoding='utf-8') +password = password_file.read_text(encoding='utf-8').strip() +values = { + 'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger', + 'cache_url': 'redis://redis:6379/0', + 'cursor_key': secrets.token_hex(32), + 's3_access_key_id': 'lab4-' + secrets.token_hex(8), + 's3_secret_access_key': secrets.token_hex(24), +} +for name, value in values.items(): + target = folder / name + if not target.exists(): + target.write_text(value, encoding='utf-8') +if not (root / '.env').exists(): + (root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8084\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\nAPI_REPLICAS=2\n', encoding='utf-8') +print('Local .env and secrets ready (existing values preserved).') diff --git a/scripts/labctl.py b/scripts/labctl.py new file mode 100644 index 00000000..08ce2189 --- /dev/null +++ b/scripts/labctl.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Operate only the lab4 Compose project; restart hooks never delete volumes.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8').strip() + + +def compose(*args): + return command('docker', 'compose', '--env-file', '.env', *args) + + +def containers(service=None, all_states=False): + ids = compose('ps', '-q', *(['--all'] if all_states else []), *([service] if service else [])).split() + if not ids: + return [] + result = [] + for item in json.loads(command('docker', 'inspect', *ids)): + result.append(dict(id=item['Id'], name=item['Name'].lstrip('/'), + service=item['Config']['Labels']['com.docker.compose.service'], + hostname=item['Config']['Hostname'], pid=item['State']['Pid'], + started_at=item['State']['StartedAt'], status=item['State']['Status'], + health=item['State'].get('Health', {}).get('Status'), image=item['Image'], + memory_limit=item['HostConfig']['Memory'], + cpu_limit=item['HostConfig']['NanoCpus'] / 1e9, + published_ports=item['NetworkSettings']['Ports'], + mounts=[dict(name=m.get('Name'), destination=m['Destination']) + for m in item['Mounts'] if m['Type'] == 'volume'])) + return sorted(result, key=lambda x: x['name']) + + +def save(name, data): + folder = ROOT / 'evidence' + folder.mkdir(exist_ok=True) + (folder / name).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + + +def restore(): + compose('up', '-d', '--no-deps', '--no-recreate', 'api') + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + items = containers('api') + if len(items) == 2 and all(x['health'] == 'healthy' for x in items): + time.sleep(3) + return + time.sleep(.5) + raise RuntimeError('Two API replicas did not become healthy') + + +def stop_one(): + target = os.environ.get('TARGET_INSTANCE_ID') + before = containers('api') + assert len(before) >= 2, 'Need two live replicas' + selected = [x for x in before if target and target.startswith(x['hostname'] + '-')] + assert len(selected) == 1, 'TARGET_INSTANCE_ID must identify one live API process' + started = time.monotonic() + command('docker', 'stop', '--time', '8', selected[0]['id']) + after = containers('api', True) + assert next(x for x in after if x['id'] == selected[0]['id'])['status'] == 'exited' + save('stop-one.json', dict(target_instance=target, before=before, after=after, + hook_seconds=time.monotonic() - started)) + print('Selected API stopped; another replica remains running.') + + +def restart(): + before = containers() + compose('restart', 'api', 'db', 'redis', 's3', 'lb') + after = containers() + for old in before: + new = next(x for x in after if x['id'] == old['id']) + assert old['started_at'] != new['started_at'], old['name'] + ' did not restart' + assert old['mounts'] == new['mounts'], 'Persistent volumes changed' + save('restart.json', dict(before=before, after=after)) + print('API, PostgreSQL, Redis, MinIO and LB restarted; same volumes preserved.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('action', choices=['restart', 'stop-one', 'restore', 'state']) + action = parser.parse_args().action + if action == 'state': + print(json.dumps(containers(all_states=True), indent=2)) + else: + {'restart': restart, 'stop-one': stop_one, 'restore': restore}[action]() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..68015828 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0\.." +python -X utf8 scripts\labctl.py restart +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a6c6bde8 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/labctl.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..327f1c81 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0\.." +python -X utf8 scripts\labctl.py stop-one +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..c38a7ca2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/labctl.py stop-one diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..5a641c26 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Run the grade 3 checks and keep small sanitized, reproducible evidence files.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / 'evidence' +BASE = os.getenv('BASE_URL', 'http://127.0.0.1:8084') +ENV = dict(os.environ, BASE_URL=BASE, PYTHONUTF8='1', PYTHONIOENCODING='utf-8') + + +def run(name, args): + print('Running', name, flush=True) + proc = subprocess.run(args, cwd=ROOT, env=ENV, text=True, encoding='utf-8', + errors='replace', stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = '$ ' + ' '.join(args).replace(sys.executable, 'python') + '\n' + proc.stdout + output = '\n'.join(line.rstrip() for line in output.splitlines()).rstrip() + '\n' + (EVIDENCE / (name + '.txt')).write_text(output, encoding='utf-8') + print(proc.stdout, end='', flush=True) + if proc.returncode: + raise SystemExit('Check failed: ' + name) + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + run('check', [sys.executable, 'scripts/check_contract.py']) + run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet']) + run('smoke', [sys.executable, 'tests/smoke.py', '--base-url', BASE]) + run('idempotency', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_idempotency.py']) + run('limits', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_limits.py']) + run('attachments-grade3', [sys.executable, 'student-tests/grade3.py']) + run('persistence', [sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--base-url', BASE, + '--action-script', 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh']) + run('failover', [sys.executable, 'tests/resilience.py', '--mode', 'failover', '--base-url', BASE, + '--action-script', 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh']) + run('restore', [sys.executable, 'scripts/labctl.py', 'restore']) + run('smoke-after', [sys.executable, 'tests/smoke.py', '--base-url', BASE]) + run('processes', [sys.executable, 'scripts/labctl.py', 'state']) + info = json.loads(subprocess.check_output(['docker', 'info', '--format', '{{json .}}'], env=ENV, text=True)) + environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), host_os=platform.platform(), + client_python=platform.python_version(), + docker={k: info[k] for k in ('ServerVersion', 'OperatingSystem', 'KernelVersion', + 'NCPU', 'MemTotal', 'Architecture')}, images={}) + environment['app_python'] = subprocess.check_output( + ['docker', 'compose', 'exec', '-T', 'api', 'python', '--version'], text=True, env=ENV).strip() + for name in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm', 'haproxy:3.0.16-alpine', + 'minio/minio:RELEASE.2025-04-22T22-12-26Z'): + environment['images'][name] = json.loads(subprocess.check_output( + ['docker', 'image', 'inspect', name, '--format', '{{json .RepoDigests}}'], text=True, env=ENV)) + (EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8') + print('PASS: all required grade 3 scenarios completed.') + + +if __name__ == '__main__': + main() diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..a4cab749 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,19 @@ +"""Wait for the public entrypoint after starting the Compose stack.""" +import os +import time +import urllib.error +import urllib.request + +base = os.getenv('BASE_URL', 'http://127.0.0.1:8084').rstrip('/') +deadline = time.monotonic() + 90 +while True: + try: + with urllib.request.urlopen(base + '/health/ready', timeout=3) as response: + if response.status == 200: + print('READY:', base) + break + except (urllib.error.URLError, OSError): + pass + if time.monotonic() >= deadline: + raise SystemExit('Public API did not become ready within 90 seconds') + time.sleep(.5) diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..2f7a605a --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1 @@ +"""Lab 1: bounded, single-process in-memory messenger.""" diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..2e07df8d --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,102 @@ +"""Synchronous original-only uploads. PostgreSQL holds metadata; S3 holds bytes.""" +import hashlib +from io import BytesIO +import os +import unicodedata +import uuid +import warnings + +import boto3 +from botocore.config import Config as S3Config +from PIL import Image, UnidentifiedImageError + +from .core import APIError, public_row, secret_file, string, uuid_value + + +def s3_client(): + return boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'], + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=secret_file('S3_ACCESS_KEY_ID_FILE'), + aws_secret_access_key=secret_file('S3_SECRET_ACCESS_KEY_FILE'), + config=S3Config(connect_timeout=1, read_timeout=2, + retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}, + request_checksum_calculation='when_required', + response_checksum_validation='when_required')) + + +def filename_value(value): + value = string(value, 255) + if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): + raise APIError(400, 'Invalid filename') + return value + + +def validate_image(raw, declared_type, max_pixels): + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + # open parses the header lazily. Check dimensions before load/verify. + with Image.open(BytesIO(raw), formats=['PNG', 'JPEG']) as image: + if image.width * image.height > max_pixels: + raise APIError(413, 'Image exceeds pixel limit') + actual_type = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format] + if actual_type != declared_type: + raise APIError(415, 'Image type differs from Content-Type') + image.verify() + # JPEG verify does not decode entropy data; load catches truncated files. + with Image.open(BytesIO(raw), formats=['PNG', 'JPEG']) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, 'Image exceeds pixel limit') from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError): + raise APIError(415, 'Invalid PNG/JPEG') from None + return actual_type + + +def metadata(row): + result = public_row({k: v for k, v in row.items() if k != 'object_key'}) + return dict(result, status='ready', variants=[], error_code=None, updated_at=result['created_at']) + + +class Attachments: + def __init__(self, store): + self.store = store + self.s3 = s3_client() + self.bucket = os.environ['S3_BUCKET'] + + def ready(self): + self.s3.head_bucket(Bucket=self.bucket) + + def upload(self, actor, chat_id, filename, content_type, raw): + actual = validate_image(raw, content_type, self.store.config.max_image_pixels) + identifier = uuid.uuid4() + key = 'originals/' + uuid.uuid4().hex + digest = hashlib.sha256(raw).hexdigest() + # No shared transaction with S3: failures after PUT may leave an orphan. + # Grade 3 intentionally has no automatic reconciliation/cleanup mechanism. + self.s3.put_object(Bucket=self.bucket, Key=key, Body=raw, ContentType=actual) + with self.store.pool.connection() as conn: + row = conn.execute('INSERT INTO attachments ' + '(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', + (identifier, chat_id, actor, filename, key, actual, len(raw), digest)).fetchone() + return metadata(row) + + def get(self, actor, identifier): + identifier = uuid_value(identifier) + with self.store.pool.connection() as conn: + row = conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ' + 'ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s', + (identifier, actor)).fetchone() + if row is None: + raise APIError(404, 'Object not found') + return row + + def content(self, row): + response = self.s3.get_object(Bucket=self.bucket, Key=row['object_key']) + with response['Body'] as stream: + raw = stream.read(self.store.config.max_upload_bytes + 1) + if len(raw) != row['size_bytes'] or hashlib.sha256(raw).hexdigest() != row['sha256']: + raise APIError(503, 'Original temporarily unavailable') + return raw diff --git a/src/messenger/bucket.py b/src/messenger/bucket.py new file mode 100644 index 00000000..1555ee09 --- /dev/null +++ b/src/messenger/bucket.py @@ -0,0 +1,21 @@ +"""Create a private bucket on first deployment. Existing buckets need no bootstrap.""" +import os +from .attachments import s3_client + + +def main(): + s3 = s3_client() + try: + bucket = os.environ['S3_BUCKET'] + # This Compose project owns a new volume/bucket. For subsequent runs, + # list names first; do not modify existing policies or objects. + names = [item['Name'] for item in s3.list_buckets()['Buckets']] + if bucket not in names: + s3.create_bucket(Bucket=bucket) + print('Private bucket available; no anonymous policy is installed.') + finally: + s3.close() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/core.py b/src/messenger/core.py new file mode 100644 index 00000000..2ad03843 --- /dev/null +++ b/src/messenger/core.py @@ -0,0 +1,356 @@ +"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT.""" +import base64 +from dataclasses import dataclass, fields +from datetime import datetime, timezone +import hashlib +import hmac +import json +import os +from pathlib import Path +import re +import secrets +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +class APIError(Exception): + def __init__(self, status, message): + self.status, self.message = status, message + + +@dataclass +class Config: + app_host: str = '0.0.0.0' + app_port: int = 8080 + workers: int = 16 + request_queue: int = 32 + socket_timeout: int = 2 + max_body_bytes: int = 65536 + session_ttl_seconds: int = 3600 + idempotency_ttl_seconds: int = 86400 + max_upload_bytes: int = 10485760 + max_image_pixels: int = 40000000 + + @classmethod + def from_env(cls): + values = {} + for field in fields(cls): + raw = os.environ.get(field.name.upper(), field.default) + values[field.name] = str(raw) if field.type is str else int(raw) + if field.type is int and values[field.name] < 1: + raise ValueError(field.name + ' must be positive') + result = cls(**values) + if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128: + raise ValueError('port/workers/queue outside bounds') + if result.max_body_bytes > 1048576 or result.socket_timeout > 5: + raise ValueError('body <= 1 MiB and socket timeout <= 5 required') + if os.getenv('SESSION_STORE', 'redis') != 'redis': + raise ValueError('This solution uses Redis sessions') + if result.idempotency_ttl_seconds < 86400: + raise ValueError('Idempotency retention must be at least 24 hours') + if result.max_upload_bytes > 10485760 or result.max_image_pixels > 40000000: + raise ValueError('Upload limits exceed the contract') + return result + + +def secret_file(name): + return Path(os.environ[name]).read_text(encoding='utf-8').strip() + + +def db_connect_kwargs(): + return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 ' + '-c idle_in_transaction_session_timeout=5000', row_factory=dict_row) + + +def uuid_value(value): + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value): + raise APIError(400, 'Invalid UUID') + return str(uuid.UUID(value)) + + +def string(value, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value: + raise APIError(400, 'Invalid string') + try: + value.encode('utf-8') + except UnicodeError: + raise APIError(400, 'Invalid Unicode') from None + return value + + +def object_fields(body, expected): + if not isinstance(body, dict) or set(body) != set(expected): + raise APIError(400, 'Invalid object fields') + + +def password_value(value): + if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128: + raise APIError(400, 'Invalid password length') + try: + value.encode('utf-8') + except UnicodeError: + raise APIError(400, 'Invalid Unicode') from None + return value + + +def password_hash(value, salt=None): + salt = salt or secrets.token_bytes(16) + digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32) + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +def password_matches(value, encoded): + # Only this versioned format is generated by registration. + salt = bytes.fromhex(encoded.split('$')[4]) + return hmac.compare_digest(password_hash(value, salt), encoded) + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def public_row(row): + return {key: ([str(item) for item in value] if key == 'attachment_ids' else + str(value) if isinstance(value, uuid.UUID) else stamp(value) + if isinstance(value, datetime) else value) + for key, value in row.items() if key != 'password_hash'} + + +USER_COLUMNS = 'id, username, display_name, created_at' +MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at, attachment_ids' +# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters. +HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s ' + 'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s') + + +class Store: + def __init__(self, config): + self.config = config + from .attachments import Attachments + self.attachments = Attachments(self) + self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE')) + if len(self.secret) < 32: + raise ValueError('Cursor key must be at least 256 bits') + self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8, + timeout=1, max_waiting=16, kwargs=db_connect_kwargs(), + check=ConnectionPool.check_connection, open=True) + self.cache_pool = redis.ConnectionPool.from_url( + secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5, + socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0)) + self.cache = redis.Redis(connection_pool=self.cache_pool) + self.dummy_hash = password_hash('dummy-password-for-timing') + + def close(self): + self.attachments.s3.close() + self.pool.close(timeout=2) + self.cache_pool.disconnect() + + def ready(self): + with self.pool.connection() as conn: + conn.execute('SELECT 1') + self.cache.ping() + self.attachments.ready() + + def create_user(self, body): + object_fields(body, ('username', 'display_name', 'password')) + username = string(body['username'], 32) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise APIError(400, 'Invalid username') + display = string(body['display_name'], 100) + hashed = password_hash(password_value(body['password'])) + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) ' + 'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS, + (uuid.uuid4(), username, display, hashed)).fetchone() + except psycopg.errors.UniqueViolation: + raise APIError(409, 'Username already exists') from None + return public_row(row) + + def login(self, authorization): + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic' or len(encoded) > 1024: + raise ValueError() + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise ValueError() + password_value(password) + except (AttributeError, ValueError, UnicodeError, APIError): + raise APIError(401, 'Invalid credentials') from None + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone() + matched = password_matches(password, row['password_hash'] if row else self.dummy_hash) + if not matched or row is None: + raise APIError(401, 'Invalid credentials') + token = secrets.token_urlsafe(32) + expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000 + self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms) + return dict(token=token, token_type='Bearer', + expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)), + user=public_row(row)) + + @staticmethod + def session_key(token): + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def auth(self, header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError() + except (ValueError, AttributeError): + raise APIError(401, 'Invalid session') from None + key = self.session_key(token) + actor = self.cache.get(key) + if actor is None: + raise APIError(401, 'Invalid session') + return actor, key + + def me(self, actor): + with self.pool.connection() as conn: + row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone() + if row is None: + raise APIError(401, 'Invalid session') + return public_row(row) + + def chat_access(self, actor, identifier): + identifier = uuid_value(identifier) + with self.pool.connection() as conn: + row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', + (identifier, actor)).fetchone() + if row is None: + raise APIError(404, 'Object not found') + return identifier + + @staticmethod + def chat_row(conn, row): + members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position', + (row['id'],)).fetchall() + return dict(public_row(row), member_ids=[str(x['user_id']) for x in members]) + + def chat(self, chat_id): + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone() + return self.chat_row(conn, row) + + def create_chat(self, actor, body): + object_fields(body, ('title', 'member_ids')) + title = string(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise APIError(400, 'Invalid members') + members = [uuid_value(x) for x in members] + if len(members) != len(set(members)) or actor not in members: + raise APIError(400, 'Invalid members') + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *', + (uuid.uuid4(), title)).fetchone() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)', + [(row['id'], member, pos) for pos, member in enumerate(members)]) + except psycopg.errors.ForeignKeyViolation: + raise APIError(400, 'Invalid members') from None + return dict(public_row(row), member_ids=members) + + def create_message(self, actor, chat_id, body, key): + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise APIError(400, 'One valid Idempotency-Key is required') + if not isinstance(body, dict) or set(body) not in ({'text'}, {'text', 'attachment_ids'}): + raise APIError(400, 'Invalid object fields') + ids = body.get('attachment_ids', []) + if not isinstance(ids, list) or len(ids) > 10: + raise APIError(400, 'Invalid attachment list') + ids = [uuid_value(item) for item in ids] + if len(ids) != len(set(ids)): + raise APIError(400, 'Duplicate attachment') + text = body['text'] + if not isinstance(text, str) or len(text) > 4000 or '\x00' in text or (not text.strip() and not ids): + raise APIError(400, 'Invalid message text') + try: + text.encode('utf-8') + except UnicodeError: + raise APIError(400, 'Invalid Unicode') from None + digest = hashlib.sha256(json.dumps({'text': text, 'attachment_ids': ids}, + ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + scope = json.dumps([actor, 'POST', '/api/v1/chats/' + chat_id + '/messages', key]) + lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True) + with self.pool.connection() as conn: + # Transaction lock spans SELECT, message INSERT, result INSERT and COMMIT. + # All replicas use PostgreSQL; a timeout rolls back and may be retried. + conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,)) + previous = conn.execute('SELECT fingerprint, response, expires_at > clock_timestamp() AS valid ' + 'FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s', + (actor, chat_id, key)).fetchone() + if previous and previous['valid']: + if previous['fingerprint'] != digest: + raise APIError(409, 'Idempotency-Key already used with another body') + return previous['response'] + if previous: + conn.execute('DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s', + (actor, chat_id, key)) + if ids: + found = conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])', + (chat_id, ids)).fetchall() + if len(found) != len(ids): + raise APIError(404, 'Object not found') + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) ' + 'VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING ' + MESSAGE_COLUMNS, + (uuid.uuid4(), chat_id, actor, text, ids)).fetchone() + result = public_row(row) + conn.execute('INSERT INTO message_requests(actor_id,chat_id,key,fingerprint,response,expires_at) ' + "VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')", + (actor, chat_id, key, digest, Jsonb(result), self.config.idempotency_ttl_seconds)) + return result + + def encode_cursor(self, scope, key): + payload = json.dumps([1, scope, *key], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode() + + def decode_cursor(self, cursor, scope): + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor): + raise ValueError() + raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True) + if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor: + raise ValueError() + payload, signature = raw[:-32], raw[-32:] + if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')): + raise ValueError() + version, resource, timestamp, identifier = json.loads(payload) + if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'): + raise ValueError() + return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier) + except (ValueError, TypeError, UnicodeError, APIError): + raise APIError(400, 'Invalid cursor') from None + + def page(self, actor, chat_id, query): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + raise APIError(400, 'Invalid query') + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100: + raise APIError(400, 'Invalid limit') + limit = int(limit) + scope = 'messages:' + chat_id if chat_id else 'chats:' + actor + key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else ( + datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000') + with self.pool.connection() as conn: + if chat_id: + rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall() + items = [public_row(row) for row in rows[:limit]] + else: + rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) ' + 'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall() + items = [self.chat_row(conn, row) for row in rows[:limit]] + cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None + return dict(items=items, next_cursor=cursor) diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e05e2556 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,30 @@ +"""Transactional, versioned migrations; repeat application is a no-op.""" +import hashlib +from pathlib import Path +import psycopg +from .core import secret_file, db_connect_kwargs + + +def main(): + with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2026, 2)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(name text PRIMARY KEY, sha256 text NOT NULL, ' + 'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())') + for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')): + content = path.read_bytes() + digest = hashlib.sha256(content).hexdigest() + row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone() + if row: + if row['sha256'] != digest: + raise RuntimeError('Applied migration checksum mismatch') + print('Already applied:', path.name) + continue + conn.execute(content.decode('utf-8'), prepare=False) + conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest)) + print('Applied:', path.name) + print('Migrations committed.') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/server.py b/src/messenger/server.py new file mode 100644 index 00000000..7265caa3 --- /dev/null +++ b/src/messenger/server.py @@ -0,0 +1,376 @@ +"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies.""" +from collections import Counter +from http.server import BaseHTTPRequestHandler, HTTPServer +import http.client +import json +import logging +import os +import queue +import signal +import socket +import threading +import time +from urllib.parse import parse_qs, urlsplit +import uuid + +import psycopg +from psycopg_pool import PoolTimeout, TooManyRequests +import redis +from botocore.exceptions import BotoCoreError, ClientError + +from .core import APIError, Config, Store +from .attachments import filename_value, metadata + + +CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict", + 413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited", + 503: "unavailable", 500: "internal_error"} + + +def error_body(status, message, request_id): + return {"error": {"code": CODES.get(status, "invalid_request"), + "message": message, "request_id": request_id}} + + +def log_event(**data): + logging.info(json.dumps(data, ensure_ascii=True)) + + +class Server(HTTPServer): + allow_reuse_address = True + + def __init__(self, config): + self.config, self.store = config, Store(config) + self.instance_id = os.environ.get('HOSTNAME', 'api') + '-' + uuid.uuid4().hex[:12] + self.jobs = queue.Queue(maxsize=config.request_queue) + self.draining = threading.Event() + self.stats_lock = threading.Lock() + self.statuses = Counter() + self.active = 0 + self.clients = set() + self.started = time.monotonic() + self.request_queue_size = config.request_queue + super().__init__((config.app_host, config.app_port), Handler) + self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True) + for n in range(config.workers)] + for thread in self.threads: + thread.start() + + def get_request(self): + client, address = super().get_request() + client.settimeout(self.config.socket_timeout) + return client, address + + def record(self, status): + with self.stats_lock: + self.statuses[str(status)] += 1 + + def process_request(self, request, address): + try: + if self.draining.is_set(): + raise queue.Full() + self.jobs.put_nowait((request, address)) + except queue.Full: + # Transport admission before HTTP parsing: bounded time, no extra thread. + identifier = str(uuid.uuid4()) + body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode() + headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n" + "Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier + + "\r\nX-Instance-Id: " + self.instance_id + + "\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode() + try: + request.settimeout(.05) + # Wait for the first request bytes: closing before a local client + # finishes sendall can otherwise discard the 503 with a TCP reset. + try: + request.recv(16384) + except OSError: + pass + request.sendall(headers + body) + request.shutdown(socket.SHUT_WR) + request.settimeout(.01) + for _ in range(5): + if not request.recv(16384): + break + except OSError: + pass + self.shutdown_request(request) + self.record(503) + log_event(event="queue_rejected", status=503, request_id=identifier) + + def worker(self): + while not self.draining.is_set() or not self.jobs.empty(): + try: + request, address = self.jobs.get(timeout=.1) + except queue.Empty: + continue + with self.stats_lock: + self.active += 1 + self.clients.add(request) + try: + self.finish_request(request, address) + except OSError: + pass + except Exception: + log_event(event="worker_error") + finally: + self.shutdown_request(request) + with self.stats_lock: + self.active -= 1 + self.clients.discard(request) + self.jobs.task_done() + + def server_close(self): + self.draining.set() + super().server_close() + # Stop listening, reject queued work, let accepted in-flight work COMMIT + # and send its response before forcibly closing any remaining socket. + deadline = time.monotonic() + 6 + for thread in self.threads: + thread.join(timeout=max(0, deadline - time.monotonic())) + with self.stats_lock: + clients = list(self.clients) + for client in clients: + try: + client.shutdown(socket.SHUT_RDWR) + except OSError: + pass + while True: + try: + request, _ = self.jobs.get_nowait() + except queue.Empty: + break + self.shutdown_request(request) + self.jobs.task_done() + self.store.close() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "Messenger" + sys_version = "" + + def setup(self): + super().setup() + self.request_id = str(uuid.uuid4()) + self.received = time.monotonic() + + def log_message(self, *_): + pass + + def parse_request(self): + original = self.rfile + + class HeaderBudget: + remaining = 16384 + + def readline(self, limit=-1): + line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1) + self.remaining -= len(line) + if self.remaining < 0: + raise http.client.LineTooLong("header section") + return line + + self.rfile = HeaderBudget() + try: + return super().parse_request() + finally: + self.rfile = original + + def send_error(self, code, message=None, explain=None): + self.reply(code if code in CODES else 400, error_body( + code if code in CODES else 400, "Invalid HTTP request", self.request_id)) + + def reply(self, status, body): + if isinstance(body, tuple): + raw, content_type = body + else: + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + content_type = 'application/json' + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(raw))) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.server.instance_id) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Connection", "close") + if status == 401: + challenge = ('Basic realm="messenger", charset="UTF-8"' + if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.send_header('WWW-Authenticate', challenge) + if status in (429, 503): + self.send_header("Retry-After", "1") + self.end_headers() + self.close_connection = True + self.wfile.write(raw) + self.server.record(status) + # No URL query, header, username, text, or exception detail in logs. + log_event(event="request", method=self.command, status=status, + request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3)) + + def handle_expect_100(self): + # Do not acknowledge a large unauthorized request before checking identity. + return True + + def read_body(self, maximum, content_types): + if self.headers.get_all("Transfer-Encoding"): + raise APIError(400, "Transfer-Encoding is unsupported") + sizes = self.headers.get_all("Content-Length", []) + if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10: + raise APIError(400, "One valid Content-Length is required") + size = int(sizes[0]) + if size > maximum: + raise APIError(413, "Request body exceeds byte limit") + if len(self.headers.get_all('Content-Type', [])) != 1 or self.headers.get_content_type() not in content_types: + raise APIError(415, "Unsupported Content-Type") + if self.headers.get("Expect", "").lower() == "100-continue": + self.send_response_only(100) + self.end_headers() + raw = self.rfile.read(size) + if len(raw) != size: + raise APIError(400, "Incomplete request body") + return raw + + def body(self): + raw = self.read_body(self.server.config.max_body_bytes, ('application/json',)) + + def pairs(values): + result = {} + for key, value in values: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + def constant(_): + raise ValueError("non JSON constant") + + try: + return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant) + except (ValueError, UnicodeError, RecursionError): + raise APIError(400, "Invalid JSON") from None + + def dispatch(self): + try: + url = urlsplit(self.path) + except ValueError: + raise APIError(400, "Invalid request target") from None + path, method = url.path, self.command + store = self.server.store + if self.server.draining.is_set(): + raise APIError(503, "Shutting down") + if method == 'GET' and path == '/health/live': + return 200, {'status': 'ok'} + if method == 'GET' and path == '/health/ready': + store.ready() + return 200, {'status': 'ok'} + if path == '/api/v1/users' and method == 'POST': + return 201, store.create_user(self.body()) + headers = self.headers.get_all('Authorization', []) + authorization = headers[0] if len(headers) == 1 else None + if path == '/api/v1/auth/sessions' and method == 'POST': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + raise APIError(400, 'Login takes no request body') + return 201, store.login(authorization) + actor, session_key = store.auth(authorization) + parts = path.strip('/').split('/') + if len(parts) in (4, 5) and parts[:3] == ['api', 'v1', 'attachments'] and method == 'GET': + row = store.attachments.get(actor, parts[3]) + if len(parts) == 4: + if url.query: + raise APIError(400, 'Invalid query') + return 200, metadata(row) + if parts[4] == 'content': + query = self.query(url) + if set(query) - {'variant'}: + raise APIError(400, 'Invalid query') + if query.get('variant', ['original'])[0] != 'original': + raise APIError(404, 'Object not found') + return 200, (store.attachments.content(row), row['content_type']) + if method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.cache.delete(session_key) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, store.me(actor) + chat_id = None + if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]: + chat_id = store.chat_access(actor, parts[3]) + if method == "POST" and path == "/api/v1/chats": + return 201, store.create_chat(actor, self.body()) + if method == 'POST' and chat_id and len(parts) == 5 and parts[4] == 'attachments': + query = self.query(url) + if set(query) != {'filename'}: + raise APIError(400, 'One filename is required') + filename = filename_value(query['filename'][0]) + raw = self.read_body(self.server.config.max_upload_bytes, ('image/png', 'image/jpeg')) + return 201, store.attachments.upload(actor, chat_id, filename, self.headers.get_content_type(), raw) + if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages": + keys = self.headers.get_all('Idempotency-Key', []) + key = keys[0] if len(keys) == 1 else None + return 201, store.create_message(actor, chat_id, self.body(), key) + if method == "GET": + if chat_id and len(parts) == 4: + return 200, store.chat(chat_id) + if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"): + return 200, store.page(actor, chat_id, self.query(url)) + raise APIError(404, "Object not found") + + @staticmethod + def query(url): + try: + query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True, + max_num_fields=8, errors='strict') + if any(len(values) != 1 for values in query.values()): + raise ValueError() + return query + except (ValueError, UnicodeError): + raise APIError(400, 'Invalid query') from None + + def handle_api(self): + try: + status, result = self.dispatch() + except APIError as error: + status = error.status + result = error_body(status, error.message, self.request_id) + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError, + BotoCoreError, ClientError): + status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id) + except (TimeoutError, socket.timeout): + status, result = 400, error_body(400, "Request read timeout", self.request_id) + except Exception: + status, result = 500, error_body(500, "Internal error", self.request_id) + log_event(event="internal_error", request_id=self.request_id) + try: + self.reply(status, result) + except OSError: + pass + + do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(message)s") + # Pool diagnostics may include connection details; application emits sanitized errors. + logging.getLogger('psycopg.pool').disabled = True + config = Config.from_env() + server = Server(config) + + def stop(signum, _): + if not server.draining.is_set(): + server.draining.set() + log_event(event="shutdown_started", signal=signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + log_event(event="started", instance_id=server.instance_id, port=config.app_port, workers=config.workers) + try: + server.serve_forever(poll_interval=.1) + finally: + server.server_close() + log_event(event="shutdown_complete") + + +if __name__ == "__main__": + main() diff --git a/student-tests/grade3.py b/student-tests/grade3.py new file mode 100644 index 00000000..9594d4fd --- /dev/null +++ b/student-tests/grade3.py @@ -0,0 +1,136 @@ +"""Required original-file, replica-switch, bucket and S3-outage demonstrations.""" +import hashlib +from io import BytesIO +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT / 'tests'), str(ROOT / 'scripts')] +from client import Client, check +from labctl import containers, restore, save + + +def compose(*args, payload=None): + process = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + input=payload, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, encoding='utf-8') + if process.returncode: + raise RuntimeError('Compose action failed: ' + args[0]) + return process.stdout.strip() + + +def wait_ready(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready', schema='Health') + return + except AssertionError: + time.sleep(.5) + raise RuntimeError('Readiness did not recover') + + +def inside_probe(): + """Called in one API container with existing synthetic actors over stdin.""" + sys.path.insert(0, str(ROOT / 'src')) + from messenger.core import Config, Store + from PIL import Image + data = json.load(sys.stdin) + c = Client('http://localhost:8080') + alice, bob, eve, chat, att = (data[x] for x in ('alice', 'bob', 'eve', 'chat', 'att')) + if data['mode'] == 'private': + store = Store(Config.from_env()) + try: + row = store.attachments.get(alice['id'], att['id']) + url = os.environ['S3_ENDPOINT'] + '/' + os.environ['S3_BUCKET'] + '/' + row['object_key'] + try: + urllib.request.urlopen(url, timeout=3) + raise AssertionError('Anonymous object read succeeded') + except urllib.error.HTTPError as error: + check(error.code == 403, 'Expected anonymous S3 403') + images = [] + for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: + buf = BytesIO() + Image.new('RGB', (32, 24), 'blue').save(buf, format=fmt) + raw = buf.getvalue() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=real.' + fmt.lower(), + 201, actor=alice, raw=raw, headers={'Content-Type': mime}, schema='Attachment')['json'] + downloaded = c.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob) + check(downloaded['bytes'] == raw, 'Original changed') + images.append(dict(format=fmt, bytes=len(raw), sha256=uploaded['sha256'])) + print(json.dumps(dict(anonymous_get=403, real_images=images))) + finally: + store.close() + else: + c.call('GET', '/health/live', schema='Health') + c.call('GET', '/health/ready', 503) + raw = (ROOT / 'tests/fixtures/sample.png').read_bytes() + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png', + 503, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment') + print(json.dumps(dict(live=200, ready=503, upload=503, content=503, metadata=200))) + + +def main(): + c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8084')) + alice, bob, eve, chat = c.bootstrap() + raw = (ROOT / 'tests/fixtures/sample.png').read_bytes() + response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment') + att, uploader = response['json'], response['headers']['X-Instance-Id'] + message = c.message(alice, chat, '', attachments=[att['id']])['json'] + before = containers('api') + target = next(x for x in before if uploader.startswith(x['hostname'] + '-')) + try: + subprocess.run(['docker', 'stop', '--time', '8', target['id']], check=True, stdout=subprocess.PIPE) + after = containers('api', True) + check(next(x for x in after if x['id'] == target['id'])['status'] == 'exited', 'Uploader not stopped') + wait_ready(c) + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + reader = downloaded['headers']['X-Instance-Id'] + check(reader != uploader and downloaded['bytes'] == raw, 'Replica-switch original mismatch') + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json'] + check(message in page['items'], 'Attached message missing from history') + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + result = dict(uploader=uploader, downloader=reader, before=before, after=after, + bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest(), + same_original=True, attached_message_read=True, stranger_metadata=404, stranger_content=404) + save('upload-failover.json', result) + print('PASS: uploading API stopped; another API returned identical original and attached message.') + finally: + restore() + data = dict(alice=alice, bob=bob, eve=eve, chat=chat, att=att, mode='private') + result = compose('exec', '-T', 'api', 'python', 'student-tests/grade3.py', '--inside', + payload=json.dumps(data)) + save('private-and-formats.json', json.loads(result)) + print('PASS: real PNG/JPEG originals and anonymous S3 403.') + before = containers('s3') + started = time.monotonic() + try: + compose('stop', '--timeout', '8', 's3') + check(not containers('s3'), 'MinIO still running') + data['mode'] = 'outage' + result = compose('exec', '-T', 'api', 'python', 'student-tests/grade3.py', '--inside', + payload=json.dumps(data)) + observation = json.loads(result) + finally: + compose('start', 's3') + wait_ready(c) + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(downloaded['bytes'] == raw, 'Original changed after S3 outage') + save('s3-outage.json', dict(before=before, after=containers('s3'), observation=observation, + recovered_within_seconds=round(time.monotonic() - started, 3), + original_preserved=True)) + print('PASS: unavailable S3 yields 503; restart restores the same original and existing session.') + + +if __name__ == '__main__': + inside_probe() if '--inside' in sys.argv else main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..21914bae --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,55 @@ +"""Run inside API network: same session and simultaneous requests at distinct processes.""" +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import socket +import sys +import threading +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, check + + +def main(): + clients = [Client('http://' + ip + ':8080') for ip in sorted(set(socket.gethostbyname_ex('api')[2]))] + check(len(clients) == 2, 'Expected two API addresses in Docker DNS') + c = clients[0] + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + key = uuid.uuid4().hex + barrier = threading.Barrier(16) + + def send(n): + barrier.wait(timeout=10) + return clients[n % 2].call('POST', path, 201, actor=alice, + body={'attachment_ids': [], 'text': 'Параллельный повтор'} if n % 2 else + {'text': 'Параллельный повтор'}, + headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(send, range(16))) + messages = [x['json'] for x in responses] + instances = sorted({x['headers']['X-Instance-Id'] for x in responses}) + check(len(instances) == 2, 'Requests did not reach distinct API processes') + check(all(x == messages[0] for x in messages), 'Concurrent responses differ') + history = clients[1].call('GET', path, actor=bob, schema='MessagePage')['json']['items'] + check(history == [messages[0]], 'Concurrent requests created duplicates') + clients[1].call('POST', path, 409, actor=alice, body={'text': 'Другой body'}, headers={'Idempotency-Key': key}) + # Scope is user + chat; an independent user and another chat may reuse the key. + other = clients[1].message(bob, chat, 'Другой пользователь', key=key)['json'] + chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json'] + third = clients[1].message(alice, chat2, 'Другой чат', key=key)['json'] + check(len({messages[0]['id'], other['id'], third['id']}) == 3, 'Incorrect idempotency scope') + for invalid in ('', 'contains space', 'x' * 129): + c.call('POST', path, 400, actor=alice, body={'text': 'Invalid key'}, headers={'Idempotency-Key': invalid}) + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + clients[1].call('POST', path, 401, actor=alice, body={'text': 'Параллельный повтор'}, + headers={'Idempotency-Key': key}) + print(json.dumps(dict(result='PASS', parallel_requests=16, api_instances=instances, + unique_messages=1, scope_conflict_logout='PASS'), ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py new file mode 100644 index 00000000..9b4ae8cc --- /dev/null +++ b/student-tests/test_limits.py @@ -0,0 +1,71 @@ +"""Check the upload limits required by grade 3 before body reading/decoding.""" +from email.message import Message +from io import BytesIO +from pathlib import Path +import struct +import sys +from types import SimpleNamespace +import zlib + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check +from messenger.attachments import filename_value, validate_image +from messenger.core import APIError +from messenger.server import Handler + + +def rejected(status, action): + try: + action() + except APIError as error: + check(error.status == status, 'Unexpected rejection code') + else: + raise AssertionError('Expected rejection') + + +class NeverRead: + def read(self, *_): + raise AssertionError('Oversized body was read') + + +def main(): + headers = Message() + headers['Content-Length'] = '10485761' + headers['Content-Type'] = 'image/png' + handler = SimpleNamespace(headers=headers, rfile=NeverRead()) + rejected(413, lambda: Handler.read_body(handler, 10485760, ('image/png',))) + original = (ROOT / 'tests/fixtures/sample.png').read_bytes() + # Header-only dimension change: rejection must happen before decoding pixels. + header = original[12:16] + struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0) + huge = original[:12] + header + struct.pack('!I', zlib.crc32(header)) + original[33:] + rejected(413, lambda: validate_image(huge, 'image/png', 40000000)) + for name in ('../x.png', 'a\\x.png', 'a\0.png', 'a\n.png'): + rejected(400, lambda: filename_value(name)) + check(filename_value('фото.png') == 'фото.png', 'Unicode display filename changed') + c = Client('http://lb:8080') + alice, bob, _, chat = c.bootstrap() + # Exact byte limit is legal; trailing PNG bytes must be returned unchanged. + raw = original + b'\0' * (10485760 - len(original)) + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=limit.png', + 201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == raw, + '10 MiB original changed') + del raw + # Exact pixel limit: valid image, checked in one request without concurrency. + buffer = BytesIO() + with Image.new('RGB', (8000, 5000), 'black') as image: + image.save(buffer, format='PNG') + raw = buffer.getvalue() + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=pixels.png', + 201, actor=alice, raw=raw, headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == raw, + '40 million pixel original changed') + print('PASS: 10 MiB and 40,000,000 pixels accepted; excess bytes rejected before read;') + print('excess dimensions rejected before decode; unsafe filenames rejected; Unicode filename accepted.') + + +if __name__ == '__main__': + main()