diff --git a/.dockerignore b/.dockerignore index bc16e556..fe573460 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store +**/__pycache__ +**/*.py[cod] diff --git a/.env.example b/.env.example index d99d26a2..1a9c918d 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,5 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +HTTP_PORT=18104 +COMPOSE_PROJECT_NAME=kuznetsov-lab4 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +S3_BUCKET=kuznetsov-lab4-attachments diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4b887bf0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto eol=lf +*.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2c1cae8d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.13.14-slim-bookworm@sha256:67a1e1f215ccda113cfc024e8639049257e88f273898f595b61476d128d387e8 + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONUTF8=1 \ + APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +WORKDIR /app +COPY requirements.lock . +RUN python -m pip install --no-cache-dir -r requirements.lock \ + && useradd --uid 10001 --create-home messenger +COPY src ./src +COPY migrations ./migrations +USER messenger +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=6 \ + CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.environ['APP_PORT']+'/health/ready',timeout=3).close()" +CMD ["python", "-m", "src"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..f55d6271 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,83 @@ -# Отчёт — лабораторная 4 +# ЛР4 — вложения в мессенджере -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Кузнецов Даниил Русланович, БПИ2304. Целевая оценка — **3**. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Python 3.13 и Docker с Compose (на Windows — запущенный Docker Desktop). +Из папки репозитория: -## Архитектура и решения +```powershell +python -X utf8 scripts/lab.py verify +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Команда сама создаёт `.venv`, локальный `.env` и новые секреты, собирает образ, +запускает стенд и все проверки. Повторный запуск сохраняет настройки и данные. +API доступен на `http://127.0.0.1:18104`. TTL проверяется отдельно на порту 18114. -## Доказательства критериев +Для обычной работы: -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```powershell +python -X utf8 scripts/lab.py start +python -X utf8 scripts/lab.py test +python -X utf8 scripts/lab.py restart-check +python -X utf8 scripts/lab.py failover-check +python -X utf8 scripts/lab.py stop +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`stop` сохраняет volumes. Если нужно специально стереть учебные данные: +`docker compose -p kuznetsov-lab4 --env-file .env down --volumes`. -## Эксперименты и отказы +На Linux можно запускать те же команды через `python3`. +Команды преподавателя: `make check`, `make validate PYTHON=.venv/bin/python`, +`make test BASE_URL=http://127.0.0.1:18104`. +На Windows скрипт вызывает те же Python-проверки напрямую. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Что работает -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Клиент → HAProxy → две API-реплики → PostgreSQL + SeaweedFS. +Из ЛР3 сохранены парольный вход, Bearer-сессии, доступ по участникам чата, +курсорная пагинация и идемпотентная отправка сообщений. -## Ограничения и следующие шаги +Оригиналы лежат в приватном bucket `kuznetsov-lab4-attachments`, метаданные и связи +с сообщениями — в БД. Имена объектов назначает сервер. Загрузка принимает PNG/JPEG +до 10 MiB и 40 млн пикселей, проверяет MIME, имя, целостность и SHA-256. +Скачивание идёт через API с проверкой участника чата. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Тело пишется потоком во временный файл; файл и декодирование проверяются до публикации. +На реплику разрешены две обработки, ожидание и чтение ограничены 15 секундами. +S3 имеет таймауты 2/5 секунд; лимит контейнера API — 768 MiB. +Сначала выполняется S3 PUT, затем SQL-транзакция с метаданными. +Ответ `201 ready` возвращается после обоих сохранений. + +Python 3.13.14, PostgreSQL 17.11, SeaweedFS 4.54 и HAProxy 3.2.26 +закреплены в Dockerfile/Compose с digest; библиотеки — в lock-файлах. +Секреты передаются файлами. Наружу открыт только LB, у S3 и БД портов на хосте нет. + +## Проверки + +Полный сценарий: `python -X utf8 scripts/lab.py verify`. + +| Проверка | Результат | Доказательство | +| --- | --- | --- | +| Структура, OpenAPI, Compose, публичный smoke | PASS | [Лог](evidence/verification.txt) | +| Авторизация, Unicode, конкуренция, идемпотентность и файлы | 37 тестов, PASS | [Лог](evidence/verification.txt) | +| PNG/JPEG, точные байты и SHA-256, доступ участников, лимиты | PASS | [Тесты](student-tests/test_attachments.py) и лог | +| Прямое анонимное чтение S3 | 403 | [JSON](evidence/private-s3.json) | +| Перезапуск обеих API, БД, S3 и LB | Сессии, сообщения и файлы сохранены | [Рестарт](evidence/restart.json), [данные](evidence/persistence.json) | +| Остановка каждой загрузившей реплики | Другая возвращает тот же оригинал | [JSON](evidence/attachment-failover.json) | +| Идемпотентность после отказа обеих реплик | Без потерь подтверждённых сообщений и дублей | [JSON](evidence/failover.json) | +| Недоступность БД и S3 | live 200, ready/download 503; после восстановления 200 | [Сводка](evidence/summary.json) | +| Logout и TTL=2 после рестарта API | Отозванный/истёкший токен получает 401 | [JSON](evidence/expiry.json), [данные](evidence/persistence.json) | + +## Ограничения и источники + +Это локальный учебный стенд. Автоматической очистки остатков и эксперимента с памятью +для уровня 4 здесь нет. Если S3 PUT прошёл, а сохранение в БД не подтвердилось, +объект может остаться без метаданных. При неизвестном результате COMMIT слепое удаление +опасно: транзакция могла завершиться. Готовые вложения автоматически не удаляются. + +Основа — [реализация ЛР4 Артура Югая](https://gitea.privcloud.tech/ArturYugay/mtusi2026lab4/src/commit/8c57de52e02ac393fac7929878edfbecb9967f20). +Здесь отдельно подготовлен вариант на 3: упрощена публикация, убраны журнал и сервис +очистки, нагрузочный эксперимент; настройки, секреты, порты и проверки изолированы. +Задание, контракт и публичные тесты преподавателя сохранены. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..086a935c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,155 @@ +name: kuznetsov-lab4 + +x-app: &app + image: mtusi-messenger:kuznetsov-lab4 + build: . + environment: &environment + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-kuznetsov-lab4-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - database_url + - cursor_key + - s3_access_key_id + - s3_secret_access_key + stop_grace_period: 3s + +services: + db: + image: postgres:17.11-bookworm@sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + stop_grace_period: 10s + + s3-storage: + image: chrislusf/seaweedfs:4.54@sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15 + command: ["server", "-filer", "-dir=/data", "-ip=s3-storage", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=128", "-volume.max=64", "-master.electionTimeout=1s", + "-master.heartbeatInterval=100ms"] + volumes: + - s3data:/data + networks: + - storage + healthcheck: + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:9333/cluster/status | grep -q '\"IsLeader\":true'"] + interval: 2s + timeout: 2s + retries: 45 + stop_grace_period: 10s + + s3: + image: chrislusf/seaweedfs:4.54@sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15 + command: ["s3", "-filer=s3-storage:8888", "-port=9000", "-ip.bind=0.0.0.0", + "-config=/run/secrets/s3_config", "-iam=false", "-port.iceberg=0", "-port.lance=0"] + secrets: + - s3_config + networks: + - default + - storage + depends_on: + s3-storage: + condition: service_healthy + stop_grace_period: 3s + + migrate: + <<: *app + command: ["python", "-m", "src.migrate"] + secrets: + - database_url + healthcheck: + disable: true + depends_on: + db: + condition: service_healthy + + bootstrap: + <<: *app + command: ["python", "-m", "src.bootstrap"] + environment: + <<: *environment + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_access_key_id + - s3_secret_access_key + - s3_admin_access_key_id + - s3_admin_secret_access_key + healthcheck: + disable: true + depends_on: + s3: + condition: service_started + + api1: &api + <<: *app + mem_limit: 768m + depends_on: + db: + condition: service_healthy + migrate: + condition: service_completed_successfully + bootstrap: + condition: service_completed_successfully + + api2: + <<: *api + + + lb: + image: haproxy:3.2.26@sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953 + ports: + - "127.0.0.1:${HTTP_PORT:-18104}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api1: + condition: service_started + api2: + condition: service_started + stop_grace_period: 3s + +volumes: + pgdata: + s3data: + +networks: + storage: + internal: true + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cursor_key: + file: ./secrets/cursor_key + s3_config: + file: ./secrets/s3_config.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key diff --git a/evidence/attachment-failover.json b/evidence/attachment-failover.json new file mode 100644 index 00000000..247c8216 --- /dev/null +++ b/evidence/attachment-failover.json @@ -0,0 +1,23 @@ +{ + "status": "PASS", + "cases": [ + { + "stopped_service": "api1", + "upload_instance": "aed573c5dc1b427698cd736857260125", + "download_instance": "8686baec9bf349eaaf730210abd0f0f9", + "recovery_seconds": 5.177, + "transitional_errors": 3, + "same_original": true, + "message_preserved": true + }, + { + "stopped_service": "api2", + "upload_instance": "8686baec9bf349eaaf730210abd0f0f9", + "download_instance": "b0a517413dbf4ebc9e45ddb1aeaa17ed", + "recovery_seconds": 5.178, + "transitional_errors": 3, + "same_original": true, + "message_preserved": true + } + ] +} diff --git a/evidence/expiry.json b/evidence/expiry.json new file mode 100644 index 00000000..3e377cc3 --- /dev/null +++ b/evidence/expiry.json @@ -0,0 +1,7 @@ +{ + "ttl_seconds": 2, + "expiry_unchanged_on_restart": true, + "before_restart": 200, + "after_expiry_and_restart": 401, + "new_login": 201 +} diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 00000000..97314972 --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,157 @@ +{ + "public": { + "service": "api1", + "target_instance": "647fedc0b0b647b0b547ed5aef1f5807", + "before": { + "api1": { + "instance_id": "647fedc0b0b647b0b547ed5aef1f5807", + "container_id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa" + }, + "api2": { + "instance_id": "942c40bef1f445ac81c45a2be884bd92", + "container_id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782" + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.564801544Z", + "running": false, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.5053094Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "stop_seconds": 0.902, + "metrics": { + "observed_instances_before": 2, + "successful_writes": 102, + "failed_attempts": 4, + "first_recovery_seconds": 2.92 + } + }, + "second_replica": { + "service": "api2", + "target_instance": "942c40bef1f445ac81c45a2be884bd92", + "before": { + "api1": { + "instance_id": "aed573c5dc1b427698cd736857260125", + "container_id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa" + }, + "api2": { + "instance_id": "942c40bef1f445ac81c45a2be884bd92", + "container_id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782" + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:57:41.668499276Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.5053094Z", + "running": false, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "stop_seconds": 0.869, + "metrics": { + "successful_writes": 110, + "failed_attempts": 4, + "first_recovery_seconds": 3.111, + "replay_on_survivor": "PASS" + } + }, + "status": "PASS" +} diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..1c32346e --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,21 @@ +{ + "idempotency_after_restart": "PASS", + "public_persistence": "PASS", + "cursor_after_restart": "PASS", + "logout_after_restart": "PASS", + "migration_replay": "PASS", + "database": { + "users": 115, + "chats": 41, + "members": 80, + "messages": 138, + "sessions": 111, + "migrations": 3, + "idempotency_keys": 137, + "attachments": 13, + "message_attachments": 8, + "expiry_fingerprint": "86c021e07561e2fdb93b5c6f7af0bb4c", + "passwords_are_argon2id": true, + "session_hashes_only": true + } +} diff --git a/evidence/private-s3.json b/evidence/private-s3.json new file mode 100644 index 00000000..b9ce97e5 --- /dev/null +++ b/evidence/private-s3.json @@ -0,0 +1,3 @@ +{ + "anonymous_get": 403 +} diff --git a/evidence/restart.json b/evidence/restart.json new file mode 100644 index 00000000..b7071bee --- /dev/null +++ b/evidence/restart.json @@ -0,0 +1,115 @@ +{ + "project": "kuznetsov-lab4", + "before": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:31.941959576Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:32.080756562Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:20.249222906Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:32.342030698Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:23.090520985Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:20.435887024Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.564801544Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.5053094Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + } +} diff --git a/evidence/stop-one.json b/evidence/stop-one.json new file mode 100644 index 00000000..420494b4 --- /dev/null +++ b/evidence/stop-one.json @@ -0,0 +1,71 @@ +{ + "service": "api2", + "target_instance": "8686baec9bf349eaaf730210abd0f0f9", + "before": { + "api1": { + "instance_id": "b0a517413dbf4ebc9e45ddb1aeaa17ed", + "container_id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa" + }, + "api2": { + "instance_id": "8686baec9bf349eaaf730210abd0f0f9", + "container_id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782" + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:58:12.529976408Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:58:01.332272261Z", + "running": false, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "stop_seconds": 0.786 +} diff --git a/evidence/summary.json b/evidence/summary.json new file mode 100644 index 00000000..d7411e25 --- /dev/null +++ b/evidence/summary.json @@ -0,0 +1,294 @@ +{ + "date_utc": "2026-11-16T18:55:42.477638+00:00", + "platform": "Windows-11-10.0.26200-SP0", + "source_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5", + "worktree_dirty_at_start": true, + "project": "kuznetsov-lab4", + "status": "PASS", + "private_s3": { + "anonymous_get": 403 + }, + "persistence": { + "idempotency_after_restart": "PASS", + "public_persistence": "PASS", + "cursor_after_restart": "PASS", + "logout_after_restart": "PASS", + "migration_replay": "PASS", + "database": { + "users": 115, + "chats": 41, + "members": 80, + "messages": 138, + "sessions": 111, + "migrations": 3, + "idempotency_keys": 137, + "attachments": 13, + "message_attachments": 8, + "expiry_fingerprint": "86c021e07561e2fdb93b5c6f7af0bb4c", + "passwords_are_argon2id": true, + "session_hashes_only": true + } + }, + "failover": { + "public": { + "service": "api1", + "target_instance": "647fedc0b0b647b0b547ed5aef1f5807", + "before": { + "api1": { + "instance_id": "647fedc0b0b647b0b547ed5aef1f5807", + "container_id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa" + }, + "api2": { + "instance_id": "942c40bef1f445ac81c45a2be884bd92", + "container_id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782" + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.564801544Z", + "running": false, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.5053094Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "stop_seconds": 0.902, + "metrics": { + "observed_instances_before": 2, + "successful_writes": 102, + "failed_attempts": 4, + "first_recovery_seconds": 2.92 + } + }, + "second_replica": { + "service": "api2", + "target_instance": "942c40bef1f445ac81c45a2be884bd92", + "before": { + "api1": { + "instance_id": "aed573c5dc1b427698cd736857260125", + "container_id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa" + }, + "api2": { + "instance_id": "942c40bef1f445ac81c45a2be884bd92", + "container_id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782" + } + }, + "after": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:57:41.668499276Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:56:52.5053094Z", + "running": false, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:56:52.452142757Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:56:52.647668601Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "stop_seconds": 0.869, + "metrics": { + "successful_writes": 110, + "failed_attempts": 4, + "first_recovery_seconds": 3.111, + "replay_on_survivor": "PASS" + } + }, + "status": "PASS" + }, + "attachments_failover": { + "status": "PASS", + "cases": [ + { + "stopped_service": "api1", + "upload_instance": "aed573c5dc1b427698cd736857260125", + "download_instance": "8686baec9bf349eaaf730210abd0f0f9", + "recovery_seconds": 5.177, + "transitional_errors": 3, + "same_original": true, + "message_preserved": true + }, + { + "stopped_service": "api2", + "upload_instance": "8686baec9bf349eaaf730210abd0f0f9", + "download_instance": "b0a517413dbf4ebc9e45ddb1aeaa17ed", + "recovery_seconds": 5.178, + "transitional_errors": 3, + "same_original": true, + "message_preserved": true + } + ] + }, + "expiry": { + "ttl_seconds": 2, + "expiry_unchanged_on_restart": true, + "before_restart": 200, + "after_expiry_and_restart": 401, + "new_login": 201 + }, + "dependency": { + "db": { + "live": 200, + "ready": 503, + "download": 503, + "recovered": 200 + }, + "s3": { + "live": 200, + "ready": 503, + "download": 503, + "recovered": 200 + } + }, + "containers": { + "api1": { + "id": "4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:58:12.529976408Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "api2": { + "id": "aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782", + "image": "sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987", + "started_at": "2026-11-16T18:58:22.264835326Z", + "running": true, + "user": "messenger", + "volumes": [] + }, + "db": { + "id": "e5b5dc582a71e2f0aaed8211dd36d8907451dac0758e70d535d088718a4f1f4d", + "image": "sha256:665dc7b37ac52a17e298511414c21d23ffa4e6a6ef6225b3e5d59c7afb35e994", + "started_at": "2026-11-16T18:59:07.136566022Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_pgdata" + ] + }, + "lb": { + "id": "d22e99234e628c4cbee573a4fb0b59d65ed7288e007b7344a88ae846c2db356c", + "image": "sha256:1242e4006ac3f1ed15871efadca3d8d089e76f720017a1c51747133ee40cc953", + "started_at": "2026-11-16T18:56:52.303157992Z", + "running": true, + "user": "haproxy", + "volumes": [] + }, + "s3": { + "id": "3b7b10a5cf8e4fb9a4037439d07236b7424159509aba68df2e4dc6c90cf2ccdd", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:59:19.588694013Z", + "running": true, + "user": "", + "volumes": [ + "d4f9b2835d6deeb9a0545f16daa10201cccbbb854acffcaa757632dfcf78c66d" + ] + }, + "s3-storage": { + "id": "1a0e4fea052e90d11bb4880e83bd6e69e9604e57288059ea8b41f051433e1a83", + "image": "sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15", + "started_at": "2026-11-16T18:57:02.11531029Z", + "running": true, + "user": "", + "volumes": [ + "kuznetsov-lab4_s3data" + ] + } + }, + "docker": "29.9.3", + "resources": { + "cpus": 12, + "memory_bytes": 16591364096 + } +} diff --git a/evidence/verification.txt b/evidence/verification.txt new file mode 100644 index 00000000..76690cbb --- /dev/null +++ b/evidence/verification.txt @@ -0,0 +1,377 @@ +Environment ready; existing settings and secrets preserved +$ ".\.venv\Scripts\python.exe" -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ ".\.venv\Scripts\python.exe" -X utf8 scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +$ docker compose --project-name kuznetsov-lab4 --env-file .env config --quiet +$ docker compose --project-name kuznetsov-lab4 --env-file .env config --quiet +$ docker compose --project-name kuznetsov-lab4 --env-file .env build api1 + Image mtusi-messenger:kuznetsov-lab4 Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.73kB 0.0s done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 728B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.13.14-slim-bookworm@sha256:67a1e1f215ccda113cfc024e8639049257e88f273898f595b61476d128d387e8 +#3 ... + +#4 [auth] library/python:pull token for registry-1.docker.io +#4 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.13.14-slim-bookworm@sha256:67a1e1f215ccda113cfc024e8639049257e88f273898f595b61476d128d387e8 +#3 DONE 1.3s + +#5 [internal] load .dockerignore +#5 transferring context: 246B done +#5 DONE 0.0s + +#6 [internal] load build context +#6 transferring context: 44.11kB 0.0s done +#6 DONE 0.0s + +#7 [1/6] FROM docker.io/library/python:3.13.14-slim-bookworm@sha256:67a1e1f215ccda113cfc024e8639049257e88f273898f595b61476d128d387e8 +#7 resolve docker.io/library/python:3.13.14-slim-bookworm@sha256:67a1e1f215ccda113cfc024e8639049257e88f273898f595b61476d128d387e8 0.0s done +#7 DONE 0.0s + +#8 [3/6] COPY requirements.lock . +#8 CACHED + +#9 [2/6] WORKDIR /app +#9 CACHED + +#10 [4/6] RUN python -m pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home messenger +#10 CACHED + +#11 [5/6] COPY src ./src +#11 DONE 0.0s + +#12 [6/6] COPY migrations ./migrations +#12 DONE 0.0s + +#13 exporting to image +#13 exporting layers 0.1s done +#13 exporting manifest sha256:f8d7061a61c5b0d6dc38ff63b5c7e5fb23d9411fcf9d7071d5386919cacb1f6f 0.0s done +#13 exporting config sha256:09a3a64b7d7c548e9e5a88cceac9634b397fea6b696a7507c42a6a61f7da75a8 +#13 exporting config sha256:09a3a64b7d7c548e9e5a88cceac9634b397fea6b696a7507c42a6a61f7da75a8 0.0s done +#13 exporting attestation manifest sha256:96d8556413fd5c95153ab97264fe9d52418497607095b39a1db7be85ab6f8c5c 0.0s done +#13 exporting manifest list sha256:eee390a810c74f1161be7bf32b7af98376bc917cc6fcb6608a45e2e084a6c987 0.0s done +#13 naming to docker.io/library/mtusi-messenger:kuznetsov-lab4 done +#13 unpacking to docker.io/library/mtusi-messenger:kuznetsov-lab4 0.0s done +#13 DONE 0.2s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:kuznetsov-lab4 Built +$ docker compose --project-name kuznetsov-lab4 --env-file .env run --rm --no-deps lb haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg + Volume kuznetsov-lab4_s3data Creating + Volume kuznetsov-lab4_s3data Creating + Volume kuznetsov-lab4_pgdata Creating + Volume kuznetsov-lab4_pgdata Creating + Network kuznetsov-lab4_storage Creating + Network kuznetsov-lab4_default Creating + Network kuznetsov-lab4_default Creating + Network kuznetsov-lab4_storage Creating + Volume kuznetsov-lab4_s3data Created + Volume kuznetsov-lab4_s3data Created + Volume kuznetsov-lab4_pgdata Created + Volume kuznetsov-lab4_pgdata Created + Network kuznetsov-lab4_default Created + Network kuznetsov-lab4_default Created + Network kuznetsov-lab4_storage Created + Network kuznetsov-lab4_storage Created + Container kuznetsov-lab4-lb-run-6e253a61e397 Creating + Container kuznetsov-lab4-lb-run-6e253a61e397 Created +[NOTICE] (1) : [/usr/local/etc/haproxy/haproxy.cfg:35] : 'server api_ready/first' : could not resolve address 'api1', disabling server. +[NOTICE] (1) : [/usr/local/etc/haproxy/haproxy.cfg:36] : 'server api_ready/second' : could not resolve address 'api2', disabling server. +[NOTICE] (1) : [/usr/local/etc/haproxy/haproxy.cfg:43] : 'server api_live/first' : could not resolve address 'api1', disabling server. +[NOTICE] (1) : [/usr/local/etc/haproxy/haproxy.cfg:44] : 'server api_live/second' : could not resolve address 'api2', disabling server. +$ docker compose --project-name kuznetsov-lab4 --env-file .env up -d --no-build db migrate s3-storage s3 bootstrap api1 api2 lb + Container kuznetsov-lab4-db-1 Creating + Container kuznetsov-lab4-s3-storage-1 Creating + Container kuznetsov-lab4-s3-storage-1 Created + Container kuznetsov-lab4-s3-1 Creating + Container kuznetsov-lab4-db-1 Created + Container kuznetsov-lab4-migrate-1 Creating + Container kuznetsov-lab4-s3-1 Created + Container kuznetsov-lab4-bootstrap-1 Creating + Container kuznetsov-lab4-migrate-1 Created + Container kuznetsov-lab4-bootstrap-1 Created + Container kuznetsov-lab4-api1-1 Creating + Container kuznetsov-lab4-api2-1 Creating + Container kuznetsov-lab4-api1-1 Created + Container kuznetsov-lab4-api2-1 Created + Container kuznetsov-lab4-lb-1 Creating + Container kuznetsov-lab4-lb-1 Created + Container kuznetsov-lab4-db-1 Starting + Container kuznetsov-lab4-s3-storage-1 Starting + Container kuznetsov-lab4-db-1 Started + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Started + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-s3-1 Starting + Container kuznetsov-lab4-s3-1 Started + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited + Container kuznetsov-lab4-api1-1 Starting + Container kuznetsov-lab4-api2-1 Starting + Container kuznetsov-lab4-api1-1 Started + Container kuznetsov-lab4-api2-1 Started + Container kuznetsov-lab4-lb-1 Starting + Container kuznetsov-lab4-lb-1 Started +Two API replicas ready through LB: http://127.0.0.1:18104 +$ ".\.venv\Scripts\python.exe" -X utf8 tests/smoke.py --base-url http://127.0.0.1:18104 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +$ ".\.venv\Scripts\python.exe" -X utf8 -m unittest discover -s student-tests -v +test_auth_and_access (test_api.ApiTests.test_auth_and_access) ... ok +test_bad_cursors_and_limits (test_api.ApiTests.test_bad_cursors_and_limits) ... ok +test_chat_pagination (test_api.ApiTests.test_chat_pagination) ... ok +test_chat_validation_is_atomic (test_api.ApiTests.test_chat_validation_is_atomic) ... ok +test_invalid_json_and_content_type (test_api.ApiTests.test_invalid_json_and_content_type) ... ok +test_message_fields_and_unicode (test_api.ApiTests.test_message_fields_and_unicode) ... ok +test_message_pagination (test_api.ApiTests.test_message_pagination) ... ok +test_parallel_messages (test_api.ApiTests.test_parallel_messages) ... ok +test_parallel_username (test_api.ApiTests.test_parallel_username) ... ok +test_read_after_write (test_api.ApiTests.test_read_after_write) ... ok +test_user_fields (test_api.ApiTests.test_user_fields) ... ok +test_attachment_access_and_unknown_ids (test_attachments.AttachmentTests.test_attachment_access_and_unknown_ids) ... ok +test_bad_attachment_list_and_other_chat_do_not_create_messages (test_attachments.AttachmentTests.test_bad_attachment_list_and_other_chat_do_not_create_messages) ... ok +test_byte_limit_and_original_at_exact_boundary (test_attachments.AttachmentTests.test_byte_limit_and_original_at_exact_boundary) ... ok +test_chunked_body_cannot_bypass_size_limit (test_attachments.AttachmentTests.test_chunked_body_cannot_bypass_size_limit) ... ok +test_corrupt_images_mime_and_multipart (test_attachments.AttachmentTests.test_corrupt_images_mime_and_multipart) ... ok +test_filename_validation (test_attachments.AttachmentTests.test_filename_validation) ... ok +test_idempotency_defaults_and_attachment_replay (test_attachments.AttachmentTests.test_idempotency_defaults_and_attachment_replay) ... ok +test_messages_keep_attachment_order_and_allow_empty_text (test_attachments.AttachmentTests.test_messages_keep_attachment_order_and_allow_empty_text) ... ok +test_original_png_jpeg_and_unicode_filename (test_attachments.AttachmentTests.test_original_png_jpeg_and_unicode_filename) ... ok +test_pixel_boundary_and_decompression_bomb (test_attachments.AttachmentTests.test_pixel_boundary_and_decompression_bomb) ... ok +test_different_body_conflicts (test_idempotency.IdempotencyTests.test_different_body_conflicts) ... ok +test_failure_rolls_back_message_and_key (test_idempotency.IdempotencyTests.test_failure_rolls_back_message_and_key) ... ok +test_invalid_body_does_not_reserve_key (test_idempotency.IdempotencyTests.test_invalid_body_does_not_reserve_key) ... ok +test_key_required_and_format (test_idempotency.IdempotencyTests.test_key_required_and_format) ... ok +test_replay_normalizes_json (test_idempotency.IdempotencyTests.test_replay_normalizes_json) ... ok +test_replay_still_checks_access_and_session (test_idempotency.IdempotencyTests.test_replay_still_checks_access_and_session) ... ok +test_retention_and_expired_key_reuse (test_idempotency.IdempotencyTests.test_retention_and_expired_key_reuse) ... ok +test_same_session_on_both_instances (test_idempotency.IdempotencyTests.test_same_session_on_both_instances) ... ok +test_scope_is_user_and_chat_not_session (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat_not_session) ... ok +test_twenty_concurrent_requests_use_both_replicas (test_idempotency.IdempotencyTests.test_twenty_concurrent_requests_use_both_replicas) ... ok +test_logout_only_revokes_current_session (test_sessions.SessionTests.test_logout_only_revokes_current_session) ... ok +test_malformed_basic_and_body (test_sessions.SessionTests.test_malformed_basic_and_body) ... ok +test_only_bearer_identifies_user (test_sessions.SessionTests.test_only_bearer_identifies_user) ... ok +test_password_boundaries_unicode_colon_spaces (test_sessions.SessionTests.test_password_boundaries_unicode_colon_spaces) ... ok +test_password_required_and_limits (test_sessions.SessionTests.test_password_required_and_limits) ... ok +test_wrong_and_unknown_credentials (test_sessions.SessionTests.test_wrong_and_unknown_credentials) ... ok + +---------------------------------------------------------------------- +Ran 37 tests in 13.371s + +OK +PASS: unsigned direct S3 GET denied +$ ".\.venv\Scripts\python.exe" -X utf8 tests/resilience.py --mode persistence --action-script ".\scripts\restart.cmd" --base-url http://127.0.0.1:18104 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker compose --project-name kuznetsov-lab4 --env-file .env run --rm --no-deps migrate + Container kuznetsov-lab4-migrate-run-8b8da4ce1b7a Creating + Container kuznetsov-lab4-migrate-run-8b8da4ce1b7a Created +Already applied: 001_initial.sql +Already applied: 002_idempotency.sql +Already applied: 003_attachments.sql +$ docker compose --project-name kuznetsov-lab4 --env-file .env run --rm --no-deps migrate + Container kuznetsov-lab4-migrate-run-af84cd18946a Creating + Container kuznetsov-lab4-migrate-run-af84cd18946a Created +Already applied: 001_initial.sql +Already applied: 002_idempotency.sql +Already applied: 003_attachments.sql +PASS: old session/cursor, durable logout and repeatable migrations +$ ".\.venv\Scripts\python.exe" -X utf8 tests/resilience.py --mode failover --action-script ".\scripts\stop-one.cmd" --base-url http://127.0.0.1:18104 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 102, "failed_attempts": 4, "first_recovery_seconds": 2.92} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker compose --project-name kuznetsov-lab4 --env-file .env start api1 api2 + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited + Container kuznetsov-lab4-api1-1 Starting + Container kuznetsov-lab4-api1-1 Started +PASS: both physical API replicas stopped separately; no lost messages or duplicates +$ docker compose --project-name kuznetsov-lab4 --env-file .env start api1 api2 + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited + Container kuznetsov-lab4-api2-1 Starting + Container kuznetsov-lab4-api2-1 Started +$ ".\scripts\stop-one.cmd" +$ docker stop --timeout 3 4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa +4b7c839ee38b9f9a972dcf2bbc6c31a089bcba584db20444467fe7f8209972aa +Stopped only api1 +$ docker compose --project-name kuznetsov-lab4 --env-file .env start api1 api2 + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited + Container kuznetsov-lab4-api1-1 Starting + Container kuznetsov-lab4-api1-1 Started +$ ".\scripts\stop-one.cmd" +$ docker stop --timeout 3 aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782 +aa3d724d56ac972c06f6d3a65daf6249cc9a20165c9c829cac7fc3615cc09782 +Stopped only api2 +$ docker compose --project-name kuznetsov-lab4 --env-file .env start api1 api2 + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited + Container kuznetsov-lab4-api2-1 Starting + Container kuznetsov-lab4-api2-1 Started +PASS: stopped each uploader; other API downloaded the same original +$ docker compose --project-name kuznetsov-lab4 --env-file .env start api1 api2 + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-db-1 Waiting + Container kuznetsov-lab4-db-1 Healthy + Container kuznetsov-lab4-migrate-1 Starting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-bootstrap-1 Starting + Container kuznetsov-lab4-migrate-1 Started + Container kuznetsov-lab4-migrate-1 Waiting + Container kuznetsov-lab4-bootstrap-1 Started + Container kuznetsov-lab4-bootstrap-1 Waiting + Container kuznetsov-lab4-migrate-1 Exited + Container kuznetsov-lab4-bootstrap-1 Exited +$ docker compose --project-name kuznetsov-lab4-ttl --env-file .env up -d --no-build db migrate s3-storage s3 bootstrap api1 api2 lb + Volume kuznetsov-lab4-ttl_pgdata Creating + Volume kuznetsov-lab4-ttl_pgdata Creating + Network kuznetsov-lab4-ttl_storage Creating + Volume kuznetsov-lab4-ttl_s3data Creating + Network kuznetsov-lab4-ttl_default Creating + Network kuznetsov-lab4-ttl_storage Creating + Volume kuznetsov-lab4-ttl_s3data Creating + Network kuznetsov-lab4-ttl_default Creating + Volume kuznetsov-lab4-ttl_pgdata Created + Volume kuznetsov-lab4-ttl_pgdata Created + Volume kuznetsov-lab4-ttl_s3data Created + Volume kuznetsov-lab4-ttl_s3data Created + Network kuznetsov-lab4-ttl_storage Created + Network kuznetsov-lab4-ttl_storage Created + Container kuznetsov-lab4-ttl-s3-storage-1 Creating + Network kuznetsov-lab4-ttl_default Created + Network kuznetsov-lab4-ttl_default Created + Container kuznetsov-lab4-ttl-db-1 Creating + Container kuznetsov-lab4-ttl-s3-storage-1 Created + Container kuznetsov-lab4-ttl-s3-1 Creating + Container kuznetsov-lab4-ttl-db-1 Created + Container kuznetsov-lab4-ttl-migrate-1 Creating + Container kuznetsov-lab4-ttl-s3-1 Created + Container kuznetsov-lab4-ttl-bootstrap-1 Creating + Container kuznetsov-lab4-ttl-migrate-1 Created + Container kuznetsov-lab4-ttl-bootstrap-1 Created + Container kuznetsov-lab4-ttl-api1-1 Creating + Container kuznetsov-lab4-ttl-api2-1 Creating + Container kuznetsov-lab4-ttl-api2-1 Created + Container kuznetsov-lab4-ttl-api1-1 Created + Container kuznetsov-lab4-ttl-lb-1 Creating + Container kuznetsov-lab4-ttl-lb-1 Created + Container kuznetsov-lab4-ttl-db-1 Starting + Container kuznetsov-lab4-ttl-s3-storage-1 Starting + Container kuznetsov-lab4-ttl-db-1 Started + Container kuznetsov-lab4-ttl-db-1 Waiting + Container kuznetsov-lab4-ttl-s3-storage-1 Started + Container kuznetsov-lab4-ttl-s3-storage-1 Waiting + Container kuznetsov-lab4-ttl-db-1 Healthy + Container kuznetsov-lab4-ttl-migrate-1 Starting + Container kuznetsov-lab4-ttl-migrate-1 Started + Container kuznetsov-lab4-ttl-migrate-1 Waiting + Container kuznetsov-lab4-ttl-s3-storage-1 Healthy + Container kuznetsov-lab4-ttl-s3-1 Starting + Container kuznetsov-lab4-ttl-s3-1 Started + Container kuznetsov-lab4-ttl-bootstrap-1 Starting + Container kuznetsov-lab4-ttl-bootstrap-1 Started + Container kuznetsov-lab4-ttl-bootstrap-1 Waiting + Container kuznetsov-lab4-ttl-migrate-1 Exited + Container kuznetsov-lab4-ttl-bootstrap-1 Exited + Container kuznetsov-lab4-ttl-api1-1 Starting + Container kuznetsov-lab4-ttl-api2-1 Starting + Container kuznetsov-lab4-ttl-api1-1 Started + Container kuznetsov-lab4-ttl-api2-1 Started + Container kuznetsov-lab4-ttl-lb-1 Starting + Container kuznetsov-lab4-ttl-lb-1 Started +Two API replicas ready through LB: http://127.0.0.1:18114 +$ docker compose --project-name kuznetsov-lab4-ttl --env-file .env restart --timeout 3 api1 api2 + Container kuznetsov-lab4-ttl-api2-1 Restarting + Container kuznetsov-lab4-ttl-api1-1 Restarting + Container kuznetsov-lab4-ttl-api1-1 Started + Container kuznetsov-lab4-ttl-api2-1 Started +PASS: TTL=2, expired token rejected after API restart; fresh login works +$ docker compose --project-name kuznetsov-lab4-ttl --env-file .env stop --timeout 10 lb api1 api2 s3 s3-storage db + Container kuznetsov-lab4-ttl-lb-1 Stopping + Container kuznetsov-lab4-ttl-lb-1 Stopped + Container kuznetsov-lab4-ttl-api1-1 Stopping + Container kuznetsov-lab4-ttl-api2-1 Stopping + Container kuznetsov-lab4-ttl-api1-1 Stopped + Container kuznetsov-lab4-ttl-api2-1 Stopped + Container kuznetsov-lab4-ttl-db-1 Stopping + Container kuznetsov-lab4-ttl-s3-1 Stopping + Container kuznetsov-lab4-ttl-db-1 Stopped + Container kuznetsov-lab4-ttl-s3-1 Stopped + Container kuznetsov-lab4-ttl-s3-storage-1 Stopping + Container kuznetsov-lab4-ttl-s3-storage-1 Stopped +$ docker compose --project-name kuznetsov-lab4 --env-file .env stop --timeout 10 db + Container kuznetsov-lab4-db-1 Stopping + Container kuznetsov-lab4-db-1 Stopped +$ docker compose --project-name kuznetsov-lab4 --env-file .env start db + Container kuznetsov-lab4-db-1 Starting + Container kuznetsov-lab4-db-1 Started +PASS: db outage returns bounded errors; session and file work after recovery +$ docker compose --project-name kuznetsov-lab4 --env-file .env stop --timeout 10 s3 + Container kuznetsov-lab4-s3-1 Stopping + Container kuznetsov-lab4-s3-1 Stopped +$ docker compose --project-name kuznetsov-lab4 --env-file .env start s3 + Container kuznetsov-lab4-s3-storage-1 Waiting + Container kuznetsov-lab4-s3-storage-1 Healthy + Container kuznetsov-lab4-s3-1 Starting + Container kuznetsov-lab4-s3-1 Started +PASS: s3 outage returns bounded errors; session and file work after recovery +$ ".\.venv\Scripts\python.exe" -X utf8 tests/smoke.py --base-url http://127.0.0.1:18104 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +PASS: full lab4 verification diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..7f3de21f --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,44 @@ +global + log stdout format raw local0 + maxconn 256 + +defaults + mode http + log global + timeout connect 1s + timeout client 60s + timeout server 60s + timeout check 3s + retries 0 + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] + +resolvers docker + nameserver local 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold nx 1s + hold obsolete 1s + +frontend public + bind :8080 + unique-id-format %[uuid()] + use_backend api_live if { path /health/live } + default_backend api_ready + +backend api_ready + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server first api1:8080 check + server second api2:8080 check + +backend api_live + balance roundrobin + option httpchk GET /health/live + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server first api1:8080 check + server second api2:8080 check diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..c738da85 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,40 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CHECK (expires_at > created_at) +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..46a6ffe5 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,15 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id), + method varchar(8) NOT NULL CHECK (method = 'POST'), + path varchar(128) NOT NULL, + request_key varchar(128) NOT NULL CHECK (request_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash char(64) NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + response_status smallint NOT NULL CHECK (response_status = 201), + response_body jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, path, request_key), + CHECK (expires_at > created_at) +); +CREATE INDEX idempotency_expiry_idx ON idempotency_keys(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..a6af80a0 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,29 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); diff --git a/requirements-validation.lock b/requirements-validation.lock new file mode 100644 index 00000000..994b5fe6 --- /dev/null +++ b/requirements-validation.lock @@ -0,0 +1,18 @@ +attrs==26.1.0 +certifi==2026.10.23 +charset-normalizer==3.5.2 +idna==3.20 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +pathable==0.4.4 +pyyaml==6.0.3 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +urllib3==2.8.0 diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..bef65a47 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,30 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.16.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +boto3==1.44.3 +botocore==1.44.3 +cffi==2.1.1 +click==8.5.1 +fastapi==0.162.2 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +opentelemetry-api==1.45.2 +pillow==13.0.0 +psycopg==3.3.8 +psycopg-binary==3.3.8 +psycopg-pool==3.3.3 +pycparser==3.12 +pydantic==2.14.3 +pydantic_core==2.50.4 +python-dateutil==2.9.0.post0 +s3transfer==0.19.3 +six==1.17.0 +starlette==1.10.0 +typing-inspection==0.4.5 +typing_extensions==4.16.0 +tzdata==2026.6 +urllib3==2.8.0 +uvicorn==0.54.0 diff --git a/scripts/lab.py b/scripts/lab.py new file mode 100644 index 00000000..11c0f4c2 --- /dev/null +++ b/scripts/lab.py @@ -0,0 +1,677 @@ +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import platform +import secrets +import shutil +import socket +import subprocess +import sys +import time +from urllib.error import URLError +from urllib.request import ProxyHandler, build_opener +import venv + + +ROOT = Path(__file__).resolve().parents[1] +LOG = [] +LOCAL_HTTP = build_opener(ProxyHandler({})) + + +def settings(): + path = ROOT / ".env" + result = {} + if path.exists(): + for line in path.read_text(encoding="utf-8-sig").splitlines(): + key, separator, value = line.partition("=") + if separator and not key.strip().startswith("#"): + result[key.strip()] = value.strip() + return result + + +def environment(args): + result = os.environ.copy() + result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1", + COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port), + SESSION_TTL_SECONDS=str(args.ttl), LAB4_PROJECT=args.project, + LAB4_PORT=str(args.port), LAB4_TTL=str(args.ttl)) + for key in ("NO_PROXY", "no_proxy"): + result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"])) + return result + + +def say(value): + line = str(value).replace(str(ROOT), ".") + print(line, flush=True) + LOG.append(line) + + +def run(args, command, quiet=False): + command = [str(item) for item in command] + if not quiet: + say("$ " + subprocess.list2cmdline(command)) + with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process: + lines = [] + for line in process.stdout: + lines.append(line) + if not quiet: + say(line.rstrip()) + code = process.wait() + if code: + raise RuntimeError(f"{command[0]} exited with code {code}") + return "".join(lines) + + +def compose(args, *command, quiet=False): + if args.project != "kuznetsov-lab4" and not args.project.startswith("kuznetsov-lab4-"): + raise RuntimeError("This runner only manages Kuznetsov lab4 projects") + return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet) + + +def interpreter(): + return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + + +def evidence(name, value): + path = ROOT / "evidence" / name + path.parent.mkdir(exist_ok=True) + content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value + path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n") + + +def setup(args): + if not (ROOT / ".env").exists(): + shutil.copyfile(ROOT / ".env.example", ROOT / ".env") + folder = ROOT / "secrets" + folder.mkdir(exist_ok=True) + password_file = folder / "postgres_password" + if not password_file.exists(): + password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii") + password = password_file.read_text(encoding="ascii").strip() + values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger", + "cursor_key": secrets.token_hex(32)} + for name, value in values.items(): + path = folder / name + if not path.exists(): + path.write_text(value + "\n", encoding="ascii") + if os.name != "nt": + path.chmod(0o644) + if os.name != "nt": + folder.chmod(0o700) + password_file.chmod(0o644) + for name in ("s3_access_key_id", "s3_secret_access_key", "s3_admin_access_key_id", "s3_admin_secret_access_key"): + path = folder / name + if not path.exists(): + path.write_text(secrets.token_hex(32) + "\n", encoding="ascii") + if os.name != "nt": + path.chmod(0o644) + config = folder / "s3_config.json" + if not config.exists(): + def key(name): + return (folder / name).read_text(encoding="ascii").strip() + bucket = settings().get("S3_BUCKET", "kuznetsov-lab4-attachments") + identities = [ + {"name": "bootstrap", "credentials": [{"accessKey": key("s3_admin_access_key_id"), + "secretKey": key("s3_admin_secret_access_key")}], "actions": ["Admin", "Read", "Write", "List"]}, + {"name": "messenger", "credentials": [{"accessKey": key("s3_access_key_id"), + "secretKey": key("s3_secret_access_key")}], + "actions": [action + ":" + bucket for action in ("Read", "Write", "List")]}, + ] + config.write_text(json.dumps({"identities": identities}) + "\n", encoding="ascii") + if os.name != "nt": + config.chmod(0o644) + if not interpreter().exists(): + venv.EnvBuilder(with_pip=True).create(ROOT / ".venv") + lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"] + fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest() + marker = ROOT / ".venv/.lab-dependencies" + if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint: + run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check", + "--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]]) + marker.write_text(fingerprint, encoding="ascii") + say("Environment ready; existing settings and secrets preserved") + + +def base_url(args): + return f"http://127.0.0.1:{args.port}" + + +def wait_storage(args, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + container = compose(args, "ps", "--all", "--quiet", "s3-storage", quiet=True).strip() + if container: + status = run(args, ["docker", "inspect", "--format", "{{.State.Health.Status}}", container], quiet=True).strip() + if status == "healthy": + return + time.sleep(0.5) + raise RuntimeError("S3 storage readiness timeout") + + +def wait_ready(args, timeout=90): + wait_storage(args, timeout) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response: + if response.status == 200 and json.load(response) == {"status": "ok"}: + return + except (URLError, OSError, ValueError): + pass + time.sleep(0.3) + raise RuntimeError("API readiness timeout") + + +def build(args): + compose(args, "config", "--quiet") + compose(args, "build", "api1") + compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f", + "/usr/local/etc/haproxy/haproxy.cfg") + + +def start(args): + current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip() + if not current: + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", args.port)) + except OSError: + raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None + compose(args, "up", "-d", "--no-build", "db", "migrate", "s3-storage", "s3", "bootstrap", "api1", "api2", "lb") + wait_ready(args) + wait_replicas(args) + say("Two API replicas ready through LB: " + base_url(args)) + + +def states(args): + services = ("api1", "api2", "db", "lb", "s3", "s3-storage") + containers = compose(args, "ps", "--all", "--quiet", *services, quiet=True).split() + if len(set(containers)) != len(services): + raise RuntimeError("Missing runtime container") + result = {} + for data in json.loads(run(args, ["docker", "inspect", *containers], quiet=True)): + labels = data["Config"].get("Labels", {}) + service = labels.get("com.docker.compose.service") + if labels.get("com.docker.compose.project") != args.project or service not in services or service in result: + raise RuntimeError("Unexpected container ownership") + result[service] = { + "id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"], + "running": data["State"]["Running"], "user": data["Config"]["User"], + "volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"], + } + return result + + +def restart(args): + before = states(args) + if not all(item["running"] for item in before.values()): + raise RuntimeError("All runtime containers must be running before restart") + compose(args, "restart", "--timeout", "10", "db", "s3-storage", "s3", "api1", "api2", "lb") + wait_storage(args) + compose(args, "run", "--rm", "--no-deps", "bootstrap") + after = states(args) + for service in before: + if before[service]["started_at"] == after[service]["started_at"]: + raise RuntimeError("Container was not restarted: " + service) + if before[service]["volumes"] != after[service]["volumes"]: + raise RuntimeError("Volume changed during restart") + evidence("restart.json", {"project": args.project, "before": before, "after": after}) + say("PASS: both APIs, PostgreSQL, S3 and LB restarted, volumes preserved") + + +def contract_tests(args): + run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"]) + run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"]) + compose(args, "config", "--quiet") + + +def api_tests(args): + wait_ready(args) + run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)]) + os.environ["BASE_URL"] = base_url(args) + run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"]) + + +def client(args): + sys.path.insert(0, str(ROOT / "tests")) + from client import Client + for key in ("NO_PROXY", "no_proxy"): + os.environ[key] = environment(args)[key] + return Client(base_url(args)) + + +def database_stats(args): + sql = """SELECT json_build_object( + 'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats), + 'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages), + 'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations), + 'idempotency_keys',(SELECT count(*) FROM idempotency_keys), + 'attachments',(SELECT count(*) FROM attachments),'message_attachments',(SELECT count(*) FROM message_attachments), + 'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text, + ',' ORDER BY token_hash),'')) FROM sessions), + 'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users), + 'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))""" + result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-c", sql, quiet=True) + return json.loads(result) + + +def restart_check(args): + from urllib.parse import urlencode + c = client(args) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + replay_key = secrets.token_hex(16) + original = c.message(alice, chat, "first", key=replay_key)["json"] + last = c.message(bob, chat, "second")["json"] + page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"] + revoked = c.user("revoked") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked) + hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence", + "--action-script", hook, "--base-url", base_url(args)]) + continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}), + actor=alice, schema="MessagePage")["json"] + if continued != {"items": [last], "next_cursor": None}: + raise RuntimeError("Old cursor did not survive restart") + c.call("GET", "/api/v1/users/me", 401, actor=revoked) + c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve) + before = database_stats(args) + compose(args, "run", "--rm", "--no-deps", "migrate") + compose(args, "run", "--rm", "--no-deps", "migrate") + after = database_stats(args) + if before != after or after["migrations"] != 3: + raise RuntimeError("Migration replay changed existing data") + if not after["passwords_are_argon2id"] or not after["session_hashes_only"]: + raise RuntimeError("Unexpected credential representation in DB") + repeated = c.message(alice, chat, "first", key=replay_key)["json"] + if repeated != original: + raise RuntimeError("Idempotency result was not preserved") + result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS", + "logout_after_restart": "PASS", "migration_replay": "PASS", "database": after} + evidence("persistence.json", result) + say("PASS: old session/cursor, durable logout and repeatable migrations") + return result + + +def ttl_check(args): + test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2, + test_port=args.test_port, action="start") + try: + start(test) + c = client(test) + actor = c.user("expiry") + c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + before = database_stats(test)["expiry_fingerprint"] + compose(test, "restart", "--timeout", "3", "api1", "api2") + wait_ready(test) + after = database_stats(test)["expiry_fingerprint"] + if before != after: + raise RuntimeError("Session expiry changed during restart") + time.sleep(2.2) + c.call("GET", "/api/v1/users/me", 401, actor=actor) + registration = actor["registration"] + import base64 + basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode() + c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session") + say("PASS: TTL=2, expired token rejected after API restart; fresh login works") + result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200, + "after_expiry_and_restart": 401, "new_login": 201} + evidence("expiry.json", result) + return result + finally: + compose(test, "stop", "--timeout", "10", "lb", "api1", "api2", "s3", "s3-storage", "db") + + +def dependency_check(args): + c = client(args) + actor, bob, eve, chat = c.bootstrap() + attachment, original = c.attachment(actor, chat) + content_path = "/api/v1/attachments/" + attachment["id"] + "/content" + result = {} + for service in ("db", "s3"): + try: + compose(args, "stop", "--timeout", "10", service) + c.call("GET", "/health/live", schema="Health") + outage_response(args, "/health/ready") + outage_response(args, content_path, actor["token"]) + finally: + compose(args, "start", service) + wait_ready(args) + c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + if c.call("GET", content_path, actor=bob)["bytes"] != original: + raise RuntimeError("Original changed after dependency recovery") + result[service] = {"live": 200, "ready": 503, "download": 503, "recovered": 200} + say("PASS: " + service + " outage returns bounded errors; session and file work after recovery") + return result + + +def private_s3_check(args): + code = ( + "import os,json,boto3; from botocore import UNSIGNED; " + "from botocore.config import Config; from botocore.exceptions import ClientError\n" + "c=boto3.client('s3',endpoint_url=os.environ['S3_ENDPOINT']," + "config=Config(signature_version=UNSIGNED,connect_timeout=2,read_timeout=5,proxies={}))\n" + "try:\n r=c.get_object(Bucket=os.environ['S3_BUCKET'],Key='system/readiness'); r['Body'].close(); status=200\n" + "except ClientError as e:\n status=e.response['ResponseMetadata']['HTTPStatusCode']\n" + "finally:\n c.close()\n" + "print(json.dumps({'anonymous_get':status}))" + ) + result = json.loads(compose(args, "exec", "-T", "api1", "python", "-c", code, quiet=True)) + if result["anonymous_get"] != 403: + raise RuntimeError("S3 bucket is publicly readable") + evidence("private-s3.json", result) + say("PASS: unsigned direct S3 GET denied") + return result + + +def verify(args): + summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(), + "source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(), + "worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()), + "project": args.project, "status": "FAIL"} + try: + setup(args) + contract_tests(args) + build(args) + start(args) + api_tests(args) + summary["private_s3"] = private_s3_check(args) + summary["persistence"] = restart_check(args) + summary["failover"] = failover_check(args) + summary["attachments_failover"] = attachment_failover_check(args) + summary["expiry"] = ttl_check(args) + summary["dependency"] = dependency_check(args) + run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)]) + summary["containers"] = states(args) + summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip() + summary["resources"] = json.loads(run( + args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True)) + summary["status"] = "PASS" + say("PASS: full lab4 verification") + finally: + evidence("verification.txt", "\n".join(LOG)) + evidence("summary.json", summary) + + +def instance_map(args): + current = states(args) + result = {} + for service in ("api1", "api2"): + item = current[service] + if not item["running"]: + continue + code = ( + "import urllib.request; " + "o=urllib.request.build_opener(urllib.request.ProxyHandler({})); " + "r=o.open('http://127.0.0.1:8080/health/live',timeout=1); " + "print(r.headers['X-Instance-Id'])" + ) + alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip() + import re + if not re.fullmatch(r"[0-9a-f]{32}", alias): + raise RuntimeError("Invalid API instance alias") + result[service] = {"instance_id": alias, "container_id": item["id"]} + return result + + +def wait_replicas(args): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + mapping = instance_map(args) + if len(mapping) == 2: + c = client(args) + seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"] + for _ in range(8)} + if seen == {item["instance_id"] for item in mapping.values()}: + return mapping + except (RuntimeError, OSError, AssertionError): + pass + time.sleep(0.3) + raise RuntimeError("Two healthy API replicas were not observed through LB") + + +def restore(args): + compose(args, "start", "api1", "api2") + wait_ready(args) + return wait_replicas(args) + + +def stop_one(args): + target = os.environ.get("TARGET_INSTANCE_ID", "") + before = instance_map(args) + matches = [(name, item) for name, item in before.items() if item["instance_id"] == target] + if len(before) != 2 or len(matches) != 1: + raise RuntimeError("Target must match exactly one of two running API replicas") + service, item = matches[0] + started = time.monotonic() + run(args, ["docker", "stop", "--timeout", "3", item["container_id"]]) + after = states(args) + if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]: + raise RuntimeError("Unexpected API state after stop") + elapsed = time.monotonic() - started + if elapsed >= 10: + raise RuntimeError("Stopping the selected API exceeded 10 seconds") + evidence("stop-one.json", {"service": service, "target_instance": target, "before": before, + "after": after, "stop_seconds": round(elapsed, 3)}) + say("Stopped only " + service) + + +def failover_check(args): + wait_replicas(args) + hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + try: + output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover", + "--action-script", hook, "--base-url", base_url(args)]) + first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8")) + first["metrics"] = next(json.loads(line) for line in output.splitlines() + if line.startswith("{") and "first_recovery_seconds" in line) + if first["after"][first["service"]]["running"]: + raise RuntimeError("Public hook did not leave the target stopped") + finally: + restore(args) + other_service = "api2" if first["service"] == "api1" else "api1" + mapping = instance_map(args) + target = mapping[other_service]["instance_id"] + c = client(args) + c.timeout = 2 + alice, bob, eve, chat = c.bootstrap() + for _ in range(10): + replay_key = secrets.token_hex(16) + replay_text = "before-failure-" + replay_key + response = c.message(alice, chat, replay_text, key=replay_key) + if response["headers"]["X-Instance-Id"] == target: + original = response["json"] + break + else: + raise RuntimeError("Second API was not reached through LB") + env = environment(args) + env["TARGET_INSTANCE_ID"] = target + started = time.monotonic() + process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + from client import Failure + from resilience import history + confirmations, after_instances = [], [] + failures = 0 + recovered = None + key = secrets.token_hex(16) + text = "during-failure-" + key + try: + while time.monotonic() - started < 15: + code = process.poll() + if code not in (None, 0): + raise RuntimeError("Second stop hook failed") + if code is None and time.monotonic() - started >= 10: + raise RuntimeError("Second stop hook exceeded 10 seconds") + try: + response = c.message(alice, chat, text, key=key) + confirmations.append(response["json"]) + if code == 0: + if recovered is None: + recovered = time.monotonic() - started + after_instances.append(response["headers"]["X-Instance-Id"]) + key = secrets.token_hex(16) + text = "during-failure-" + key + except Failure: + failures += 1 + time.sleep(0.1) + if process.poll() != 0 or recovered is None or recovered > 10: + raise RuntimeError("Second API failure did not recover within 10 seconds") + if len(after_instances) < 5 or target in after_instances[-5:]: + raise RuntimeError("Stopped API still serves requests") + replayed = c.message(alice, chat, replay_text, key=replay_key) + if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target: + raise RuntimeError("Message replay through the survivor changed the result") + messages = history(c, bob, chat) + ids = [item["id"] for item in messages] + texts = [item["text"] for item in messages] + if len(ids) != len(set(ids)) or len(texts) != len(set(texts)): + raise RuntimeError("Duplicates appeared during failover") + if any(item not in messages for item in [original, *confirmations]): + raise RuntimeError("A confirmed message was lost") + second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8")) + if second["service"] != other_service: + raise RuntimeError("Wrong second API was stopped") + second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures, + "first_recovery_seconds": round(recovered, 3), + "replay_on_survivor": "PASS"} + result = {"public": first, "second_replica": second, "status": "PASS"} + evidence("failover.json", result) + say("PASS: both physical API replicas stopped separately; no lost messages or duplicates") + return result + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + restore(args) + + +def outage_response(args, path, token=None): + from urllib.error import HTTPError + from urllib.request import Request + from client import shape + headers = {"Authorization": "Bearer " + token} if token else {} + try: + response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10) + except HTTPError as error: + response = error + with response: + payload = json.load(response) + if response.status != 503 or not response.headers.get("X-Request-Id"): + raise RuntimeError("Unexpected dependency failure response") + shape(payload, "Error") + if payload["error"]["request_id"] != response.headers["X-Request-Id"]: + raise RuntimeError("LB error request ID mismatch") + + + + +def attachment_failover_check(args): + import hashlib + results = [] + raw = (ROOT / "tests/fixtures/sample.png").read_bytes() + try: + for service in ("api1", "api2"): + mapping = wait_replicas(args) + c = client(args) + alice, bob, eve, chat = c.bootstrap() + target = mapping[service]["instance_id"] + for _ in range(8): + uploaded = c.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png", 201, + actor=alice, raw=raw, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + if uploaded["headers"]["X-Instance-Id"] == target: + break + else: + raise RuntimeError("Selected upload replica was not observed") + attachment = uploaded["json"] + linked = c.message(alice, chat, "", attachments=[attachment["id"]])["json"] + hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + previous = os.environ.get("TARGET_INSTANCE_ID") + os.environ["TARGET_INSTANCE_ID"] = target + started = time.monotonic() + try: + run(args, [str(hook)]) + finally: + if previous is None: + os.environ.pop("TARGET_INSTANCE_ID", None) + else: + os.environ["TARGET_INSTANCE_ID"] = previous + from client import Client + reads = Client(base_url(args), timeout=2) + errors = 0 + while True: + try: + downloaded = reads.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + history_response = reads.call( + "GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage", + ) + break + except (AssertionError, OSError) as error: + if isinstance(error, AssertionError) and not any( + text in str(error) for text in ("received 502", "received 503", "connection/TLS failure") + ): + raise + errors += 1 + if time.monotonic() - started >= 10: + raise RuntimeError("File and history did not recover within 10 seconds") from None + time.sleep(0.1) + recovered = time.monotonic() - started + if downloaded["bytes"] != raw or hashlib.sha256(raw).hexdigest() != attachment["sha256"]: + raise RuntimeError("Original file changed after API failure") + if downloaded["headers"]["X-Instance-Id"] == target or recovered > 10: + raise RuntimeError("Upload replica was not replaced") + history = history_response["json"]["items"] + if history_response["headers"]["X-Instance-Id"] == target: + raise RuntimeError("Stopped API returned history") + if history != [linked]: + raise RuntimeError("Attached message changed during failover") + results.append({"stopped_service": service, "upload_instance": target, + "download_instance": downloaded["headers"]["X-Instance-Id"], + "recovery_seconds": round(recovered, 3), "transitional_errors": errors, + "same_original": True, "message_preserved": True}) + restore(args) + result = {"status": "PASS", "cases": results} + evidence("attachment-failover.json", result) + say("PASS: stopped each uploader; other API downloaded the same original") + return result + finally: + restore(args) + + +def main(): + saved = settings() + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one", + "restart-check", "failover-check", "restore", "stop", "verify"]) + parser.add_argument("--project", default=os.environ.get("LAB4_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "kuznetsov-lab4"))) + parser.add_argument("--port", type=int, default=int(os.environ.get("LAB4_PORT", saved.get("HTTP_PORT", "18104")))) + parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB4_TTL", saved.get("SESSION_TTL_SECONDS", "3600")))) + parser.add_argument("--test-port", type=int, default=18114) + args = parser.parse_args() + if args.project != "kuznetsov-lab4" and not args.project.startswith("kuznetsov-lab4-"): + parser.error("Use a dedicated Kuznetsov lab4 project") + if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65534 or not 1 <= args.ttl <= 86400: + parser.error("Invalid port or session TTL") + if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check"): + setup(args) + actions = {"setup": lambda item: None, "build": build, "start": start, + "test": lambda item: (contract_tests(item), api_tests(item)), + "restart": restart, "restart-check": restart_check, "verify": verify, + "stop-one": stop_one, "restore": restore, "failover-check": failover_check, + "stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "s3", "s3-storage", "db")} + actions[args.action](args) + + +if __name__ == "__main__": + try: + main() + except (RuntimeError, OSError, ValueError) as error: + print("FAIL: " + str(error), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..cecafa7f --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +python "%~dp0lab.py" restart +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..1e3d2830 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +exec python3 "$TASK_DIR/lab.py" restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..b3d8d832 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +python "%~dp0lab.py" stop-one +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..d26d99ad --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}" +TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +exec python3 "$TASK_DIR/lab.py" stop-one diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..c5f2db93 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,21 @@ +import logging +import os + +import uvicorn + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + logging.getLogger("messenger").info("environment=%s", os.environ.get("APP_ENV", "lab")) + port = int(os.environ.get("APP_PORT", "8080")) + if not 1 <= port <= 65535: + raise SystemExit("APP_PORT должен быть от 1 до 65535") + uvicorn.run( + "src.app:create_app", + factory=True, + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=port, + workers=1, + access_log=False, + timeout_graceful_shutdown=2, + ) diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..6c9b8c91 --- /dev/null +++ b/src/app.py @@ -0,0 +1,198 @@ +from contextlib import asynccontextmanager +import logging +import re +from uuid import UUID, uuid4 +from urllib.parse import quote +from typing import Literal + +from fastapi import FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException + +from .attachments import Attachments +from .objects import Objects +from .errors import ApiError, invalid +from .models import CreateChat, CreateMessage, CreateUser +from .security import basic_credentials, bearer_token +from .storage import PostgresStore + + +logger = logging.getLogger("messenger") +SESSION_PATH = "/api/v1/auth/sessions" + + +def uuid_value(value): + try: + return UUID(value) + except ValueError: + raise invalid("Неверный UUID") from None + + +def page_args(request): + raw = request.query_params.get("limit", "50") + normalized = raw.lstrip("0") or "0" + if re.fullmatch(r"[0-9]+", raw) is None or len(normalized) > 3 or not 1 <= int(normalized) <= 100: + raise invalid("Limit должен быть от 1 до 100") + return int(normalized), request.query_params.get("cursor") + + +def create_app(): + store = PostgresStore() + objects = Objects() + attachments = Attachments(store, objects) + instance_id = uuid4().hex + + @asynccontextmanager + async def lifespan(app): + await run_in_threadpool(store.open) + try: + yield + finally: + await run_in_threadpool(store.close) + objects.close() + + app = FastAPI(redirect_slashes=False, docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan) + app.state.store = store + + def error_response(request, error): + request_id = request.state.request_id + headers = dict(error.headers) + if error.status == 401: + headers["WWW-Authenticate"] = ( + 'Basic realm="messenger", charset="UTF-8"' if request.url.path == SESSION_PATH + else 'Bearer realm="messenger"' + ) + return JSONResponse( + status_code=error.status, headers=headers, + content={"error": {"code": error.code, "message": error.message, "request_id": request_id}}, + ) + + @app.middleware("http") + async def request_context(request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + registration = path == "/api/v1/users" and request.method == "POST" + login = path == SESSION_PATH and request.method == "POST" + if login: + request.state.credentials = basic_credentials(request.headers.get("Authorization")) + async for chunk in request.stream(): + if chunk: + raise invalid("Вход не принимает тело запроса") + elif path.startswith("/api/v1/") and not registration: + token = bearer_token(request.headers.get("Authorization")) + request.state.user_id = await run_in_threadpool(store.authenticate, token) + request.state.token = token + upload = re.fullmatch(r"/api/v1/chats/[^/]+/attachments", path) is not None + if request.method == "POST" and path.startswith("/api/v1/") and not login and not upload: + media_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if media_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + raise invalid("JSON должен быть в UTF-8") from None + response = await call_next(request) + except ApiError as error: + response = error_response(request, error) + except Exception as error: + logger.error("request failed: %s", type(error).__name__) + response = error_response(request, ApiError(500, "internal_error", "Внутренняя ошибка")) + response.headers["X-Instance-Id"] = instance_id + response.headers["X-Request-Id"] = request.state.request_id + logger.info("%s %s %s request_id=%s", request.method, request.url.path, response.status_code, request.state.request_id) + return response + + @app.exception_handler(ApiError) + async def api_error(request, error): + return error_response(request, error) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, error): + return error_response(request, invalid("Неверные поля запроса")) + + @app.exception_handler(HTTPException) + async def http_error(request, error): + code = "not_found" if error.status_code == 404 else "invalid_request" + return error_response(request, ApiError(error.status_code, code, "Неверный запрос")) + + @app.get("/health/live") + def live(): + return {"status": "ok"} + + @app.get("/health/ready") + def ready(): + result = store.ready() + objects.ready() + return result + + @app.post("/api/v1/users", status_code=201) + def create_user(body: CreateUser): + return store.create_user(body) + + @app.post(SESSION_PATH, status_code=201) + def login(request: Request): + return store.create_session(*request.state.credentials) + + @app.delete(SESSION_PATH + "/current", status_code=204) + def logout(request: Request): + store.logout(request.state.token) + return Response(status_code=204) + + @app.get("/api/v1/users/me") + def me(request: Request): + return store.user(request.state.user_id) + + @app.post("/api/v1/chats", status_code=201) + def create_chat(body: CreateChat, request: Request): + return store.create_chat(request.state.user_id, body) + + @app.get("/api/v1/chats") + def list_chats(request: Request): + return store.list_chats(request.state.user_id, *page_args(request)) + + @app.get("/api/v1/chats/{chat_id}") + def chat(chat_id: str, request: Request): + return store.chat(request.state.user_id, uuid_value(chat_id)) + + @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) + def create_message(chat_id: str, body: CreateMessage, request: Request): + key = request.headers.get("Idempotency-Key", "") + if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key) is None: + raise invalid("Неверный Idempotency-Key") + return store.create_message(request.state.user_id, uuid_value(chat_id), body, key) + + @app.get("/api/v1/chats/{chat_id}/messages") + def messages(chat_id: str, request: Request): + return store.list_messages(request.state.user_id, uuid_value(chat_id), *page_args(request)) + + @app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) + async def upload(chat_id: str, filename: str, request: Request): + return await attachments.receive(request, uuid_value(chat_id), filename) + + @app.get("/api/v1/attachments/{attachment_id}") + def attachment(attachment_id: str, request: Request): + return attachments.get(request.state.user_id, uuid_value(attachment_id)) + + @app.get("/api/v1/attachments/{attachment_id}/content") + def download(attachment_id: str, request: Request, variant: Literal["original"] = "original"): + row = attachments.get(request.state.user_id, uuid_value(attachment_id), internal=True) + body = objects.content(row["object_key"]) + + def chunks(): + try: + while data := body.read(65536): + yield data + finally: + body.close() + + return StreamingResponse( + chunks(), media_type=row["content_type"], background=BackgroundTask(body.close), + headers={"Content-Length": str(row["size_bytes"]), "X-Content-Type-Options": "nosniff", + "Content-Disposition": "attachment; filename*=UTF-8''" + quote(row["filename"], safe="")}, + ) + + return app diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..19607a57 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,125 @@ +import asyncio +import hashlib +import tempfile +import unicodedata +import warnings +from uuid import uuid4 + +import anyio +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from .errors import ApiError, invalid, not_found +from .storage import serialize + + +MAX_BYTES = 10485760 +MAX_PIXELS = 40000000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +FIELDS = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at" + + +def public(row): + return {**serialize(row), "status": "ready", "variants": [], "error_code": None} + + +def filename_value(value): + if not 1 <= len(value) <= 255 or not value.strip(): + raise invalid("Неверное имя файла") + if any(c in "/\\" or unicodedata.category(c) == "Cc" for c in value): + raise invalid("Неверное имя файла") + try: + value.encode("utf-8") + except UnicodeError: + raise invalid("Неверное имя файла") from None + return value + + +def validate_image(source, media_type): + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + source.seek(0) + with Image.open(source, formats=["PNG", "JPEG"]) as image: + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if actual != media_type: + raise ApiError(415, "unsupported_media_type", "Формат не совпадает с Content-Type") + image.verify() + source.seek(0) + with Image.open(source, formats=["PNG", "JPEG"]) as image: + image.load() + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from None + + +class Attachments: + def __init__(self, store, objects): + self.store = store + self.objects = objects + self.slots = asyncio.Semaphore(2) + + async def receive(self, request, chat_id, filename): + user_id = request.state.user_id + await run_in_threadpool(self.store.chat, user_id, chat_id) + filename = filename_value(filename) + media_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if media_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + declared = request.headers.get("Content-Length") + if declared is not None: + if not declared.isascii() or not declared.isdecimal() or len(declared) > 18: + raise invalid("Неверный Content-Length") + if int(declared) > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + try: + await asyncio.wait_for(self.slots.acquire(), timeout=15) + except TimeoutError: + raise ApiError(429, "rate_limited", "Загрузки заняты", {"Retry-After": "1"}) from None + try: + with tempfile.TemporaryFile(mode="w+b") as source: + size, checksum = 0, hashlib.sha256() + try: + with anyio.fail_after(15): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") + checksum.update(chunk) + await run_in_threadpool(source.write, chunk) + except (TimeoutError, ClientDisconnect): + raise invalid("Загрузка прервана или слишком медленная") from None + await run_in_threadpool(validate_image, source, media_type) + return await run_in_threadpool( + self.publish, user_id, chat_id, filename, media_type, size, checksum.hexdigest(), source, + ) + finally: + self.slots.release() + + def publish(self, user_id, chat_id, filename, media_type, size, checksum, source): + attachment_id = uuid4() + key = "originals/" + attachment_id.hex + self.objects.put(key, source, size, media_type) + with self.store.connection() as connection: + self.store.require_chat(connection, user_id, chat_id) + row = connection.execute( + "INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING " + FIELDS, + (attachment_id, chat_id, user_id, filename, media_type, size, checksum, key), + ).fetchone() + return public(row) + + def get(self, user_id, attachment_id, internal=False): + with self.store.connection() as connection: + row = connection.execute( + "SELECT " + FIELDS + (",object_key" if internal else "") + " FROM attachments a " + "WHERE a.id=%s AND EXISTS (SELECT 1 FROM chat_members m " + "WHERE m.chat_id=a.chat_id AND m.user_id=%s)", (attachment_id, user_id), + ).fetchone() + if not row: + raise not_found() + return row if internal else public(row) diff --git a/src/bootstrap.py b/src/bootstrap.py new file mode 100644 index 00000000..861da2e8 --- /dev/null +++ b/src/bootstrap.py @@ -0,0 +1,53 @@ +import logging +import time +from io import BytesIO + +from botocore.exceptions import BotoCoreError, ClientError + +from .objects import Objects +from .errors import ApiError + + +def main(): + admin, app = Objects(admin=True), Objects() + try: + deadline = time.monotonic() + 90 + while True: + try: + admin.client.list_buckets() + break + except (ClientError, BotoCoreError): + if time.monotonic() >= deadline: + raise RuntimeError("S3 startup timeout") from None + time.sleep(1) + try: + admin.client.head_bucket(Bucket=admin.bucket) + except ClientError as error: + if error.response["ResponseMetadata"]["HTTPStatusCode"] != 404: + raise RuntimeError("Bucket is not accessible") from None + admin.client.create_bucket(Bucket=admin.bucket) + acl = admin.client.get_bucket_acl(Bucket=admin.bucket) + if any("URI" in grant["Grantee"] for grant in acl["Grants"]): + raise RuntimeError("Bucket must be private") + while True: + try: + probe = b"messenger-ready" + app.put("system/readiness", BytesIO(probe), len(probe), "application/octet-stream") + app.ready() + break + except ApiError: + if time.monotonic() >= deadline: + raise RuntimeError("S3 object path is not ready") from None + time.sleep(0.5) + print("PASS: private bucket ready; application credentials work", flush=True) + finally: + admin.close() + app.close() + + +if __name__ == "__main__": + try: + main() + except Exception as error: + logging.error("S3 bootstrap failed: %s", type(error).__name__) + raise SystemExit(1) diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..fb2639da --- /dev/null +++ b/src/config.py @@ -0,0 +1,23 @@ +import os +from pathlib import Path + + +def secret(name): + value = Path(os.environ[name]).read_text(encoding="utf-8").strip() + if not value: + raise ValueError(f"Empty secret file: {name}") + return value + + +def session_ttl(): + value = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if not 1 <= value <= 86400: + raise ValueError("SESSION_TTL_SECONDS must be between 1 and 86400") + return value + + +def idempotency_ttl(): + value = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + if value < 86400: + raise ValueError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") + return value diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..b1213300 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,15 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(message="Неверный запрос"): + return ApiError(400, "invalid_request", message) + + +def not_found(): + return ApiError(404, "not_found", "Объект не найден") diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..5e13605f --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,50 @@ +import hashlib +import logging +from pathlib import Path + +import psycopg + +from .config import secret + + +def migrate(): + files = sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")) + if not files: + raise RuntimeError("No migrations found") + with psycopg.connect(secret("DATABASE_URL_FILE"), connect_timeout=5) as connection: + with connection.transaction(): + connection.execute("SELECT pg_advisory_xact_lock(230402)") + connection.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + checksum char(64) NOT NULL, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """) + for file in files: + content = file.read_text(encoding="utf-8") + checksum = hashlib.sha256(content.encode("utf-8")).hexdigest() + with connection.transaction(): + connection.execute("SELECT pg_advisory_xact_lock(230402)") + old = connection.execute( + "SELECT checksum FROM schema_migrations WHERE version = %s", (file.name,) + ).fetchone() + if old: + if old[0] != checksum: + raise RuntimeError("Applied migration was changed") + print("Already applied: " + file.name, flush=True) + continue + connection.execute(content) + connection.execute( + "INSERT INTO schema_migrations(version, checksum) VALUES (%s, %s)", + (file.name, checksum), + ) + print("Applied: " + file.name, flush=True) + + +if __name__ == "__main__": + try: + migrate() + except Exception as error: + logging.error("Migration failed: %s", type(error).__name__) + raise SystemExit(1) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..38469018 --- /dev/null +++ b/src/models.py @@ -0,0 +1,61 @@ +import re +from typing import Annotated +from uuid import UUID + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, StringConstraints, field_validator, model_validator + + +def nonblank(value: str) -> str: + if not value.strip() or "\x00" in value: + raise ValueError("Пустая строка или NUL") + value.encode("utf-8") + return value + + +Name = Annotated[str, StringConstraints(min_length=1, max_length=100), AfterValidator(nonblank)] +Text = Annotated[str, StringConstraints(min_length=1, max_length=4000), AfterValidator(nonblank)] +Password = Annotated[str, StringConstraints(min_length=12, max_length=128), AfterValidator(nonblank)] +UserId = Annotated[UUID, Field(strict=False)] + + +class RequestBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(RequestBody): + username: str + display_name: Name + password: Password = Field(repr=False) + + @field_validator("username") + @classmethod + def check_username(cls, value): + if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None: + raise ValueError("Неверный username") + return value + + +class CreateChat(RequestBody): + title: Name + member_ids: list[UserId] = Field(min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def unique_members(cls, value): + if len(set(value)) != len(value): + raise ValueError("Повтор участника") + return value + + +class CreateMessage(RequestBody): + text: str = Field(max_length=4000) + attachment_ids: list[UserId] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def check_message(self): + if "\x00" in self.text or (not self.text.strip() and not self.attachment_ids): + raise ValueError("Нужен текст или вложение") + self.text.encode("utf-8") + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Повтор вложения") + return self diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 00000000..4d13fe5a --- /dev/null +++ b/src/objects.py @@ -0,0 +1,54 @@ +import os + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import secret +from .errors import ApiError + + +class Objects: + def __init__(self, admin=False): + prefix = "S3_ADMIN_" if admin else "S3_" + self.bucket = os.environ.get("S3_BUCKET", "kuznetsov-lab4-attachments") + self.client = boto3.client( + "s3", endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=secret(prefix + "ACCESS_KEY_ID_FILE"), + aws_secret_access_key=secret(prefix + "SECRET_ACCESS_KEY_FILE"), + config=Config( + signature_version="s3v4", connect_timeout=2, read_timeout=5, + max_pool_connections=8, retries={"total_max_attempts": 1}, proxies={}, + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + def call(self, operation, **values): + try: + return getattr(self.client, operation)(Bucket=self.bucket, **values) + except (BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "Файловое хранилище недоступно") from None + + def ready(self): + self.call("head_bucket") + body = self.content("system/readiness") + try: + if body.read(32) != b"messenger-ready": + raise ApiError(503, "unavailable", "Файловое хранилище не готово") + except BotoCoreError: + raise ApiError(503, "unavailable", "Файловое хранилище недоступно") from None + finally: + body.close() + + def put(self, key, source, size, content_type): + source.seek(0) + self.call("put_object", Key=key, Body=source, ContentLength=size, ContentType=content_type) + + def content(self, key): + return self.call("get_object", Key=key)["Body"] + + def close(self): + self.client.close() diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..5d7ca2a9 --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,44 @@ +import base64 +import hashlib +import hmac +import json +from datetime import datetime, timezone +from uuid import UUID + +from .errors import invalid + + +class Pagination: + def __init__(self, secret): + self.secret = secret + + def encode(self, scope, item): + data = json.dumps([1, scope, item["created_at"], item["id"]], separators=(",", ":")) + payload = base64.urlsafe_b64encode(data.encode()).rstrip(b"=").decode() + signature = hmac.new(self.secret, payload.encode(), hashlib.sha256).hexdigest() + return payload + "." + signature + + def decode(self, scope, cursor): + try: + if not isinstance(cursor, str) or not cursor or len(cursor) > 2048: + raise ValueError() + payload, signature = cursor.split(".") + expected = hmac.new(self.secret, payload.encode("ascii"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected): + raise ValueError() + data = json.loads(base64.b64decode(payload + "=" * (-len(payload) % 4), altchars=b"-_", validate=True)) + if not isinstance(data, list) or len(data) != 4 or data[:2] != [1, scope]: + raise ValueError() + if not isinstance(data[2], str) or not data[2].endswith("Z"): + raise ValueError() + created_at = datetime.fromisoformat(data[2].replace("Z", "+00:00")) + if created_at.utcoffset() != timezone.utc.utcoffset(created_at): + raise ValueError() + return created_at, UUID(data[3]) + except (ValueError, TypeError, UnicodeError, KeyError, OverflowError): + raise invalid("Неверный cursor") from None + + def page(self, rows, scope, limit): + items = rows[:limit] + cursor = self.encode(scope, items[-1]) if len(rows) > limit else None + return {"items": items, "next_cursor": cursor} diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..294dc21a --- /dev/null +++ b/src/security.py @@ -0,0 +1,63 @@ +import base64 +import binascii +import hashlib +import re +import secrets +from threading import BoundedSemaphore + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from argon2.low_level import Type + +from .errors import ApiError + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, type=Type.ID) +HASH_SLOTS = BoundedSemaphore(4) +DUMMY_HASH = HASHER.hash(secrets.token_urlsafe(32)) + + +def unauthorized(): + return ApiError(401, "unauthorized", "Неверные данные авторизации") + + +def hash_password(password): + with HASH_SLOTS: + return HASHER.hash(password) + + +def verify_password(password_hash, password): + try: + with HASH_SLOTS: + return HASHER.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, ValueError): + return False + + +def basic_credentials(header): + try: + scheme, encoded = header.split(" ", 1) + if scheme.lower() != "basic" or len(encoded) > 2048: + raise ValueError() + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, separator, password = decoded.partition(":") + if not separator or re.fullmatch(r"[a-z0-9_]{3,32}", username) is None: + raise ValueError() + if not 12 <= len(password) <= 128 or not password.strip() or "\x00" in password: + raise ValueError() + return username, password + except (ValueError, UnicodeError, AttributeError, binascii.Error): + raise unauthorized() from None + + +def bearer_token(header): + try: + scheme, token = header.split(" ", 1) + if scheme.lower() != "bearer" or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token): + raise ValueError() + return token + except (ValueError, TypeError, AttributeError): + raise unauthorized() from None + + +def token_hash(token): + return hashlib.sha256(token.encode("ascii")).hexdigest() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..f23729c1 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,240 @@ +from contextlib import contextmanager +from datetime import datetime +import logging +import hashlib +import json +import secrets +from uuid import UUID, uuid4 + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests + +from .config import idempotency_ttl, secret, session_ttl +from .errors import ApiError, invalid, not_found +from .pagination import Pagination +from .security import hash_password, token_hash, unauthorized, verify_password + + +USER_FIELDS = "id, username, display_name, created_at" +CHAT_FIELDS = """ + c.id, c.title, c.created_at, + ARRAY(SELECT cm.user_id FROM chat_members cm WHERE cm.chat_id=c.id ORDER BY cm.user_id) AS member_ids +""" + + +def serialize(value): + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + if isinstance(value, dict): + return {key: serialize(item) for key, item in value.items()} + if isinstance(value, list): + return [serialize(item) for item in value] + return value + + +class PostgresStore: + def __init__(self): + logging.getLogger("psycopg.pool").setLevel(logging.CRITICAL) + self.pool = ConnectionPool( + secret("DATABASE_URL_FILE"), min_size=1, max_size=10, timeout=3, max_waiting=40, + open=False, check=ConnectionPool.check_connection, + kwargs={"row_factory": dict_row, "connect_timeout": 3, + "options": "-c timezone=UTC -c statement_timeout=5000 -c lock_timeout=2000"}, + ) + self.pagination = Pagination(secret("CURSOR_SECRET_FILE").encode("ascii")) + self.ttl = session_ttl() + self.idempotency_ttl = idempotency_ttl() + + def open(self): + self.pool.open() + + def close(self): + self.pool.close() + + @contextmanager + def connection(self): + try: + with self.pool.connection() as connection: + yield connection + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests, + psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable): + raise ApiError(503, "unavailable", "Хранилище временно недоступно") from None + + def ready(self): + try: + with self.connection() as connection: + row = connection.execute( + "SELECT version FROM schema_migrations WHERE version=%s", ("003_attachments.sql",) + ).fetchone() + if not row: + raise ApiError(503, "unavailable", "Схема ещё не готова") + except psycopg.errors.UndefinedTable: + raise ApiError(503, "unavailable", "Схема ещё не готова") from None + return {"status": "ok"} + + def create_user(self, body): + password_hash = hash_password(body.password) + try: + with self.connection() as connection: + row = connection.execute( + "INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING " + + USER_FIELDS, + (uuid4(), body.username, body.display_name, password_hash), + ).fetchone() + return serialize(row) + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username уже занят") from None + + def create_session(self, username, password): + with self.connection() as connection: + row = connection.execute( + "SELECT " + USER_FIELDS + ", password_hash FROM users WHERE username=%s", (username,) + ).fetchone() + if not verify_password(row["password_hash"] if row else None, password) or row is None: + raise unauthorized() + token = secrets.token_urlsafe(32) + with self.connection() as connection: + session = connection.execute(""" + INSERT INTO sessions(token_hash,user_id,created_at,expires_at) + VALUES (%s,%s,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP + %s * INTERVAL '1 second') + RETURNING expires_at + """, (token_hash(token), row["id"], self.ttl)).fetchone() + user = {key: value for key, value in row.items() if key != "password_hash"} + return {"token": token, "token_type": "Bearer", "expires_at": serialize(session["expires_at"]), + "user": serialize(user)} + + def authenticate(self, token): + with self.connection() as connection: + row = connection.execute(""" + SELECT user_id FROM sessions + WHERE token_hash=%s AND expires_at > CURRENT_TIMESTAMP + """, (token_hash(token),)).fetchone() + if row is None: + raise unauthorized() + return row["user_id"] + + def logout(self, token): + with self.connection() as connection: + result = connection.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),)) + if result.rowcount != 1: + raise unauthorized() + + def user(self, user_id): + with self.connection() as connection: + row = connection.execute("SELECT " + USER_FIELDS + " FROM users WHERE id=%s", (user_id,)).fetchone() + if not row: + raise unauthorized() + return serialize(row) + + def create_chat(self, user_id, body): + members = body.member_ids + if user_id not in members: + raise invalid("Неверный список участников") + with self.connection() as connection: + found = connection.execute("SELECT id FROM users WHERE id=ANY(%s)", (members,)).fetchall() + if len(found) != len(members): + raise invalid("Неверный список участников") + chat_id = uuid4() + # чат и участники коммитятся вместе + connection.execute("INSERT INTO chats(id,title) VALUES (%s,%s)", (chat_id, body.title)) + with connection.cursor() as cursor: + cursor.executemany( + "INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", + [(chat_id, member) for member in members], + ) + return serialize(self.require_chat(connection, user_id, chat_id)) + + def require_chat(self, connection, user_id, chat_id): + row = connection.execute( + "SELECT " + CHAT_FIELDS + """ FROM chats c WHERE c.id=%s + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s) + """, (chat_id, user_id) + ).fetchone() + if row is None: + raise not_found() + return row + + def chat(self, user_id, chat_id): + with self.connection() as connection: + return serialize(self.require_chat(connection, user_id, chat_id)) + + def list_chats(self, user_id, limit, cursor): + scope = "chats:" + str(user_id) + after = self.pagination.decode(scope, cursor) if cursor is not None else None + condition = " AND (c.created_at,c.id) > (%s,%s)" if after else "" + with self.connection() as connection: + rows = connection.execute( + "SELECT " + CHAT_FIELDS + """ FROM chats c + WHERE EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)""" + + condition + " ORDER BY c.created_at,c.id LIMIT %s", + [user_id, *(after or ()), limit + 1], + ).fetchall() + return self.pagination.page(serialize(rows), scope, limit) + + def create_message(self, user_id, chat_id, body, key): + path = f"/api/v1/chats/{chat_id}/messages" + scope = (user_id, "POST", path, key) + canonical = json.dumps(body.model_dump(mode="json"), sort_keys=True, ensure_ascii=False, separators=(",", ":")) + request_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + lock_data = json.dumps([str(user_id), "POST", path, key], separators=(",", ":")).encode() + lock_id = int.from_bytes(hashlib.sha256(lock_data).digest()[:8], "big", signed=True) + with self.connection() as connection: + self.require_chat(connection, user_id, chat_id) + # один ключ блокируется сразу для обеих реплик + connection.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + saved = connection.execute(""" + SELECT request_hash,response_body,expires_at > clock_timestamp() AS active + FROM idempotency_keys WHERE user_id=%s AND method=%s AND path=%s AND request_key=%s + """, scope).fetchone() + if saved and saved["active"]: + if saved["request_hash"] != request_hash: + raise ApiError(409, "conflict", "С этим ключом уже отправлен другой запрос") + return saved["response_body"] + if saved: + connection.execute(""" + DELETE FROM idempotency_keys WHERE user_id=%s AND method=%s AND path=%s AND request_key=%s + """, scope) + if body.attachment_ids: + found = connection.execute( + "SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s)", + (chat_id, body.attachment_ids), + ).fetchall() + if len(found) != len(body.attachment_ids): + raise not_found() + row = connection.execute(""" + INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) + RETURNING id,chat_id,sender_id,text,created_at + """, (uuid4(), chat_id, user_id, body.text)).fetchone() + for position, attachment_id in enumerate(body.attachment_ids): + connection.execute( + "INSERT INTO message_attachments(message_id,attachment_id,chat_id,position) VALUES (%s,%s,%s,%s)", + (row["id"], attachment_id, chat_id, position), + ) + row["attachment_ids"] = body.attachment_ids + message = serialize(row) + connection.execute(""" + INSERT INTO idempotency_keys + (user_id,method,path,request_key,request_hash,message_id,response_status,response_body,expires_at) + VALUES (%s,%s,%s,%s,%s,%s,201,%s,clock_timestamp() + %s * INTERVAL '1 second') + """, (*scope, request_hash, row["id"], Jsonb(message), self.idempotency_ttl)) + return message + + def list_messages(self, user_id, chat_id, limit, cursor): + scope = "messages:" + str(chat_id) + after = self.pagination.decode(scope, cursor) if cursor is not None else None + condition = " AND (created_at,id) > (%s,%s)" if after else "" + with self.connection() as connection: + self.require_chat(connection, user_id, chat_id) + rows = connection.execute( + """SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id=m.id ORDER BY ma.position) AS attachment_ids + FROM messages m WHERE chat_id=%s""" + + condition + " ORDER BY created_at,id LIMIT %s", + [chat_id, *(after or ()), limit + 1], + ).fetchall() + return self.pagination.page(serialize(rows), scope, limit) diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..7b2054f8 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,56 @@ +import io +import os +from pathlib import Path +import struct +import sys +from urllib.parse import urlsplit +import zlib + +from PIL import Image + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tests")) +from client import Client as PublicClient +from scripts import lab + + +def project_name(): + name = os.environ.get("LAB4_PROJECT", lab.settings().get("COMPOSE_PROJECT_NAME", "kuznetsov-lab4")) + if name != "kuznetsov-lab4" and not name.startswith("kuznetsov-lab4-"): + raise RuntimeError("Tests only operate on Kuznetsov lab4 projects") + return name + + +class Client(PublicClient): + def __init__(self, base_url=None, ca_file=None, timeout=10): + project_name() + port = int(os.environ.get("LAB4_PORT", lab.settings().get("HTTP_PORT", "18104"))) + target = base_url or os.environ.get("BASE_URL") or f"http://127.0.0.1:{port}" + parsed = urlsplit(target) + if parsed.scheme != "http" or parsed.hostname not in ("127.0.0.1", "localhost") or parsed.port != port: + raise RuntimeError("BASE_URL must match this lab's local port") + super().__init__(target, ca_file=ca_file, timeout=timeout) + + +def image_bytes(kind="PNG", size=(32, 24), mode="RGB"): + with Image.new(mode, size, (35, 120, 200, 255) if mode == "RGBA" else (35, 120, 200)) as image: + output = io.BytesIO() + image.save(output, format=kind) + return output.getvalue() + + +def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xffffffff) + + +def padded_png(size): + base = image_bytes() + padding = b"p" * (size - len(base) - 12) + return base[:-12] + chunk(b"raNd", padding) + base[-12:] + + +def oversized_pixels(): + header = struct.pack(">IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IEND", b"") diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..ad620810 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,182 @@ +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import sys +import unittest +from urllib.parse import urlencode +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from helpers import Client +from client import timestamp + + +class ApiTests(unittest.TestCase): + def setUp(self): + self.client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:18104")) + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = "/api/v1/chats/" + self.chat["id"] + + def error(self, method, path, status=400, **kwargs): + if method == "POST" and path.endswith("/messages"): + kwargs.setdefault("headers", {}).setdefault("Idempotency-Key", uuid4().hex) + response = self.client.call(method, path, status, **kwargs) + code = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict", 415: "unsupported_media_type"}[status] + self.assertEqual(response["json"]["error"]["code"], code) + + def get_page(self, path, actor=None, **query): + schema = "MessagePage" if path.endswith("/messages") else "ChatPage" + return self.client.call("GET", path + "?" + urlencode(query), actor=actor or self.alice, schema=schema)["json"] + + def new_chat(self): + return self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, + body={"title": "Ещё чат", "member_ids": [self.alice["id"], self.bob["id"]]}, schema="Chat")["json"] + + def test_user_fields(self): + name = "u_" + uuid4().hex[:20] + bodies = [ + {"username": "ab", "display_name": "Имя"}, + {"username": "ABC", "display_name": "Имя"}, + {"username": "valid\n", "display_name": "Имя"}, + {"username": 123, "display_name": "Имя"}, + {"username": name, "display_name": " \t\n"}, + {"username": name, "display_name": 123}, + {"username": name, "display_name": None}, + {"username": name, "display_name": "я" * 101}, + {"username": name, "display_name": "Имя", "password": "extra"}, + {"username": name}, + ] + for body in bodies: + body.setdefault("password", "T3st-password-123") + with self.subTest(fields=sorted(body)): + self.error("POST", "/api/v1/users", body=body) + user = self.client.call("POST", "/api/v1/users", 201, + body={"username": name, "display_name": "👋" * 100, "password": "T3st-password-123"}, schema="User")["json"] + self.assertEqual(user["display_name"], "👋" * 100) + self.error("POST", "/api/v1/users", 409, body={"username": name, "display_name": "Повтор", "password": "T3st-password-123"}) + + def test_invalid_json_and_content_type(self): + for raw in (b'{"username":', b"null", b"[]", b"true", b"\xff", '{}'.encode('utf-16')): + with self.subTest(raw=raw): + self.error("POST", "/api/v1/users", raw=raw, headers={"Content-Type": "application/json"}) + self.error("POST", "/api/v1/users", 415, raw=b"text", headers={"Content-Type": "text/plain"}) + self.error("POST", "/api/v1/users", 415, raw=b"{}") + + def test_auth_and_access(self): + self.error("GET", "/api/v1/users/me", 401) + for value in (str(uuid4()), "bad-id"): + self.error("GET", "/api/v1/users/me", 401, headers={"X-User-Id": value}) + self.error("GET", "/api/v1/chats/bad-id", 401) + self.error("GET", "/api/v1/chats/bad-id", actor=self.alice) + self.error("GET", "/api/v1/chats/" + str(uuid4()), 404, actor=self.alice) + self.error("GET", self.path, 404, actor=self.eve) + self.error("GET", self.path + "/messages", 404, actor=self.eve) + self.error("POST", self.path + "/messages", 404, actor=self.eve, body={"text": "Чужое"}) + self.assertEqual(self.get_page("/api/v1/chats", self.eve)["items"], []) + + def test_chat_validation_is_atomic(self): + before = self.get_page("/api/v1/chats")["items"] + valid = [self.alice["id"], self.bob["id"]] + for members in ([], [self.bob["id"]], [self.alice["id"], str(uuid4())], + [self.alice["id"], self.alice["id"].upper()], ["bad-id"], valid * 51): + self.error("POST", "/api/v1/chats", actor=self.alice, body={"title": "Чат", "member_ids": members}) + for title in ("", " \t", "я" * 101, 123): + self.error("POST", "/api/v1/chats", actor=self.alice, body={"title": title, "member_ids": valid}) + self.error("POST", "/api/v1/chats", actor=self.alice, body={"title": "Чат", "member_ids": valid, "extra": 1}) + self.assertEqual(before, self.get_page("/api/v1/chats")["items"]) + + def test_message_fields_and_unicode(self): + bodies = [{"text": value} for value in ("", " \t\n", "я" * 4001, None, 123, True, [])] + bodies += [{}, {"text": "Hi", "sender_id": self.bob["id"]}, {"text": "Hi", "unexpected": []}] + for body in bodies: + self.error("POST", self.path + "/messages", actor=self.alice, body=body) + for text in (" Привет, мир 👋\n", "👋" * 4000): + message = self.client.message(self.alice, self.chat, text)["json"] + self.assertEqual(message["text"], text) + self.assertEqual(message["sender_id"], self.alice["id"]) + + def test_message_pagination(self): + self.assertEqual(self.get_page(self.path + "/messages"), {"items": [], "next_cursor": None}) + self.assertEqual(self.get_page(self.path + "/messages", limit="0000000001")["items"], []) + sent = [self.client.message(self.alice if i % 2 else self.bob, self.chat, str(i))["json"] for i in range(6)] + received, cursor = [], None + for limit in (2, 3, 1): + query = {"limit": limit} + if cursor: + query["cursor"] = cursor + page = self.get_page(self.path + "/messages", self.bob, **query) + received.extend(page["items"]) + cursor = page["next_cursor"] + self.assertIsNone(cursor) + self.assertEqual(received, sorted(sent, key=lambda item: (timestamp(item["created_at"]), item["id"]))) + + def test_chat_pagination(self): + chats = [self.chat] + [self.new_chat() for _ in range(4)] + received, cursor = [], None + while True: + query = {"limit": 2} + if cursor: + query["cursor"] = cursor + page = self.get_page("/api/v1/chats", **query) + received += page["items"] + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(received, sorted(chats, key=lambda item: (timestamp(item["created_at"]), item["id"]))) + cursor = self.get_page("/api/v1/chats", limit=1)["next_cursor"] + self.error("GET", "/api/v1/chats?" + urlencode({"cursor": cursor}), actor=self.bob) + + def test_bad_cursors_and_limits(self): + for i in range(2): + self.client.message(self.alice, self.chat, str(i)) + cursor = self.get_page(self.path + "/messages", limit=1)["next_cursor"] + for value in ("", "bad", cursor[:-1] + ("0" if cursor[-1] != "0" else "1")): + self.error("GET", self.path + "/messages?" + urlencode({"cursor": value}), actor=self.alice) + other = self.new_chat() + self.error("GET", "/api/v1/chats/" + other["id"] + "/messages?" + urlencode({"cursor": cursor}), actor=self.alice) + self.error("GET", "/api/v1/chats?" + urlencode({"cursor": cursor}), actor=self.alice) + for path in ("/api/v1/chats", self.path + "/messages"): + for value in ("", "0", "-1", "101", "1.5", "true", "9" * 500): + self.error("GET", path + "?" + urlencode({"limit": value}), actor=self.alice) + + def test_read_after_write(self): + first = self.client.message(self.alice, self.chat, "Первое")["json"] + second = self.client.message(self.bob, self.chat, "Второе")["json"] + page = self.get_page(self.path + "/messages", limit=1) + third = self.client.message(self.bob, self.chat, "Новое")["json"] + rest = self.get_page(self.path + "/messages", self.bob, cursor=page["next_cursor"]) + self.assertEqual(page["items"] + rest["items"], [first, second, third]) + self.assertIsNone(rest["next_cursor"]) + + def test_parallel_messages(self): + def send(worker): + client = Client(self.client.base) + return [client.message(self.alice, self.chat, f"{worker}:{i}")["json"]["id"] for i in range(10)] + with ThreadPoolExecutor(max_workers=10) as pool: + sent = [item for group in pool.map(send, range(10)) for item in group] + received, cursor = [], None + while True: + query = {"limit": 17} + if cursor: + query["cursor"] = cursor + page = self.get_page(self.path + "/messages", self.bob, **query) + received += page["items"] + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(len(received), 100) + self.assertEqual(len({item["text"] for item in received}), 100) + self.assertEqual({item["id"] for item in received}, set(sent)) + + def test_parallel_username(self): + body = {"username": "u_" + uuid4().hex[:20], "display_name": "Один", "password": "T3st-password-123"} + def create(_): + return Client(self.client.base).call("POST", "/api/v1/users", [201, 409], body=body, schema="User")["status"] + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(create, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..04b675ca --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,117 @@ +from concurrent.futures import ThreadPoolExecutor +import hashlib +from urllib.parse import urlencode +from uuid import uuid4 +import unittest + +from helpers import Client, ROOT, image_bytes, oversized_pixels, padded_png + + +class AttachmentTests(unittest.TestCase): + def setUp(self): + self.client = Client(timeout=60) + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = "/api/v1/chats/" + self.chat["id"] + + def upload(self, content=None, media="image/png", name="photo.png", expected=201, actor=None): + return self.client.call( + "POST", self.path + "/attachments?" + urlencode({"filename": name}), expected, + actor=actor or self.alice, raw=image_bytes() if content is None else content, + headers={"Content-Type": media}, schema="Attachment" if expected == 201 else None, + ) + + def test_original_png_jpeg_and_unicode_filename(self): + for kind, media in (("PNG", "image/png"), ("JPEG", "image/jpeg")): + raw = image_bytes(kind) + att = self.upload(raw, media, "фото 👋." + kind.lower())["json"] + self.assertEqual(att["owner_id"], self.alice["id"]) + self.assertEqual(att["sha256"], hashlib.sha256(raw).hexdigest()) + self.assertEqual(att["size_bytes"], len(raw)) + self.assertEqual((att["status"], att["variants"], att["error_code"]), ("ready", [], None)) + path = "/api/v1/attachments/" + att["id"] + metadata = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"] + self.assertEqual(metadata, att) + downloaded = self.client.call("GET", path + "/content", actor=self.bob) + self.assertEqual(downloaded["bytes"], raw) + self.assertEqual(downloaded["headers"].get_content_type(), media) + self.assertNotIn("object_key", metadata) + self.assertNotIn("Location", downloaded["headers"]) + + def test_filename_validation(self): + for name in ("", " \t", "../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\n.png", "a\x7f.png", "я" * 256): + self.upload(name=name, expected=400) + self.assertEqual(self.upload(name="я" * 255)["json"]["filename"], "я" * 255) + + def test_corrupt_images_mime_and_multipart(self): + png, jpeg = image_bytes(), image_bytes("JPEG") + for content, media in ((b"", "image/png"), (b"not an image", "image/png"), + (png[:-8], "image/png"), (jpeg[:-8], "image/jpeg"), + (png, "image/jpeg"), (jpeg, "image/png"), + (image_bytes("GIF"), "image/png"), (png, "multipart/form-data")): + self.upload(content, media, expected=415) + + def test_byte_limit_and_original_at_exact_boundary(self): + raw = padded_png(10485760) + att = self.upload(raw)["json"] + downloaded = self.client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=self.bob) + self.assertEqual(downloaded["bytes"], raw) + self.upload(raw + b"x", expected=413) + + def test_chunked_body_cannot_bypass_size_limit(self): + def blocks(): + for _ in range(161): + yield b"x" * 65536 + self.upload(blocks(), expected=413) + + def test_pixel_boundary_and_decompression_bomb(self): + self.upload(oversized_pixels(), expected=413) + self.assertEqual(self.upload(image_bytes(size=(8000, 5000)))["json"]["status"], "ready") + + def test_attachment_access_and_unknown_ids(self): + att = self.upload()["json"] + for suffix in ("", "/content"): + self.client.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=self.eve) + self.client.call("GET", "/api/v1/attachments/" + str(uuid4()) + suffix, 404, actor=self.alice) + self.client.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 401) + self.upload(actor=self.eve, expected=404) + + def test_messages_keep_attachment_order_and_allow_empty_text(self): + ids = [self.upload()["json"]["id"] for _ in range(2)] + for text in ("", " \t\n", "с картинками"): + message = self.client.message(self.bob, self.chat, text, attachments=ids[::-1])["json"] + self.assertEqual(message["attachment_ids"], ids[::-1]) + self.assertEqual(message["text"], text) + page = self.client.call("GET", self.path + "/messages", actor=self.alice, schema="MessagePage")["json"] + self.assertEqual([m["attachment_ids"] for m in page["items"]], [ids[::-1]] * 3) + + def test_bad_attachment_list_and_other_chat_do_not_create_messages(self): + att = self.upload()["json"]["id"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, + body={"title": "other", "member_ids": [self.alice["id"]]}, schema="Chat")["json"] + foreign = self.client.attachment(self.alice, other)[0]["id"] + for ids, status in (([att, att], 400), ([str(uuid4()) for _ in range(11)], 400), + (["bad"], 400), ([str(uuid4())], 404), ([foreign], 404)): + self.client.call("POST", self.path + "/messages", status, actor=self.alice, + body={"text": "test", "attachment_ids": ids}, headers={"Idempotency-Key": uuid4().hex}) + page = self.client.call("GET", self.path + "/messages", actor=self.alice, schema="MessagePage")["json"] + self.assertEqual(page["items"], []) + + def test_idempotency_defaults_and_attachment_replay(self): + key = uuid4().hex + original = self.client.message(self.alice, self.chat, "text", key=key)["json"] + explicit = self.client.message(self.alice, self.chat, "text", key=key, attachments=[])["json"] + self.assertEqual(original, explicit) + att = self.upload()["json"]["id"] + key = uuid4().hex + def send(_): + return Client(timeout=30).message(self.alice, self.chat, "", key=key, attachments=[att]) + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(send, range(10))) + self.assertEqual(len({r["json"]["id"] for r in responses}), 1) + self.assertEqual(len({r["headers"]["X-Instance-Id"] for r in responses}), 2) + self.client.call("POST", self.path + "/messages", 409, actor=self.alice, + body={"text": "different", "attachment_ids": [att]}, headers={"Idempotency-Key": key}) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..c93c44be --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,140 @@ +from concurrent.futures import ThreadPoolExecutor +import base64 +import json +import os +from pathlib import Path +import subprocess +import sys +import unittest +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from helpers import Client, project_name + + +class IdempotencyTests(unittest.TestCase): + def setUp(self): + self.client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:18104")) + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = "/api/v1/chats/" + self.chat["id"] + "/messages" + + def history(self): + return self.client.call("GET", self.path + "?limit=100", actor=self.alice, schema="MessagePage")["json"]["items"] + + def sql(self, statement): + project = project_name() + command = ["docker", "compose", "--project-name", project, "--env-file", ".env", + "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger", + "-v", "ON_ERROR_STOP=1", "-At", "-c", statement] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8") + if result.returncode: + raise AssertionError("Database test command failed") + return result.stdout.strip() + + def test_key_required_and_format(self): + for key in (None, "", "two words", "a/b", "é", "a" * 129): + headers = {} if key is None else {"Idempotency-Key": key} + self.client.call("POST", self.path, 400, actor=self.alice, body={"text": "test"}, headers=headers) + for key in ("a", ".:_-", "a" * 128): + self.client.message(self.alice, self.chat, key, key=key) + self.assertEqual(len(self.history()), 3) + + def test_replay_normalizes_json(self): + key = uuid4().hex + text = "Привет 👋" + first = self.client.message(self.alice, self.chat, text, key=key) + raw = json.dumps({"text": text}, ensure_ascii=True, indent=4).encode("ascii") + second = self.client.call("POST", self.path, 201, actor=self.alice, raw=raw, + headers={"Idempotency-Key": key, "Content-Type": "application/json"}, schema="Message") + self.assertEqual(first["json"], second["json"]) + self.assertNotEqual(first["headers"]["X-Request-Id"], second["headers"]["X-Request-Id"]) + self.assertEqual(len(self.history()), 1) + + def test_different_body_conflicts(self): + key = uuid4().hex + original = self.client.message(self.alice, self.chat, "first", key=key)["json"] + self.client.call("POST", self.path, 409, actor=self.alice, + body={"text": "second"}, headers={"Idempotency-Key": key}) + self.assertEqual(self.client.message(self.alice, self.chat, "first", key=key)["json"], original) + self.assertEqual(self.history(), [original]) + + def test_twenty_concurrent_requests_use_both_replicas(self): + key = uuid4().hex + def send(_): + return Client(self.client.base).message(self.alice, self.chat, "one message", key=key) + with ThreadPoolExecutor(max_workers=20) as pool: + responses = list(pool.map(send, range(20))) + self.assertEqual(len({item["json"]["id"] for item in responses}), 1) + self.assertEqual(len({item["headers"]["X-Instance-Id"] for item in responses}), 2) + self.assertTrue(all(item["json"] == responses[0]["json"] for item in responses)) + self.assertEqual(self.history(), [responses[0]["json"]]) + + def test_scope_is_user_and_chat_not_session(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "shared", key=key)["json"] + second = self.client.message(self.bob, self.chat, "shared", key=key)["json"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, + body={"title": "other", "member_ids": [self.alice["id"]]}, schema="Chat")["json"] + third = self.client.message(self.alice, other, "shared", key=key)["json"] + self.assertEqual(len({first["id"], second["id"], third["id"]}), 3) + source = self.alice["registration"] + value = base64.b64encode((source["username"] + ":" + source["password"]).encode()).decode() + session = self.client.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + value}, schema="Session")["json"] + actor = {"id": self.alice["id"], "token": session["token"]} + self.assertEqual(self.client.message(actor, self.chat, "shared", key=key)["json"], first) + + def test_replay_still_checks_access_and_session(self): + key = uuid4().hex + self.client.message(self.alice, self.chat, "private", key=key) + self.client.call("POST", self.path, 404, actor=self.eve, + body={"text": "private"}, headers={"Idempotency-Key": key}) + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice) + self.client.call("POST", self.path, 401, actor=self.alice, + body={"text": "private"}, headers={"Idempotency-Key": key}) + + def test_failure_rolls_back_message_and_key(self): + key = "fault-" + uuid4().hex + constraint = "test_reject_" + uuid4().hex[:12] + self.sql("ALTER TABLE idempotency_keys ADD CONSTRAINT " + constraint + + " CHECK(request_key <> '" + key + "') NOT VALID") + try: + self.client.call("POST", self.path, 500, actor=self.alice, + body={"text": "rollback"}, headers={"Idempotency-Key": key}) + self.assertEqual(self.history(), []) + self.assertEqual(self.sql("SELECT count(*) FROM idempotency_keys WHERE request_key='" + key + "'"), "0") + finally: + self.sql("ALTER TABLE idempotency_keys DROP CONSTRAINT " + constraint) + result = self.client.message(self.alice, self.chat, "rollback", key=key)["json"] + self.assertEqual(self.client.message(self.alice, self.chat, "rollback", key=key)["json"], result) + self.assertEqual(self.history(), [result]) + + def test_retention_and_expired_key_reuse(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "old", key=key)["json"] + retained = self.sql("SELECT expires_at-created_at > INTERVAL '23 hours 59 minutes 59 seconds'" + + " FROM idempotency_keys WHERE request_key='" + key + "'") + self.assertEqual(retained, "t") + self.sql("UPDATE idempotency_keys SET expires_at=created_at + INTERVAL '1 microsecond'" + + " WHERE request_key='" + key + "'") + second = self.client.message(self.alice, self.chat, "new", key=key)["json"] + self.assertNotEqual(first["id"], second["id"]) + self.assertEqual(self.history(), [first, second]) + + def test_same_session_on_both_instances(self): + responses = [self.client.call("GET", "/api/v1/users/me", actor=self.alice, schema="User") + for _ in range(12)] + self.assertEqual(len({item["headers"]["X-Instance-Id"] for item in responses}), 2) + self.assertTrue(all(item["json"] == self.alice["user"] for item in responses)) + + def test_invalid_body_does_not_reserve_key(self): + key = uuid4().hex + self.client.call("POST", self.path, 400, actor=self.alice, + body={"text": ""}, headers={"Idempotency-Key": key}) + result = self.client.message(self.alice, self.chat, "fixed", key=key)["json"] + self.assertEqual(self.history(), [result]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..a4af8186 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,96 @@ +import base64 +import json +import os +from pathlib import Path +import re +import sys +import unittest +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from helpers import Client + + +def basic(username, password): + value = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + value} + + +class SessionTests(unittest.TestCase): + def setUp(self): + self.client = Client(os.environ.get("BASE_URL", "http://127.0.0.1:18104")) + self.actor = self.client.user("auth") + + def register(self, password): + body = {"username": "p_" + uuid4().hex[:20], "display_name": "Пароль", "password": password} + user = self.client.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + session = self.client.call("POST", "/api/v1/auth/sessions", 201, + headers=basic(body["username"], password), schema="Session")["json"] + self.assertEqual(session["user"], user) + self.assertEqual(set(session), {"token", "token_type", "expires_at", "user"}) + self.assertTrue(re.fullmatch(r"[A-Za-z0-9_-]{43}", session["token"]), "Invalid token format") + return body, session + + def test_password_required_and_limits(self): + source = self.actor["registration"] + for password in ("x" * 11, "x" * 129, " " * 12, None, 123, True, "a" * 12 + "\x00"): + body = {**source, "username": "p_" + uuid4().hex[:20], "password": password} + self.client.call("POST", "/api/v1/users", 400, body=body) + missing = {key: value for key, value in source.items() if key != "password"} + self.client.call("POST", "/api/v1/users", 400, body=missing) + raw = json.dumps({**source, "username": "p_" + uuid4().hex[:20], "password": "\ud800" * 12}).encode() + self.client.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"}) + + def test_password_boundaries_unicode_colon_spaces(self): + for password in ("x" * 12, "🙂" * 127 + ":", " Привет:пароль🙂 "): + body, session = self.register(password) + actor = {"id": session["user"]["id"], "token": session["token"]} + self.client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + changed = password.strip() if password != password.strip() else password + "!" + self.client.call("POST", "/api/v1/auth/sessions", 401, + headers=basic(body["username"], changed)) + + def test_wrong_and_unknown_credentials(self): + source = self.actor["registration"] + wrong = self.client.call("POST", "/api/v1/auth/sessions", 401, + headers=basic(source["username"], "wrong-password-123")) + unknown = self.client.call("POST", "/api/v1/auth/sessions", 401, + headers=basic("unknown_" + uuid4().hex[:12], source["password"])) + self.assertEqual(wrong["json"]["error"]["message"], unknown["json"]["error"]["message"]) + self.assertEqual(wrong["headers"]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"') + + def test_malformed_basic_and_body(self): + for value in ("", "Basic", "Basic !!!", "Bearer " + self.actor["token"], + "Basic " + base64.b64encode(b"missing-colon").decode(), + "Basic " + base64.b64encode(b"\xff:some-password").decode()): + self.client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": value}) + source = self.actor["registration"] + self.client.call("POST", "/api/v1/auth/sessions", 400, body={}, + headers=basic(source["username"], source["password"])) + + def test_only_bearer_identifies_user(self): + source = self.actor["registration"] + for value in ("", "Bearer", "Bearer bad", "Bearer " + "x" * 43, + basic(source["username"], source["password"])["Authorization"]): + self.client.call("GET", "/api/v1/users/me", 401, headers={"Authorization": value}) + self.client.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": self.actor["id"]}) + other = self.client.user("other") + result = self.client.call("GET", "/api/v1/users/me", actor=self.actor, + headers={"X-User-Id": other["id"]}, schema="User") + self.assertEqual(result["json"]["id"], self.actor["id"]) + self.client.call("GET", "/api/v1/users/me", schema="User", + headers={"Authorization": "bEaReR " + self.actor["token"]}) + + def test_logout_only_revokes_current_session(self): + source = self.actor["registration"] + second = self.client.call("POST", "/api/v1/auth/sessions", 201, + headers=basic(source["username"], source["password"]), schema="Session")["json"] + other_session = {"id": self.actor["id"], "token": second["token"]} + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.actor) + for method, path in (("GET", "/api/v1/users/me"), ("DELETE", "/api/v1/auth/sessions/current")): + self.client.call(method, path, 401, actor=self.actor) + self.client.call("GET", "/api/v1/users/me", actor=other_session, schema="User") + + +if __name__ == "__main__": + unittest.main(verbosity=2)