diff --git a/.dockerignore b/.dockerignore index bc16e556..ee6acd96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +**/__pycache__ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..d5f60538 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,13 @@ -## Лабораторная 4 +## Лабораторная 4. Приватное S3-хранилище и вложения -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор и группа — Samodurov Nikita, bpi2303 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка — 4 -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +API принимает оригиналы PNG и JPEG, хранит их в приватном SeaweedFS S3 и выдаёт участникам чата через любую реплику. PostgreSQL хранит метаданные и связи с сообщениями. + +Команды запуска и ограничения описаны в [REPORT.md](REPORT.md). Результаты smoke, перезапуска и дополнительных сценариев сохранены в [evidence/local-checks.txt](evidence/local-checks.txt). + +Источники — материалы и контракт курса. + +Использование ИИ — консультации и проверки ИИ. diff --git a/.github/workflows/contracts.yml b/.github/workflows/contracts.yml index 11cf289f..805743c8 100644 --- a/.github/workflows/contracts.yml +++ b/.github/workflows/contracts.yml @@ -12,5 +12,30 @@ jobs: python-version: '3.12' - run: python -m pip install -r requirements-dev.txt - run: make check validate PYTHON=python -# This job validates the skeleton, not a student solution. -# Add build, readiness and make test in a separate application job. + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build and start stack + run: | + python scripts/setup.py + chmod +x scripts/restart.sh scripts/stop-one.sh + docker compose build migrate + docker compose up -d --wait + - name: Run HTTP and resilience checks + run: | + make test + make test-persistence ACTION_SCRIPT=scripts/restart.sh + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + docker compose start api-a api-b + python student-tests/check_attachments.py + - name: Show service status + if: failure() + run: docker compose ps -a + - name: Stop stack + if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0b38d1e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.13.5-slim-bookworm@sha256:4c2cf9917bd1cbacc5e9b07320025bdb7cdf2df7b0ceaccb55e9dd7e30987419 + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY migrations/ ./migrations/ +COPY src/ ./src/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=3s --timeout=3s --start-period=5s --retries=4 \ + CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=2).close()"] +CMD ["python", "src/server.py"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..78932666 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,57 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор и группа — Samodurov Nikita, bpi2303 +- Целевая оценка — 4 +- Источники — материалы и контракт курса. +- Использование ИИ — консультации и проверки ИИ. -## Запуск из чистого clone +## Запуск из чистого клона -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Контейнер использует Python 3.13.5. По умолчанию API доступен на http://127.0.0.1:8080. При занятом порте задайте `PUBLIC_PORT` и соответствующий `BASE_URL`. Команда setup создаёт локальные `.env` и `secrets/`. -## Архитектура и решения +```bash +python scripts/setup.py +make check +docker compose config --quiet +docker compose build migrate +docker compose up -d --wait +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose start api-a api-b +python student-tests/check_attachments.py +docker compose exec -T api-a python src/cleanup_attachments.py +docker compose down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +В Windows PowerShell используйте следующие проверки после запуска стенда. Перед проверками установите `$env:PYTHONUTF8 = '1'`. -## Доказательства критериев +```powershell +python scripts/check_contract.py +python tests/smoke.py +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +docker compose start api-a api-b +python student-tests/check_attachments.py +docker compose exec -T api-a python src/cleanup_attachments.py +``` -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Для полной валидации контракта установите `requirements-dev.txt` в отдельное окружение и выполните `python scripts/validate_openapi.py`. Команда `docker compose down` оставляет именованные тома. Для намеренного сброса учебных данных выполните `docker compose down -v`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура -## Эксперименты и отказы +Из ЛР 3 перенесены HAProxy, общие сессии и идемпотентная отправка сообщений. PostgreSQL хранит метаданные, SeaweedFS хранит байты оригиналов. S3 bootstrap создаёт bucket при необходимости. Административные credentials доступны bootstrap, API получает отдельную учётную запись с действиями Read, Write и List только для bucket вложений. Порты SeaweedFS с хоста не опубликованы. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +HTTP-обработчик принимает одно изображение как двоичный поток. Лимит составляет 10 MiB, временный файл заполняется порциями по 64 KiB. Сервер проверяет формат и целостность изображения через Pillow. Лимит 40 млн пикселей проверяется до декодирования. Два параллельных декодирования на реплику ограничивают потребление RAM. Сервер генерирует ключ объекта из UUID. Исходное имя хранится как метаданные. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Сначала создаётся запись о незавершённой загрузке. После успешного PUT она становится доступной участникам чата. Сценарий очистки удаляет записи старше часа и соответствующие S3-объекты. Блокировка строки защищает от одновременной очистки, готовые записи не выбираются. Если S3 недоступен, запись остаётся для следующей попытки очистки. -## Ограничения и следующие шаги +Для вложений в сообщении используются уникальные UUID того же чата. В fingerprint идемпотентности входят текст и список вложений. Отсутствующий список эквивалентен пустому. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Проверки + +Результаты локального Docker-стенда и команды приведены в [evidence/local-checks.txt](evidence/local-checks.txt). + +## Ограничения + +При недоступности S3 readiness возвращает 503. Одноузловые PostgreSQL и SeaweedFS сохраняют данные при рестарте с томами, отказ диска или хоста находится за пределами стенда. Удаление незавершённых загрузок запускает оператор. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..738ef1ad --- /dev/null +++ b/compose.yaml @@ -0,0 +1,118 @@ +x-app: &app + image: mtusi-messenger:lab4 + cpus: 1.0 + mem_limit: 640m + env_file: .env + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url, cursor_key] + +services: + db: + cpus: 1.0 + mem_limit: 384m + image: postgres:17.5-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + cache: + cpus: 0.25 + mem_limit: 96m + image: redis:7.4.3-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "32mb", "--maxmemory-policy", "allkeys-lru"] + + migrate: + <<: *app + build: . + command: ["python", "src/migrate.py"] + healthcheck: + disable: true + depends_on: + db: + condition: service_healthy + + s3: + image: chrislusf/seaweedfs:3.97 + cpus: 1.0 + mem_limit: 512m + command: ["server", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: [objects:/data] + + s3-init: + <<: *app + command: ["python", "src/bootstrap_s3.py"] + environment: + S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key + S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key + secrets: [s3_admin_access_key, s3_admin_secret_key] + healthcheck: + disable: true + depends_on: [s3] + + api-a: &api + <<: *app + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + INSTANCE_ID: api-a + secrets: [database_url, cache_url, cursor_key, s3_access_key_id, s3_secret_access_key] + depends_on: + s3-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + cache: + condition: service_started + stop_grace_period: 8s + + api-b: + <<: *api + environment: + CURSOR_KEY_FILE: /run/secrets/cursor_key + INSTANCE_ID: api-b + + lb: + image: haproxy:3.0.11-alpine + cpus: 0.5 + mem_limit: 96m + ports: ["127.0.0.1:${PUBLIC_PORT:-8080}:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + cursor_key: + file: ./secrets/cursor_key + s3_config: + file: ./secrets/s3_config + s3_admin_access_key: + file: ./secrets/s3_admin_access_key + s3_admin_secret_key: + file: ./secrets/s3_admin_secret_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + pgdata: + objects: diff --git a/evidence/local-checks.txt b/evidence/local-checks.txt new file mode 100644 index 00000000..5605aa8e --- /dev/null +++ b/evidence/local-checks.txt @@ -0,0 +1,32 @@ +2026-11-12. Windows, Python 3.13.5, Docker Desktop с Linux-контейнерами. +PUBLIC_PORT=18084, BASE_URL=http://127.0.0.1:18084 + +> python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +> python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +> python tests/smoke.py --base-url http://127.0.0.1:18084 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. + +> python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +Hook: docker compose restart db cache s3 api-a api-b lb. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +> python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +{"observed_instances_before": 2, "successful_writes": 103, "failed_attempts": 4, "first_recovery_seconds": 1.593} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +> python student-tests/check_attachments.py +PASS: size, pixels, malformed image, MIME, filename and cross-chat access +PASS: original bytes read through api-a after api-b stopped +PASS: anonymous direct access to S3 object returned 403 +PASS: 10 concurrent uploads of 787252 bytes; cgroup memory baseline={'api-a': 64753664, 'api-b': 58634240}, sampled_peak={'api-a': 78761984, 'api-b': 68505600} +PASS: readiness returned 503 during S3 outage, original survived restart + +> docker compose exec -T api-a python src/cleanup_attachments.py +Removed unfinished uploads: 0 + +Размеры памяти в байтах. Это наблюдаемый пик выборки cgroup, включая +память процесса и файловый кэш. Ограничение на контейнер API — 640 MiB. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..fa9f06b6 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,34 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option http-server-close + timeout connect 1s + timeout client 15s + timeout server 12s + timeout http-request 8s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 500ms + timeout retry 500ms + hold valid 1s + hold nx 1s + +frontend messenger + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 1 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api-a api-a:8080 + server api-b api-b:8080 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..3da9a650 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE members ( + chat_id uuid REFERENCES chats(id), + user_id uuid REFERENCES users(id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX members_user_idx ON members (user_id, chat_id); +CREATE INDEX chats_order_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members (chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..01d78cb9 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_requests ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL, + fingerprint bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (actor_id, chat_id, key) +); + +CREATE INDEX message_requests_expiry_idx ON message_requests (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..23ff1d0d --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL, + ready boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, owner_id) REFERENCES members (chat_id, user_id) +); +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE NOT ready; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000 AND + cardinality(attachment_ids) <= 10 AND (char_length(text) > 0 OR cardinality(attachment_ids) > 0)); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b5b2f831 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +typing-extensions==4.14.0 +boto3==1.39.3 +botocore==1.39.3 +jmespath==1.0.1 +python-dateutil==2.9.0.post0 +s3transfer==0.13.0 +six==1.17.0 +urllib3==2.5.0 +Pillow==12.2.0 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..06f628e6 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +docker compose restart db cache s3 api-a api-b lb diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..ccbea8bb --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart db cache s3 api-a api-b lb diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..9e9108a6 --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,47 @@ +from pathlib import Path +import json +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / "secrets" +directory.mkdir(mode=0o700, exist_ok=True) + + +def ensure(name, value): + path = directory / name + try: + with path.open("x", encoding="utf-8") as file: + file.write(value + "\n") + except FileExistsError: + pass + return path.read_text(encoding="utf-8").strip() + + +password = ensure("db_password", secrets.token_hex(24)) +ensure("database_url", f"postgresql://messenger:{password}@db:5432/messenger") +ensure("cache_url", "redis://cache:6379/0") +ensure("cursor_key", secrets.token_hex(32)) +admin = ensure("s3_admin_access_key", "admin-" + secrets.token_hex(8)) +admin_password = ensure("s3_admin_secret_key", secrets.token_hex(24)) +app = ensure("s3_access_key_id", "app-" + secrets.token_hex(8)) +app_password = ensure("s3_secret_access_key", secrets.token_hex(20)) + +try: + with (root / ".env").open("xb") as file: + file.write((root / ".env.example").read_bytes()) +except FileExistsError: + pass + + +env = dict( + line.split("=", 1) for line in (root / ".env").read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") and "=" in line +) +bucket = env["S3_BUCKET"] +config = {"identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": admin, "secretKey": admin_password}], + "actions": ["Admin"]}, + {"name": "messenger", "credentials": [{"accessKey": app, "secretKey": app_password}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, +]} +(directory / "s3_config").write_text(json.dumps(config) + "\n", encoding="utf-8") diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..c7718fe6 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api-a" goto stop +if "%TARGET_INSTANCE_ID%"=="api-b" goto stop +exit /b 2 +:stop +docker compose stop --timeout 5 %TARGET_INSTANCE_ID% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..1f987b89 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +case "${TARGET_INSTANCE_ID:-}" in + api-a|api-b) docker compose stop --timeout 5 "$TARGET_INSTANCE_ID" ;; + *) exit 2 ;; +esac diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..dfe4303f --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,164 @@ +import hashlib +import os +import re +import time +import unicodedata +from tempfile import TemporaryFile +from threading import BoundedSemaphore +from uuid import uuid4 + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image + +from config import positive_int, secret +from storage import unavailable +from validation import ApiError, invalid, not_found + +MIME_FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +S3_ERRORS = (BotoCoreError, ClientError) +CHUNK = 64 * 1024 + + +class Attachments: + def __init__(self, store): + self.store = store + self.bucket = os.environ["S3_BUCKET"] + self.max_bytes = min(positive_int("MAX_UPLOAD_BYTES", 10485760), 10485760) + self.max_pixels = min(positive_int("MAX_IMAGE_PIXELS", 40000000), 40000000) + self.slots = BoundedSemaphore(2) + self.s3 = boto3.client( + "s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.environ["S3_REGION"], + aws_access_key_id=secret("S3_ACCESS_KEY_ID"), aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"), + config=Config(connect_timeout=1, read_timeout=3, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", s3={"addressing_style": "path"}), + ) + + def ready(self): + try: + self.s3.head_bucket(Bucket=self.bucket) + except S3_ERRORS: + raise unavailable() from None + + @staticmethod + def key(attachment_id): + return f"attachments/{attachment_id}" + + def upload(self, request, actor, chat_id, filename): + if (not 1 <= len(filename) <= 255 or filename in (".", "..") + or any(c in "/\\" or unicodedata.category(c) in ("Cc", "Cs") for c in filename)): + raise invalid("Invalid filename") + content_type = request.headers.get_content_type() + if content_type not in MIME_FORMATS: + raise ApiError(415, "unsupported_media_type", "Expected PNG or JPEG") + lengths = request.headers.get_all("Content-Length", []) + if (request.headers.get("Transfer-Encoding") or len(lengths) != 1 + or not re.fullmatch(r"[0-9]{1,19}", lengths[0])): + raise invalid("Content-Length required") + length = int(lengths[0]) + if length > self.max_bytes: + raise ApiError(413, "payload_too_large", "Attachment is too large") + if not self.slots.acquire(timeout=2): + raise unavailable() + try: + with TemporaryFile() as file: + checksum = self.receive(request, file, length) + self.validate(file, content_type) + attachment_id = uuid4() + self.reserve(attachment_id, chat_id, actor, filename, content_type, length, checksum) + try: + self.s3.put_object(Bucket=self.bucket, Key=self.key(attachment_id), Body=file, + ContentLength=length, ContentType=content_type) + except S3_ERRORS: + raise unavailable() from None + with self.store.pool.connection() as db: + row = db.execute(""" + UPDATE attachments SET ready=true,updated_at=clock_timestamp() + WHERE id=%s AND NOT ready RETURNING * + """, (attachment_id,)).fetchone() + if row is None: + raise unavailable() + return self.public(row) + finally: + self.slots.release() + + @staticmethod + def receive(request, file, length): + checksum = hashlib.sha256() + deadline = time.monotonic() + 10 + remaining = length + while remaining: + if time.monotonic() > deadline: + raise ApiError(408, "invalid_request", "Upload timed out") + chunk = request.rfile.read1(min(CHUNK, remaining)) + if not chunk: + raise invalid("Incomplete attachment") + file.write(chunk) + checksum.update(chunk) + remaining -= len(chunk) + file.seek(0) + return checksum.hexdigest() + + def validate(self, file, content_type): + try: + with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image: + if image.width * image.height > self.max_pixels: + raise ApiError(413, "payload_too_large", "Image pixel limit exceeded") + image.verify() + file.seek(0) + with Image.open(file, formats=[MIME_FORMATS[content_type]]) as image: + image.load() + except Image.DecompressionBombError: + raise ApiError(413, "payload_too_large", "Image pixel limit exceeded") from None + except (OSError, ValueError, SyntaxError, EOFError): + raise ApiError(415, "unsupported_media_type", "Invalid image") from None + file.seek(0) + + def reserve(self, attachment_id, chat_id, actor, filename, content_type, length, checksum): + with self.store.pool.connection() as db: + db.execute(""" + INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256) + VALUES (%s,%s,%s,%s,%s,%s,%s) + """, (attachment_id, chat_id, actor, filename, content_type, length, checksum)) + + @staticmethod + def public(row): + return {**{name: value for name, value in row.items() if name != "ready"}, + "status": "ready", "variants": [], "error_code": None} + + def get(self, actor, attachment_id): + with self.store.pool.connection() as db: + row = db.execute(""" + SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id + WHERE a.id=%s AND a.ready AND m.user_id=%s + """, (attachment_id, actor)).fetchone() + if row is None: + raise not_found() + return self.public(row) + + def download(self, attachment): + try: + result = self.s3.get_object(Bucket=self.bucket, Key=self.key(attachment["id"])) + except S3_ERRORS: + raise unavailable() from None + if result["ContentLength"] != attachment["size_bytes"]: + result["Body"].close() + raise unavailable() + return result["Body"] + + def cleanup(self): + count = 0 + while True: + with self.store.pool.connection() as db: + row = db.execute(""" + SELECT id FROM attachments + WHERE NOT ready AND created_at < now() - interval '1 hour' + ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED + """).fetchone() + if row is None: + return count + attachment_id = row["id"] + self.s3.delete_object(Bucket=self.bucket, Key=self.key(attachment_id)) + db.execute("DELETE FROM attachments WHERE id=%s", (attachment_id,)) + count += 1 diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 00000000..29b7be9f --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,42 @@ +import os +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, EndpointConnectionError + +from config import secret + + +def main(): + bucket = os.environ["S3_BUCKET"] + s3 = boto3.client( + "s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.environ["S3_REGION"], + aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY"), + aws_secret_access_key=secret("S3_ADMIN_SECRET_KEY"), + config=Config(connect_timeout=1, read_timeout=2, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", s3={"addressing_style": "path"}), + ) + deadline = time.monotonic() + 45 + while True: + try: + s3.head_bucket(Bucket=bucket) + break + except ClientError as error: + if error.response["ResponseMetadata"]["HTTPStatusCode"] != 404: + raise + try: + s3.create_bucket(Bucket=bucket) + except ClientError as conflict: + if conflict.response["Error"]["Code"] != "BucketAlreadyOwnedByYou": + raise + break + except EndpointConnectionError: + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + print("Private S3 bucket is ready") + + +if __name__ == "__main__": + main() diff --git a/src/cleanup_attachments.py b/src/cleanup_attachments.py new file mode 100644 index 00000000..fc8f5f77 --- /dev/null +++ b/src/cleanup_attachments.py @@ -0,0 +1,9 @@ +from attachments import Attachments +from storage import Store + +if __name__ == "__main__": + store = Store() + try: + print(f"Removed unfinished uploads: {Attachments(store).cleanup()}") + finally: + store.close() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..fecaff1e --- /dev/null +++ b/src/config.py @@ -0,0 +1,13 @@ +import os +from pathlib import Path + + +def secret(name): + return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip() + + +def positive_int(name, default): + value = int(os.getenv(name, default)) + if value < 1: + raise ValueError(f"{name} must be positive") + return value diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1c8bbfd8 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +from pathlib import Path +import psycopg +from config import secret + + +def main(): + with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as connection: + connection.execute("SELECT pg_advisory_xact_lock(230301)") + connection.execute("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY)") + for path in sorted(Path("migrations").glob("*.sql")): + if connection.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone(): + continue + connection.execute(path.read_text(encoding="utf-8")) + connection.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,)) + print(f"Applied {path.name}") + + +if __name__ == "__main__": + main() diff --git a/src/profile_cache.py b/src/profile_cache.py new file mode 100644 index 00000000..c07ae5b8 --- /dev/null +++ b/src/profile_cache.py @@ -0,0 +1,56 @@ +from concurrent.futures import Future, TimeoutError +from queue import Empty, Full, Queue +from threading import Event, Thread + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +class ProfileCache: + def __init__(self, url): + self.client = redis.Redis.from_url( + url, socket_timeout=0.25, socket_connect_timeout=0.25, + max_connections=1, retry=Retry(NoBackoff(), 0), + ) + self.work = Queue(maxsize=4) + self.stopping = Event() + Thread(target=self.run, daemon=True, name="profile-cache").start() + + def close(self): + self.stopping.set() + + def run(self): + try: + while not self.stopping.is_set(): + try: + result, operation, args = self.work.get(timeout=0.25) + except Empty: + continue + if not result.set_running_or_notify_cancel(): + continue + try: + result.set_result(operation(*args)) + except redis.RedisError: + result.set_result(None) + finally: + self.client.close() + + def submit(self, operation, *args): + result = Future() + try: + self.work.put_nowait((result, operation, args)) + except Full: + result.set_result(None) + return result + + def get(self, key): + result = self.submit(self.client.get, key) + try: + return result.result(timeout=0.35) + except TimeoutError: + result.cancel() + return None + + def set(self, key, value): + self.submit(self.client.setex, key, 300, value) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..046b4b48 --- /dev/null +++ b/src/server.py @@ -0,0 +1,343 @@ +import base64 +import json +import logging +import os +import re +import signal +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, unquote, urlsplit +from uuid import uuid4 + +from attachments import Attachments, S3_ERRORS +from storage import DB_UNAVAILABLE, Store, json_value, unauthorized, unavailable +from validation import ApiError, canonical_id, invalid, not_found, require_fields, require_text + +CHAT_PATH = re.compile(r"/api/v1/chats/([^/]+)(?:/(messages|attachments))?") +ATTACHMENT_PATH = re.compile(r"/api/v1/attachments/([^/]+)(/content)?") +MAX_JSON_BYTES = 64 * 1024 +LOG = logging.getLogger("messenger") + + +def unique_object(pairs): + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError + return result + + +def page_query(query): + try: + params = parse_qs(query, keep_blank_values=True, max_num_fields=16) + except ValueError: + raise invalid("Invalid query") from None + limits = params.get("limit", ["50"]) + cursors = params.get("cursor", [None]) + if len(limits) != 1 or not re.fullmatch(r"[0-9]{1,3}", limits[0]): + raise invalid("Invalid limit") + limit = int(limits[0]) + if not 1 <= limit <= 100: + raise invalid("Invalid limit") + if len(cursors) != 1: + raise invalid("Invalid cursor") + return limit, cursors[0] + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.handle_api() + + def do_POST(self): + self.handle_api() + + def do_DELETE(self): + self.handle_api() + + def log_message(self, format, *args): + pass + + def log_request(self, code="-", size="-"): + LOG.info("request_id=%s status=%s", self.request_id, code) + + def send_error(self, code, message=None, explain=None): + self.request_id = uuid4().hex + self.response_headers = {} + self.send_json(code, {"error": { + "code": "invalid_request", + "message": "Invalid HTTP request", + "request_id": self.request_id, + }}) + + def handle_api(self): + self.request_id = uuid4().hex + self.response_headers = {} + try: + try: + status, payload = self.route() + except ApiError as error: + status, payload = self.error_payload(error) + except DB_UNAVAILABLE: + status, payload = self.error_payload(unavailable()) + except (TimeoutError, ConnectionError): + return + except Exception: + LOG.error("request_id=%s unexpected error", self.request_id) + status, payload = self.error_payload( + ApiError(500, "internal_error", "Internal server error") + ) + if isinstance(payload, tuple): + self.send_attachment(*payload) + else: + self.send_json(status, payload) + except (TimeoutError, ConnectionError, *S3_ERRORS): + return + + def error_payload(self, error): + self.response_headers = error.headers + return error.status, {"error": { + "code": error.code, + "message": str(error), + "request_id": self.request_id, + }} + + def send_json(self, status, payload): + data = b"" if status == 204 else json.dumps(payload, ensure_ascii=False, default=json_value).encode("utf-8") + self.send_response(status) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.server.instance_id) + if status != 204: + self.send_header("Content-Type", "application/json") + for name, value in self.response_headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(data) + + def send_attachment(self, stream, attachment): + with stream: + self.send_response(200) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.server.instance_id) + self.send_header("Content-Type", attachment["content_type"]) + self.send_header("Content-Length", str(attachment["size_bytes"])) + self.end_headers() + for chunk in stream.iter_chunks(64 * 1024): + self.wfile.write(chunk) + + def read_json(self): + lengths = self.headers.get_all("Content-Length", []) + if ( + self.headers.get("Transfer-Encoding") + or len(lengths) != 1 + or not re.fullmatch(r"[0-9]{1,10}", lengths[0]) + or self.headers.get_content_type() != "application/json" + ): + raise invalid("Expected a JSON body") + length = int(lengths[0]) + if length > MAX_JSON_BYTES: + raise invalid("JSON body is too large") + try: + data = self.rfile.read(length) + if len(data) != length: + raise ValueError + return json.loads(data.decode("utf-8"), object_pairs_hook=unique_object) + except (ValueError, RecursionError): + raise invalid("Invalid JSON") from None + + def route(self): + try: + target = urlsplit(self.path) + except ValueError: + raise invalid("Invalid request path") from None + path, method = target.path, self.command + app = self.server.app + + if method == "GET" and path == "/health/live": + return 200, {"status": "ok"} + if method == "GET" and path == "/health/ready": + app.ready() + self.server.attachments.ready() + return 200, {"status": "ok"} + if method == "POST" and path == "/api/v1/users": + body = self.read_json() + require_fields(body, "username", "display_name", "password") + username = body["username"] + if not isinstance(username, str) or not re.fullmatch(r"[a-z0-9_]{3,32}", username): + raise invalid("Invalid username") + password = body["password"] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise invalid("Invalid password") + try: + password.encode("utf-8") + except UnicodeError: + raise invalid("Invalid password") from None + return 201, app.register(username, require_text(body["display_name"], 100), password) + if method == "POST" and path == "/api/v1/auth/sessions": + return 201, app.login(*self.basic_credentials()) + + match = CHAT_PATH.fullmatch(path) + attachment_match = ATTACHMENT_PATH.fullmatch(path) + if path not in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions/current") and not (match or attachment_match): + raise not_found() + header = self.headers.get("Authorization", "") + bearer = re.fullmatch(r"(?i:Bearer) ([A-Za-z0-9_-]{43})", header) + if not bearer: + raise unauthorized("Bearer") + digest = app.token_hash(bearer[1]) + actor = app.actor(digest) + if path == "/api/v1/auth/sessions/current" and method == "DELETE": + app.logout(digest) + return 204, None + if path == "/api/v1/users/me" and method == "GET": + return 200, app.user(actor) + if path == "/api/v1/chats": + if method == "POST": + body = self.read_json() + require_fields(body, "title", "member_ids") + title = require_text(body["title"], 100) + members = body["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise invalid("Invalid member list") + members = [canonical_id(member) for member in members] + if actor not in members or len(set(members)) != len(members): + raise invalid("Invalid member list") + return 201, app.create_chat(title, members) + if method == "GET": + return 200, app.chat_page(actor, *page_query(target.query)) + if attachment_match and method == "GET": + attachment_id = canonical_id(unquote(attachment_match[1])) + attachment = self.server.attachments.get(actor, attachment_id) + if not attachment_match[2]: + return 200, attachment + params = self.attachment_query(target.query) + if params.get("variant", ["original"]) != ["original"]: + raise not_found() + return 200, (self.server.attachments.download(attachment), attachment) + if match: + chat_id = canonical_id(unquote(match[1])) + chat = app.chat(actor, chat_id) + if match[2] == "attachments" and method == "POST": + filenames = self.attachment_query(target.query).get("filename", []) + if len(filenames) != 1: + raise invalid("Filename required") + return 201, self.server.attachments.upload(self, actor, chat_id, filenames[0]) + if match[2] == "messages": + if method == "POST": + body = self.read_json() + if not isinstance(body, dict) or "text" not in body or body.keys() - {"text", "attachment_ids"}: + raise invalid("Invalid request fields") + attachment_ids = body.get("attachment_ids", []) + if not isinstance(attachment_ids, list) or len(attachment_ids) > 10: + raise invalid("Invalid attachment IDs") + attachment_ids = [canonical_id(item) for item in attachment_ids] + if len(set(attachment_ids)) != len(attachment_ids): + raise invalid("Duplicate attachment IDs") + text = body["text"] + if text != "" or not attachment_ids: + text = require_text(text, 4000) + key = self.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise invalid("Invalid Idempotency-Key") + return 201, app.send(actor, chat_id, text, key, attachment_ids) + if method == "GET": + return 200, app.message_page(chat_id, *page_query(target.query)) + elif match[2] is None and method == "GET": + return 200, chat + raise not_found() + + @staticmethod + def attachment_query(query): + try: + return parse_qs(query, keep_blank_values=True, max_num_fields=16, errors="strict") + except ValueError: + raise invalid("Invalid query") from None + + def basic_credentials(self): + if self.headers.get("Content-Length", "0") != "0" or self.headers.get("Transfer-Encoding"): + raise invalid("Login does not accept a body") + match = re.fullmatch(r"(?i:Basic) ([A-Za-z0-9+/=]{1,800})", + self.headers.get("Authorization", "")) + if not match: + raise unauthorized("Basic") + try: + value = base64.b64decode(match[1], validate=True).decode("utf-8") + except ValueError: + raise unauthorized("Basic") from None + username, separator, password = value.partition(":") + if not separator or not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise unauthorized("Basic") + return username, password + + +class Server(ThreadingHTTPServer): + timeout = 0.25 + request_queue_size = 64 + + def __init__(self, address): + self.app = Store() + self.attachments = Attachments(self.app) + self.instance_id = os.getenv("INSTANCE_ID") or uuid4().hex + self.stopping = False + self.active = 0 + self.workers = threading.Condition() + super().__init__(address, Handler) + + def get_request(self): + connection, address = super().get_request() + connection.settimeout(4) + return connection, address + + def verify_request(self, request, client_address): + return not self.stopping + + def process_request(self, request, client_address): + with self.workers: + self.active += 1 + try: + super().process_request(request, client_address) + except Exception: + self.worker_done() + raise + + def process_request_thread(self, request, client_address): + try: + super().process_request_thread(request, client_address) + finally: + self.worker_done() + + def worker_done(self): + with self.workers: + self.active -= 1 + self.workers.notify_all() + + def handle_error(self, request, client_address): + LOG.error("HTTP connection handler failed") + + def stop(self, signum, frame): + self.stopping = True + + def run(self): + while not self.stopping: + self.handle_request() + self.server_close() + with self.workers: + self.workers.wait_for(lambda: self.active == 0, timeout=5) + self.app.close() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + host = os.getenv("APP_HOST", "0.0.0.0") + port = int(os.getenv("APP_PORT", "8080")) + environment = os.getenv("APP_ENV", "lab") + with Server((host, port)) as server: + signal.signal(signal.SIGTERM, server.stop) + signal.signal(signal.SIGINT, server.stop) + LOG.info("started host=%s port=%s env=%s", host, server.server_port, environment) + server.run() + LOG.info("stopped") + + +if __name__ == "__main__": + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..7ff7c1d2 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,243 @@ +import base64 +import hashlib +import hmac +import json +import logging +import secrets +from datetime import datetime +from threading import BoundedSemaphore +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from config import positive_int, secret +from profile_cache import ProfileCache +from validation import ApiError, invalid, not_found + +USER = "id, username, display_name, created_at" +MESSAGE = "id, chat_id, sender_id, text, created_at, attachment_ids" +CHAT = """c.id, c.title, c.created_at, + ARRAY(SELECT user_id FROM members WHERE chat_id=c.id ORDER BY position) AS member_ids""" +DB_UNAVAILABLE = (psycopg.OperationalError, PoolTimeout, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable) +LOG = logging.getLogger("messenger") + + +def json_value(value): + if isinstance(value, datetime): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + if isinstance(value, UUID): + return str(value) + raise TypeError(type(value).__name__) + + +def unavailable(): + return ApiError(503, "unavailable", "Dependency temporarily unavailable") + + +def unauthorized(scheme): + challenge = f'{scheme} realm="messenger"' + if scheme == "Basic": + challenge += ', charset="UTF-8"' + return ApiError(401, "unauthorized", "Invalid credentials", {"WWW-Authenticate": challenge}) + + +class Store: + def __init__(self): + self.pool = ConnectionPool( + secret("DATABASE_URL"), min_size=1, max_size=8, max_waiting=24, timeout=1.5, + kwargs={"row_factory": dict_row, "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"}, + open=True, check=ConnectionPool.check_connection, + ) + self.cache = ProfileCache(secret("CACHE_URL")) + self.cursor_key = bytes.fromhex(secret("CURSOR_KEY")) + self.ttl = positive_int("SESSION_TTL_SECONDS", 3600) + self.idempotency_ttl = positive_int("IDEMPOTENCY_TTL_SECONDS", 86400) + self.hash_slots = BoundedSemaphore(4) + self.dummy_hash = bytes(16 + 32) + + def close(self): + self.cache.close() + self.pool.close() + + def ready(self): + with self.pool.connection() as db: + db.execute("SELECT 1 FROM users LIMIT 1") + + def password_hash(self, password, salt): + if not self.hash_slots.acquire(timeout=0.5): + raise unavailable() + try: + return salt + hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=2**15, r=8, p=1, + maxmem=64 * 1024**2, dklen=32, + ) + finally: + self.hash_slots.release() + + def register(self, username, name, password): + password_hash = self.password_hash(password, secrets.token_bytes(16)) + try: + with self.pool.connection() as db: + return db.execute(f""" + INSERT INTO users (username,display_name,password_hash) VALUES (%s,%s,%s) + RETURNING {USER} + """, (username, name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Username already exists") from None + + def login(self, username, password): + with self.pool.connection() as db: + row = db.execute(f"SELECT {USER},password_hash FROM users WHERE username=%s", + (username,)).fetchone() + stored = row.pop("password_hash") if row else self.dummy_hash + matches = hmac.compare_digest(self.password_hash(password, stored[:16]), stored) + if not matches or row is None: + raise unauthorized("Basic") + token = secrets.token_urlsafe(32) + digest = self.token_hash(token) + with self.pool.connection() as db: + db.execute("DELETE FROM sessions WHERE expires_at <= now()") + session = db.execute(""" + INSERT INTO sessions (token_hash,user_id,expires_at) + VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') + RETURNING expires_at + """, (digest, row["id"], self.ttl)).fetchone() + return {"token": token, "token_type": "Bearer", **session, "user": row} + + @staticmethod + def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + def actor(self, digest): + with self.pool.connection() as db: + row = db.execute(""" + SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).fetchone() + if row is None: + raise unauthorized("Bearer") + return row["user_id"] + + def user(self, user_id): + key = f"user:v1:{user_id}" + value = self.cache.get(key) + if value is not None: + try: + return json.loads(value) + except ValueError: + LOG.info("invalid profile cache value") + with self.pool.connection() as db: + user = db.execute(f"SELECT {USER} FROM users WHERE id=%s", (user_id,)).fetchone() + self.cache.set(key, json.dumps(user, default=json_value)) + return user + + def logout(self, digest): + with self.pool.connection() as db: + removed = db.execute(""" + DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() + """, (digest,)).rowcount + if not removed: + raise unauthorized("Bearer") + + def create_chat(self, title, members): + try: + with self.pool.connection() as db: + chat = db.execute("INSERT INTO chats (title) VALUES (%s) RETURNING *", + (title,)).fetchone() + with db.cursor() as cursor: + cursor.executemany("INSERT INTO members (chat_id,user_id,position) VALUES (%s,%s,%s)", + [(chat["id"], member, index) for index, member in enumerate(members)]) + return {**chat, "member_ids": members} + except psycopg.errors.ForeignKeyViolation: + raise invalid("Unknown member") from None + + def chat(self, actor, chat_id): + with self.pool.connection() as db: + row = db.execute(f""" + SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s AND c.id=%s + """, (actor, chat_id)).fetchone() + if row is None: + raise not_found() + return row + + def send(self, actor, chat_id, text, key, attachment_ids): + fingerprint = hashlib.sha256(text.encode("utf-8")).digest() + if attachment_ids: + payload = json.dumps([text, attachment_ids], default=json_value).encode("utf-8") + fingerprint = hashlib.sha256(b"\x00" + payload).digest() + identity = (actor, chat_id, key) + with self.pool.connection() as db: + db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", + (f"{actor}:{chat_id}:{key}",)) + request = db.execute(""" + SELECT fingerprint,message_id FROM message_requests + WHERE actor_id=%s AND chat_id=%s AND key=%s AND expires_at > clock_timestamp() + """, identity).fetchone() + if request is not None: + if not hmac.compare_digest(request["fingerprint"], fingerprint): + raise ApiError(409, "conflict", "Idempotency key already used for another body") + return db.execute(f"SELECT {MESSAGE} FROM messages WHERE id=%s", + (request["message_id"],)).fetchone() + if attachment_ids: + row = db.execute(""" + SELECT count(*) AS found FROM attachments + WHERE id=ANY(%s) AND chat_id=%s AND ready + """, (attachment_ids, chat_id)).fetchone() + if row["found"] != len(attachment_ids): + raise not_found() + message = db.execute(f""" + INSERT INTO messages (chat_id,sender_id,text,attachment_ids) + VALUES (%s,%s,%s,%s) RETURNING {MESSAGE} + """, (chat_id, actor, text, attachment_ids)).fetchone() + db.execute(""" + INSERT INTO message_requests (actor_id,chat_id,key,fingerprint,message_id,expires_at) + VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second') + ON CONFLICT (actor_id,chat_id,key) DO UPDATE + SET fingerprint=excluded.fingerprint,message_id=excluded.message_id,expires_at=excluded.expires_at + """, (*identity, fingerprint, message["id"], self.idempotency_ttl)) + return message + + def chat_page(self, actor, limit, cursor): + return self.page(f"""SELECT {CHAT} FROM members m JOIN chats c ON c.id=m.chat_id + WHERE m.user_id=%s""", "c.", ("chats", actor), limit, cursor) + + def message_page(self, chat_id, limit, cursor): + return self.page(f"SELECT {MESSAGE} FROM messages WHERE chat_id=%s", + "", ("messages", chat_id), limit, cursor) + + def page(self, query, prefix, scope, limit, cursor): + columns = f"{prefix}created_at, {prefix}id" + params = [scope[1]] + if cursor is not None: + params.extend(self.decode_cursor(cursor, scope)) + query += f" AND ({columns}) > (%s,%s)" + with self.pool.connection() as db: + rows = db.execute(f"{query} ORDER BY {columns} LIMIT %s", (*params, limit + 1)).fetchall() + items = rows[:limit] + next_cursor = self.encode_cursor(scope, items[-1]) if len(rows) > limit else None + return {"items": items, "next_cursor": next_cursor} + + def encode_cursor(self, scope, row): + payload = json.dumps([scope, row["created_at"], row["id"]], + separators=(",", ":"), default=json_value).encode("ascii") + signature = hmac.digest(self.cursor_key, payload, "sha256") + return base64.urlsafe_b64encode(signature + payload).decode("ascii").rstrip("=") + + def decode_cursor(self, cursor, scope): + try: + if not 1 <= len(cursor) <= 256 or not cursor.isascii(): + raise ValueError + data = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + signature, payload = data[:32], data[32:] + if not hmac.compare_digest(signature, hmac.digest(self.cursor_key, payload, "sha256")): + raise ValueError + saved_scope, timestamp, record = json.loads(payload) + if saved_scope != [scope[0], str(scope[1])]: + raise ValueError + return datetime.fromisoformat(timestamp), UUID(record) + except ValueError: + raise invalid("Invalid cursor") from None diff --git a/src/validation.py b/src/validation.py new file mode 100644 index 00000000..66b881aa --- /dev/null +++ b/src/validation.py @@ -0,0 +1,43 @@ +import re +from uuid import UUID + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status = status + self.code = code + self.headers = headers or {} + + +def invalid(message): + return ApiError(400, "invalid_request", message) + + +def not_found(): + return ApiError(404, "not_found", "Resource not found") + + +def canonical_id(value): + if not isinstance(value, str) or not re.fullmatch( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value + ): + raise invalid("Invalid UUID") + return UUID(value) + + +def require_fields(body, *fields): + if not isinstance(body, dict) or body.keys() != set(fields): + raise invalid("Invalid request fields") + + +def require_text(value, maximum): + if ( + not isinstance(value, str) + or not value.strip() + or len(value) > maximum + or "\x00" in value + or re.search(r"[\ud800-\udfff]", value) + ): + raise invalid("Invalid text field") + return value diff --git a/student-tests/check_attachments.py b/student-tests/check_attachments.py new file mode 100644 index 00000000..c9988dc1 --- /dev/null +++ b/student-tests/check_attachments.py @@ -0,0 +1,153 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from random import Random +import struct +import sys +from threading import Barrier, Event, Thread +import time +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import urlopen +import uuid +import zlib + +from check_replicas import compose, wait_instances, REPLICAS +from client import Client, Failure, check + +ROOT = Path(__file__).resolve().parents[1] + + +def png(width, height, textured=False): + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + random = Random(2026) + scanlines = b"".join( + b"\x00" + (random.randbytes(width * 3) if textured else bytes(width * 3)) + for _ in range(min(height, 512)) + ) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(scanlines)) + + chunk(b"IEND", b"")) + + +def upload(client, owner, chat, data, expected=201, name="image.png", mime="image/png"): + path = f"/api/v1/chats/{chat['id']}/attachments?filename={quote(name, safe='')}" + return client.call("POST", path, expected, actor=owner, raw=data, + headers={"Content-Type": mime}, schema="Attachment" if expected == 201 else None) + + +def memory(): + return {name: int(compose("exec", "-T", name, "cat", "/sys/fs/cgroup/memory.current")) + for name in REPLICAS} + + +def run_concurrent(owner, chat, data): + ready = Barrier(10) + stop = Event() + baseline = memory() + peak = dict(baseline) + + def sample(): + while not stop.is_set(): + for name, value in memory().items(): + peak[name] = max(peak[name], value) + stop.wait(0.1) + + def send(_): + ready.wait(timeout=10) + return upload(Client(timeout=15), owner, chat, data)["json"]["id"] + + sampler = Thread(target=sample) + sampler.start() + try: + with ThreadPoolExecutor(max_workers=10) as pool: + ids = list(pool.map(send, range(10))) + finally: + stop.set() + sampler.join() + check(len(set(ids)) == 10, "uploads returned duplicate IDs") + print(f"PASS: 10 concurrent uploads of {len(data)} bytes; " + f"cgroup memory baseline={baseline}, sampled_peak={peak}", flush=True) + + +def main(): + compose("start", *REPLICAS) + wait_instances(set(REPLICAS)) + client = Client(timeout=6) + owner, member, outsider, chat = client.bootstrap() + data = png(512, 512) + sent = upload(client, owner, chat, data) + attachment = sent["json"] + serving = sent["headers"]["X-Instance-Id"] + path = f"/api/v1/attachments/{attachment['id']}" + upload(client, owner, chat, data, 400, name="../image.png") + upload(client, owner, chat, data, 415, mime="image/jpeg") + upload(client, owner, chat, b"broken image", 415) + upload(client, owner, chat, b"\0" * (10 * 1024**2 + 1), 413) + upload(client, owner, chat, png(10000, 4001), 413) + client.call("GET", path, 404, actor=outsider) + client.call("GET", path + "/content", 404, actor=outsider) + other = client.call("POST", "/api/v1/chats", 201, actor=owner, body={ + "title": "Other chat", "member_ids": [owner["id"]], + }, schema="Chat")["json"] + wrong_path = f"/api/v1/chats/{other['id']}/messages" + client.call("POST", wrong_path, 404, actor=owner, + body={"text": "", "attachment_ids": [attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + print("PASS: size, pixels, malformed image, MIME, filename and cross-chat access", flush=True) + + compose("stop", serving) + try: + survivor = next(name for name in REPLICAS if name != serving) + deadline = time.monotonic() + 10 + while True: + try: + result = client.call("GET", path + "/content", actor=member) + break + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(0.2) + check(result["bytes"] == data and result["headers"]["X-Instance-Id"] == survivor, + "original unavailable on surviving replica") + finally: + compose("start", serving) + wait_instances(set(REPLICAS)) + print(f"PASS: original bytes read through {survivor} after {serving} stopped", flush=True) + + compose("exec", "-T", "api-a", "python", "-c", + "import os,urllib.error,urllib.request,sys;" + "url=os.environ['S3_ENDPOINT']+'/'+os.environ['S3_BUCKET']+'/'+sys.argv[1];" + "exec('try:\\n urllib.request.urlopen(url,timeout=3)\\nexcept urllib.error.HTTPError as e:" + "\\n assert e.code == 403\\nelse:\\n raise AssertionError()')", + f"attachments/{attachment['id']}") + print("PASS: anonymous direct access to S3 object returned 403", flush=True) + + run_concurrent(owner, chat, png(512, 512, textured=True)) + compose("stop", "s3") + try: + status = None + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + with urlopen(client.base + "/health/ready", timeout=2): + pass + except HTTPError as error: + if error.code == 503: + status = error.code + break + except OSError: + pass + time.sleep(0.1) + check(status == 503, "API did not report S3 failure") + finally: + compose("start", "s3") + wait_instances(set(REPLICAS)) + check(client.call("GET", path + "/content", actor=owner)["bytes"] == data, + "original lost after restarting S3") + print("PASS: readiness returned 503 during S3 outage, original survived restart") + + +if __name__ == "__main__": + main() diff --git a/student-tests/check_replicas.py b/student-tests/check_replicas.py new file mode 100644 index 00000000..f5941001 --- /dev/null +++ b/student-tests/check_replicas.py @@ -0,0 +1,118 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import subprocess +import sys +from threading import Barrier, Event +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, Failure, check + +REPLICAS = ("api-a", "api-b") + + +def compose(*args): + return subprocess.check_output(["docker", "compose", *args], text=True, timeout=45).strip() + + +def wait_instances(expected): + client = Client(timeout=1) + deadline = time.monotonic() + 25 + seen = set() + while time.monotonic() < deadline: + try: + instance = client.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"] + seen.add(instance) + if seen == expected: + return + except Failure: + pass + time.sleep(0.1) + raise AssertionError(f"Expected {expected}, observed {seen}") + + +def parallel_replay(actor, chat): + start = Barrier(8) + key = uuid.uuid4().hex + + def write(_): + start.wait(timeout=10) + return Client().message(actor, chat, "Concurrent replay", key=key) + + with ThreadPoolExecutor(max_workers=8) as workers: + results = list(workers.map(write, range(8))) + ids = {item["json"]["id"] for item in results} + instances = {item["headers"]["X-Instance-Id"] for item in results} + check(len(ids) == 1 and instances == set(REPLICAS), "parallel replay across replicas failed") + path = f"/api/v1/chats/{chat['id']}/messages" + page = Client().call("GET", path, actor=actor, schema="MessagePage")["json"] + check(sum(item["id"] in ids for item in page["items"]) == 1, "duplicate in history") + print("PASS: 8 concurrent replays through both replicas, 1 message", flush=True) + + +def exercise(actor, chat, target, recreate): + stop = Event() + results = [] + client = Client(timeout=2) + started = time.monotonic() + + def traffic(): + key = uuid.uuid4().hex + while not stop.is_set(): + try: + response = client.message(actor, chat, "Write " + key, key=key) + results.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"])) + key = uuid.uuid4().hex + except Failure: + results.append((time.monotonic(), None, None)) + stop.wait(0.05) + + old_id = compose("ps", "-q", target) + with ThreadPoolExecutor(max_workers=1) as workers: + worker = workers.submit(traffic) + try: + time.sleep(0.5) + changed = time.monotonic() + if recreate: + compose("up", "-d", "--no-deps", "--force-recreate", target) + check(old_id != compose("ps", "-q", target), "container was not replaced") + wait_instances(set(REPLICAS)) + else: + compose("stop", "--timeout", "5", target) + check(not compose("ps", "-q", "--status", "running", target), "replica is still running") + time.sleep(2.5) + finally: + stop.set() + worker.result(timeout=5) + if not recreate: + compose("start", target) + wait_instances(set(REPLICAS)) + + survivor = next(name for name in REPLICAS if name != target) + successful = [row for row in results if row[1] is not None] + first = next(row[0] for row in successful if row[0] >= changed and row[1] == survivor) + check(first - changed <= 10, "surviving replica took too long") + path = f"/api/v1/chats/{chat['id']}/messages?limit=100" + observed = set() + while path is not None: + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + observed.update(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={cursor}" if cursor else None + check({row[2] for row in successful} <= observed, "confirmed writes lost") + action = "recreate" if recreate else "stop" + elapsed = results[-1][0] - started + print(f"PASS: {action} {target}, {len(successful) / elapsed:.1f} write/s, " + f"errors={len(results) - len(successful)}, survivor={first - changed:.3f}s", flush=True) + + +if __name__ == "__main__": + compose("start", *REPLICAS) + wait_instances(set(REPLICAS)) + owner, _, _, chat = Client().bootstrap() + parallel_replay(owner, chat) + for name in REPLICAS: + exercise(owner, chat, name, recreate=False) + for name in REPLICAS: + exercise(owner, chat, name, recreate=True)