From a97c35fc12217715d77dd3dec3366ff209a59973 Mon Sep 17 00:00:00 2001 From: Ilya <145450507+IIcon18@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:00:45 +0300 Subject: [PATCH] =?UTF-8?q?=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=9B=D0=A04=20=D0=BD=D0=B0=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket, отдельные ограниченные ключи API, служебные порты закрыты; идемпотентный s3-init с проверкой приватности и прогревом записи - вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс, тип по содержимому, проверка filename, скачивание только участникам - согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper с grace и dry-run; attachment_ids в сообщениях, FK по чату - back-pressure на S3 PUT после найденного OOM SeaweedFS - тесты student-tests, evidence, REPORT.md, DEMO.md --- DEMO.md | 158 ++++++++++ Dockerfile | 23 ++ Makefile | 46 ++- REPORT.md | 146 ++++++++- app/__init__.py | 0 app/auth.py | 113 +++++++ app/cache.py | 125 ++++++++ app/config.py | 69 +++++ app/errors.py | 31 ++ app/images.py | 84 +++++ app/main.py | 374 +++++++++++++++++++++++ app/migrate.py | 63 ++++ app/models.py | 98 ++++++ app/objects.py | 151 +++++++++ app/s3_bootstrap.py | 76 +++++ app/storage.py | 407 +++++++++++++++++++++++++ app/sweeper.py | 130 ++++++++ compose.yaml | 242 +++++++++++++++ evidence/attachments.txt | 50 +++ evidence/bootstrap-idempotent.txt | 6 + evidence/consistency.txt | 10 + evidence/failover-load/stop-api-1.json | 153 ++++++++++ evidence/failover-load/stop-api-2.json | 153 ++++++++++ evidence/failover.txt | 14 + evidence/idempotency.txt | 13 + evidence/load.json | 17 ++ evidence/memory/ram-10-noise.json | 49 +++ evidence/memory/ram-20-noise.json | 49 +++ evidence/memory/ram-40-noise.json | 49 +++ evidence/memory/stream-10-noise.json | 49 +++ evidence/memory/stream-10-pixels.json | 49 +++ evidence/memory/stream-20-noise.json | 49 +++ evidence/memory/stream-40-noise.json | 49 +++ evidence/persistence.txt | 31 ++ evidence/s3-oom-before-fix.txt | 23 ++ evidence/s3-security.txt | 18 ++ evidence/smoke.txt | 3 + evidence/stop-one.txt | 4 + evidence/topology.txt | 45 +++ evidence/upload-failover.txt | 7 + infra/haproxy/haproxy.cfg | 52 ++++ migrations/0001_init.sql | 49 +++ migrations/0002_idempotency.sql | 12 + migrations/0003_attachments.sql | 49 +++ requirements.txt | 8 + scripts/init-secrets.sh | 47 +++ scripts/restart.sh | 8 + scripts/restore.sh | 5 + scripts/rolling-update.sh | 31 ++ scripts/stop-container.sh | 9 + scripts/stop-one.sh | 19 ++ student-tests/attachments_test.py | 154 ++++++++++ student-tests/chaos_load.py | 147 +++++++++ student-tests/consistency_test.py | 152 +++++++++ student-tests/fixtures/sample.jpg | Bin 0 -> 3973 bytes student-tests/idempotency_test.py | 106 +++++++ student-tests/s3_security_test.py | 82 +++++ student-tests/stack.py | 61 ++++ student-tests/upload_failover.py | 62 ++++ student-tests/upload_memory.py | 107 +++++++ 60 files changed, 4369 insertions(+), 17 deletions(-) create mode 100644 DEMO.md create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/auth.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/errors.py create mode 100644 app/images.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/models.py create mode 100644 app/objects.py create mode 100644 app/s3_bootstrap.py create mode 100644 app/storage.py create mode 100644 app/sweeper.py create mode 100644 compose.yaml create mode 100644 evidence/attachments.txt create mode 100644 evidence/bootstrap-idempotent.txt create mode 100644 evidence/consistency.txt create mode 100644 evidence/failover-load/stop-api-1.json create mode 100644 evidence/failover-load/stop-api-2.json create mode 100644 evidence/failover.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/load.json create mode 100644 evidence/memory/ram-10-noise.json create mode 100644 evidence/memory/ram-20-noise.json create mode 100644 evidence/memory/ram-40-noise.json create mode 100644 evidence/memory/stream-10-noise.json create mode 100644 evidence/memory/stream-10-pixels.json create mode 100644 evidence/memory/stream-20-noise.json create mode 100644 evidence/memory/stream-40-noise.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/s3-oom-before-fix.txt create mode 100644 evidence/s3-security.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stop-one.txt create mode 100644 evidence/topology.txt create mode 100644 evidence/upload-failover.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 requirements.txt create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-container.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/attachments_test.py create mode 100644 student-tests/chaos_load.py create mode 100644 student-tests/consistency_test.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/idempotency_test.py create mode 100644 student-tests/s3_security_test.py create mode 100644 student-tests/stack.py create mode 100644 student-tests/upload_failover.py create mode 100644 student-tests/upload_memory.py diff --git a/DEMO.md b/DEMO.md new file mode 100644 index 00000000..a4e49be1 --- /dev/null +++ b/DEMO.md @@ -0,0 +1,158 @@ +# Демонстрация лабораторной 4 на защите + +Сценарий повторяет «Сценарий защиты» из README: загрузка и скачивание вторым участником со сравнением SHA-256, 404 постороннему и отказ анонимного S3, persistence/failover, отсутствие локального файла на API, затем отказы и эксперименты уровня 4. + +**Важно:** команды выполняются по порядку **в одном окне терминала** — переменные `B`, `TA`, `TB`, `TE`, `CHAT`, `ATT` и функции `j`, `pg`, `mkuser` живут только в нём. + +--- + +## 0. Подготовка: чистый clone + +```bash +(cd ~/Desktop/вуз/ВНП/lab4/mtusi2026lab4 && docker compose stop) +``` + +```bash +git clone -b solution/bvt2304-koshelev ssh://git@gitea.privcloud.tech:2222/Iicon18/mtusi2026lab4.git ~/Desktop/lab4-demo +``` + +```bash +cd ~/Desktop/lab4-demo && export COMPOSE_PROJECT_NAME=lab4-demo && cp .env.example .env +``` + +Если 8080 занят, поменять `APP_PORT` в `.env` (у меня 8088): + +```bash +B=http://127.0.0.1:$(sed -n 's/^APP_PORT=//p' .env); echo "$B" +``` + +```bash +j() { python3 -c 'import sys,json;print(json.load(sys.stdin)[sys.argv[1]])' "$1"; } +pg() { docker compose exec -T postgres psql -U messenger -d messenger -c "$1"; } +``` + +> **Сказать:** «Свежий clone в отдельный compose-проект: новые пустые volumes `pgdata` и `s3data`. Секреты и `.env` не в Git». + +## 1. Секреты и запуск + +```bash +make secrets && ls secrets/ +``` + +> «Две пары S3-ключей: `s3_admin_*` нужен только one-shot сервису `s3-init` для создания bucket, `s3_access_key_*` — для API с правами Read/Write/List только на один bucket. `seaweedfs_s3.json` — конфигурация identities для SeaweedFS». + +```bash +docker compose --env-file .env config --quiet && docker compose --env-file .env up -d --build --wait +``` + +```bash +docker compose ps -a --format 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' +``` + +> «Порт опубликован только у `lb`. У `s3` портов на хосте нет. `migrate` и `s3-init` — one-shot, завершились с кодом 0». + +```bash +docker compose logs --no-log-prefix migrate s3-init +``` + +> «Bucket создан, анонимный доступ проверен и запрещён, запись прогрета. API стартует только после успеха `s3-init`». + +## 2. Публичный smoke + +```bash +make test BASE_URL=$B +``` + +## 3. Загрузка, сообщение, скачивание вторым участником, SHA-256 + +```bash +mkuser() { curl -s -X POST $B/api/v1/users -H 'Content-Type: application/json' -d "{\"username\":\"$1_$RANDOM\",\"display_name\":\"$1\",\"password\":\"Demo-password-123\"}"; } +login() { curl -s -X POST $B/api/v1/auth/sessions -u "$1:Demo-password-123" | j token; } +UA=$(mkuser alice); UB=$(mkuser bob); UE=$(mkuser eve) +A=$(echo "$UA" | j id); Bo=$(echo "$UB" | j id) +TA=$(login $(echo "$UA" | j username)); TB=$(login $(echo "$UB" | j username)); TE=$(login $(echo "$UE" | j username)) +CHAT=$(curl -s -X POST $B/api/v1/chats -H "Authorization: Bearer $TA" -H 'Content-Type: application/json' -d "{\"title\":\"Демо\",\"member_ids\":[\"$A\",\"$Bo\"]}" | j id); echo "$CHAT" +``` + +```bash +curl -s -i -X POST "$B/api/v1/chats/$CHAT/attachments?filename=sample.png" -H "Authorization: Bearer $TA" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png | tee /tmp/att.txt | sed -n '1p;/^X-Instance-Id/p;$p' +ATT=$(tail -1 /tmp/att.txt | j id) +``` + +> «201, `status=ready`, `variants=[]`. 201 возвращается только после того, как S3 подтвердил PUT (и сам сверил SHA-256) и в PostgreSQL зафиксирован `ready`». + +```bash +curl -s -X POST $B/api/v1/chats/$CHAT/messages -H "Authorization: Bearer $TA" -H "Idempotency-Key: demo-$RANDOM" -H 'Content-Type: application/json' -d "{\"text\":\"\",\"attachment_ids\":[\"$ATT\"]}" +``` + +```bash +curl -s $B/api/v1/attachments/$ATT/content -H "Authorization: Bearer $TB" | shasum -a 256; shasum -a 256 tests/fixtures/sample.png +``` + +## 4. Посторонний и прямой доступ к S3 + +```bash +for s in "" /content; do curl -s -o /dev/null -w "eve $s -> %{http_code}\n" $B/api/v1/attachments/$ATT$s -H "Authorization: Bearer $TE"; done +``` + +```bash +make test-s3-security BASE_URL=$B +``` + +> «Анонимный GET объекта по настоящему ключу изнутри сети — 403. Ключами API нельзя создать bucket, повесить публичную policy или ACL. Служебные порты SeaweedFS (master/filer/volume) слушают только loopback. Но случайный ключ и приватный bucket не заменяют ACL приложения: API всё равно проверяет членство в чате при каждом скачивании». + +```bash +pg "SELECT id, filename, object_key, status FROM attachments WHERE id='$ATT'" +``` + +> «`object_key` — случайный, его генерирует сервер; `filename` только для отображения». + +## 5. На API нет локального оригинала; failover и persistence + +```bash +docker compose config --format json | python3 -c 'import json,sys; print("api volumes:", json.load(sys.stdin)["services"]["api"].get("volumes"))' +``` + +```bash +make test-upload-failover BASE_URL=$B +``` + +> «Каждый раунд останавливает именно ту реплику, которая приняла файл, и скачивает через другую». + +```bash +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$B; ./scripts/restore.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$B +``` + +## 6. Уровень 4: ошибки, отказы, память + +```bash +make test-attachments BASE_URL=$B | tail -25 +``` + +> «Большой файл 413 (и с Content-Length, и chunked), пиксельная бомба 413 до декодирования, повреждённые и подменённые файлы 415, чужое вложение 404, опасные имена 400; ни одного 500». + +```bash +make test-consistency BASE_URL=$B +``` + +> «S3 остановлен — upload 503, сообщения и readiness работают. Обрыв загрузки не оставляет ничего. Sweeper: dry-run только отчитывается, реальный проход удаляет внедрённые старые pending/orphan, свежие и ready не трогает, повтор — пустой». + +```bash +make memory BASE_URL=$B | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["statuses"], {k: v["delta_anon"] for k, v in d["memory_mib"].items()})' +``` + +> «10 одновременных загрузок по 9.6 MiB: прирост памяти API ~35 MiB и почти не меняется при 20 и 40 — тело идёт потоком в spool. Таблица и найденный OOM SeaweedFS — в REPORT.md». + +## Ответы на вопросы + +- **Файл в volume API ≠ объектное хранилище:** volume привязан к одному контейнеру — другая реплика его не видит, при пересоздании контейнера он теряется. S3 — общий сервис со своим volume и API для всех реплик. +- **Когда честно вернуть 201:** после подтверждённого S3 PUT с проверкой checksum **и** COMMIT `ready` в PostgreSQL. +- **S3 PUT прошёл, COMMIT нет:** pending-строка + объект; API пытается удалить объект сразу, а sweeper через grace удаляет и строку, и объект. Клиент получил 503, а не 201. +- **Почему случайный ключ и приватный bucket не заменяют ACL:** ключ может утечь (логи, подписанные URL), а участники чата меняются по смыслу приложения — право читать определяется членством в чате, которое знает только API. + +## Уборка + +```bash +docker compose down -v && cd ~ && rm -rf ~/Desktop/lab4-demo +``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b8408c2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.8-slim-bookworm + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations + +RUN useradd --system --no-create-home --uid 10001 appuser +USER appuser + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:'+os.environ.get('APP_PORT','8080')+'/health/ready',timeout=2).status==200 else 1)" + +CMD ["sh", "-c", "exec uvicorn app.main:app --host ${APP_HOST} --port ${APP_PORT} --timeout-graceful-shutdown 8 --timeout-keep-alive 30"] diff --git a/Makefile b/Makefile index 5d86d171..832f49bc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover secrets up down restore rolling-update test-student test-attachments test-upload-failover test-consistency test-s3-security sweep-dry-run memory chaos help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,47 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +secrets: + sh scripts/init-secrets.sh + +up: secrets + docker compose up -d --build --wait + +down: + docker compose down + +restore: + sh scripts/restore.sh + +rolling-update: + sh scripts/rolling-update.sh + +# Own tests (student-tests/); they use the running compose stack, some stop/start containers. +test-attachments: + $(PYTHON) student-tests/attachments_test.py --base-url "$(BASE_URL)" + +test-upload-failover: + $(PYTHON) student-tests/upload_failover.py --base-url "$(BASE_URL)" + +test-consistency: + $(PYTHON) student-tests/consistency_test.py --base-url "$(BASE_URL)" + +test-s3-security: + $(PYTHON) student-tests/s3_security_test.py --base-url "$(BASE_URL)" + +test-student: test-attachments test-s3-security test-upload-failover test-consistency + $(PYTHON) student-tests/idempotency_test.py --base-url "$(BASE_URL)" + +sweep-dry-run: + docker compose run --rm --no-deps -T sweeper python -m app.sweeper --once --dry-run + +memory: + $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" --concurrency 10 + +chaos: + @for n in 1 2; do \ + name=$$(docker compose ps --format '{{.Name}}' api | sed -n "$${n}p"); \ + $(PYTHON) student-tests/chaos_load.py --base-url "$(BASE_URL)" --action "scripts/stop-container.sh $$name" --output evidence/failover-load/stop-api-$$n.json | $(PYTHON) -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps({k: d[k] for k in ("action_output","requests","errors","successful_rps","latency_ms_success","recovery_s_after_action","integrity")},ensure_ascii=False,indent=1))'; \ + sh scripts/restore.sh >/dev/null 2>&1; sleep 3; \ + done diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..acb4cd8f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,151 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кошелев, БВТ2304 +- Целевая оценка: **4** (трек 5 не заявлен) +- Commit/PR: ветка `solution/bvt2304-koshelev`, PR в `mtusi2026/mtusi2026lab4` (main) +- Использованные источники, библиотеки и помощь ИИ: FastAPI, uvicorn, pydantic, asyncpg, redis-py, argon2-cffi (из lab3); **boto3** (S3-клиент), **Pillow** (проверка изображений); SeaweedFS 4.51 (документация S3 API, `-s3.config` identities/actions, `concurrentUploadLimitMB`); AWS S3 docs (PutObject, `x-amz-checksum-sha256`); Go runtime `GOMEMLIMIT`. Код перенесён из собственной lab3. При доработке, тестах и отчёте использовался ИИ-ассистент Claude (Anthropic); решения проверены и объяснимы автором. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Engine + Compose v2 (проверено: Docker Desktop 28.0.1, Compose 2.33.1), Python 3.9+ для тестов (проверено 3.14), `make`. Стенд: Apple M3, 8 CPU, 16 GiB, Docker VM 8 CPU / 7.6 GiB. + +| Роль | Service | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | `haproxy:3.2.4-alpine` | `monitor-uri /lb-health` | — | `edge`; **единственный опубликованный порт** `127.0.0.1:${APP_PORT:-8080}` | нет | +| API × N | `api` (`replicas: ${API_REPLICAS:-2}`) | `mtusi-messenger:lab4` (сборка из `Dockerfile`, `python:3.12.8-slim-bookworm`) | `/health/ready` (PostgreSQL) | **нет** (временный spool загрузки удаляется в конце запроса) | `edge` + `data`, 8080 не публикуется | secrets `database_url`, `cache_url`, `s3_access_key_id`, `s3_secret_access_key` | +| S3 | `s3` | `chrislusf/seaweedfs:4.51` | `/healthz` S3 **и** зарегистрированный volume-сервер в топологии master | `s3data:/data` | `data` (internal), S3 на :9000; master/filer/volume слушают только `127.0.0.1` внутри контейнера | secret `seaweedfs_s3_config` (identities) | +| Bootstrap bucket | `s3-init` (one-shot) | `mtusi-messenger:lab4` | exit 0 | — | `data` | secrets `s3_admin_*` (только здесь) | +| Миграции | `migrate` (one-shot) | `mtusi-messenger:lab4` | exit 0 | — | `data` | `database_url` | +| Очистка | `sweeper` | `mtusi-messenger:lab4` | heartbeat-файл не старше 3 интервалов | — | `data` | `database_url`, API-ключи S3 | +| БД (источник истины) | `postgres` | `postgres:17.6-alpine3.22` | `pg_isready` | `pgdata` | `data` | `postgres_password` | +| Кэш сессий | `valkey` | `valkey/valkey:8.1.4-alpine3.22` | `valkey-cli ping` | нет | `data` | `valkey_conf` | + +**Почему SeaweedFS, а не MinIO.** Образы `minio/minio` и `quay.io/minio/minio` в Docker Hub/Quay на момент работы недоступны (`pull access denied` / manifest not found), поэтому закрепить проверенную версию MinIO нельзя. SeaweedFS 4.51 даёт: статическую конфигурацию S3-identities с правами на конкретный bucket (ограниченные credentials API), запрет анонимного доступа по умолчанию и возможность убрать с сети все служебные HTTP-интерфейсы (`-ip.bind=127.0.0.1`, наружу только `-s3.ip.bind=0.0.0.0`). + +```sh +./scripts/init-secrets.sh # ./secrets/* (в .gitignore): пароли PG/Valkey, 2 пары S3-ключей, seaweedfs_s3.json +cp .env.example .env # при занятом 8080 поменяйте APP_PORT (у автора 8088) +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# postgres, valkey, s3 -> s3-init (bucket + проверка приватности + прогрев записи) и migrate -> api x2, sweeper -> lb +make test # BASE_URL=http://localhost:8088 при другом порту +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh ; ./scripts/restore.sh +make test-student # attachments, s3-security, upload-failover, consistency (останавливает/запускает s3), idempotency +make memory # эксперимент с параллельными загрузками +make chaos # из lab3: остановка каждой реплики под нагрузкой +``` + +- Короткие цели: `make up` (= secrets + `up -d --build --wait`), `make restore`, `make rolling-update`, `make sweep-dry-run`, `make down`. Сценарий защиты — [DEMO.md](DEMO.md). +- Bootstrap и миграции **идемпотентны** и выполняются при каждом `up`: `docker compose run --rm --no-deps s3-init` → «bucket already exists», `migrate` → «0 applied now» (`evidence/bootstrap-idempotent.txt`). +- Безопасная остановка: `docker compose stop` или `down` (volumes сохраняются). **Намеренное удаление учебных данных:** `docker compose down -v` (удаляет `pgdata` и `s3data`). + +Соответствие `.env.example`: `S3_ENDPOINT=http://s3:9000` (порт S3-шлюза SeaweedFS выбран 9000 под пример), `S3_REGION`, `S3_BUCKET` → API/sweeper/s3-init; `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` → Docker secrets; `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000` → API. Остальные переменные — как в lab3. Дополнительно (значения по умолчанию в `compose.yaml`): `UPLOAD_CONCURRENCY=16`, `DECODE_CONCURRENCY=1`, `S3_PUT_CONCURRENCY=4`, `UPLOAD_SPOOL_BYTES=1048576`, `ORPHAN_GRACE_SECONDS=900`, `SWEEP_INTERVAL_SECONDS=60`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client ─► lb (HAProxy :8080) ─round-robin─► api-1 ┐ ┌─► postgres: users, chats, messages, sessions, + └─────────► api-2 ┼── сеть data ────┤ idempotency_keys, attachments (метаданные), + сеть edge │ (internal) │ message_attachments + sweeper ──────┤ ├─► valkey: кэш сессий + s3-init ──────┘ └─► s3 (SeaweedFS :9000): приватный bucket, только байты +``` + +**Из lab3 перенесено без изменений:** балансировка, сессии в PostgreSQL + кэш Valkey, идемпотентность сообщений, `X-Instance-Id`, graceful shutdown, hooks `restart/stop-one/restore/rolling-update`, `chaos_load.py`, `idempotency_test.py`. + +**Новое в lab4:** + +1. **Модель данных** (`migrations/0003_attachments.sql`). `attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, …)` — только метаданные. `object_key = originals/<случайный 128-битный hex>` назначает сервер; он не связан ни с `id`, ни с `filename` и наружу не отдаётся. `filename` — только отображаемое имя. `message_attachments(message_id, attachment_id, chat_id, position)` с **составными FK** `(message_id, chat_id) → messages` и `(attachment_id, chat_id) → attachments`: даже в обход API БД не позволит привязать файл другого чата. Ограничение длины текста сообщения ослаблено до `≤ 4000` (пустой текст допустим только с вложениями — проверяет API). Миграция аддитивна, старые сообщения получают `attachment_ids=[]`. +2. **Загрузка** `POST /chats/{id}/attachments?filename=` (`app/main.py`, `app/images.py`). Порядок: аутентификация → членство (иначе 404) → `filename` (1–255, без `/ \`, NUL, управляющих и bidi-символов, не `.`/`..`; иначе 400) → `Content-Type` ∈ {png, jpeg} (иначе 415) → `Content-Length > 10 MiB` → 413 **до чтения тела**. Тело читается **потоком** в `SpooledTemporaryFile` (в RAM ≤ 1 MiB на загрузку, дальше — временный файл) с подсчётом размера и SHA-256; превышение лимита во время потока (chunked без Content-Length) → 413. Тип определяется **по сигнатуре** и должен совпасть с заявленным (иначе 415). Затем Pillow: размеры читаются из заголовка, `w×h > 40 000 000` → 413 **до декодирования**; потом PNG `verify()` (CRC всех чанков) и полное декодирование (JPEG — с DCT-масштабированием 1/8, весь поток всё равно проверяется); ошибка → 415. Декодирование выполняется в пуле потоков под семафором `DECODE_CONCURRENCY=1` — это единственный шаг, чья память растёт с числом пикселей. +3. **Когда честно вернуть 201.** Только после того как (а) S3 подтвердил PUT, причём S3 сам сверил `x-amz-checksum-sha256` с нашей суммой, и (б) в PostgreSQL зафиксирован `status='ready'`. Порядок записи: + 1. `INSERT attachments … status='pending'` (намерение; pending не видно через API — для всех запросов это 404); + 2. `PUT` объекта в S3 (под семафором `S3_PUT_CONCURRENCY`); + 3. `UPDATE … SET status='ready' WHERE status='pending' RETURNING …`. +4. **Что если S3 PUT прошёл, а COMMIT — нет** (и другие сбои): + | Сбой | Что остаётся | Кто чинит | + | --- | --- | --- | + | Клиент оборвал тело / таймаут 60 с | ничего (S3 не трогали, строки нет) | — | + | S3 недоступен при PUT | компенсация: delete объекта + delete pending-строки, ответ 503 | если компенсация не дошла — sweeper | + | PUT прошёл, `UPDATE ready` упал (БД) | компенсирующий delete объекта; pending-строка | sweeper удалит строку (и объект, если delete не дошёл) | + | Процесс API убит между шагами | pending-строка (± объект) | sweeper | + | Объект без строки (строка удалена, delete не дошёл) | orphan в S3 | sweeper (сверка листинга `originals/` с БД) | + + **Sweeper** (`app/sweeper.py`, сервис `sweeper`, раз в 60 с, `pg_try_advisory_lock` против параллельных запусков): удаляет pending-строки и их объекты **старше grace = 900 с**, затем перечисляет `originals/` и удаляет объекты старше grace, на которые нет строки. Grace на порядки больше максимальной длительности живой загрузки (таймаут тела 60 с + S3 таймауты 2+5 с), поэтому незавершённая, но живая загрузка не может быть удалена; `ready`-строки и объекты, на которые ссылается строка, не удаляются никогда. Шаги идемпотентны (сначала объект, потом строка), есть `--dry-run`, минимальный grace защищён (`< 60 с` запрещено). +5. **Скачивание** `GET /attachments/{id}` и `/content`: метаданные и байты отдаются только участнику чата (JOIN с `chat_members`), иначе 404; **без redirect**, байты потоково проксируются из S3 кусками по 64 KiB (`StreamingResponse`), `Content-Type` — истинный тип, `Content-Length`, `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`. `variant=original` по умолчанию; `thumbnail/crop/optimized` в lab4 → 404, неизвестное имя → 400. S3 недоступен → 503. +6. **Сообщения с `attachment_ids`**: 0–10 уникальных UUID, по умолчанию `[]`; `text=""` разрешён при ≥1 вложении. Внутри той же транзакции, что и идемпотентный ключ, проверяется, что все id существуют, `ready` и принадлежат **этому** чату; иначе 404 (ключ не фиксируется). Порядок вложений сохраняется. Отпечаток идемпотентности считается от канонического JSON после применения default, поэтому отсутствующий `attachment_ids` ≡ `[]`. +7. **Readiness = только PostgreSQL.** S3 критичен лишь для загрузки/скачивания файлов; если включить его в readiness, отказ S3 вывел бы из LB все реплики и остановил бы ещё и текстовые сообщения. Поэтому при недоступном S3: `/health/ready` 200, сообщения 201, метаданные 200, upload/content — быстрый 503 (`evidence/consistency.txt`). +8. **S3: приватность и минимальные права.** `init-secrets.sh` генерирует две identity: `bootstrap-admin` (только для `s3-init`: создание bucket) и `messenger-api` с действиями `Read/Write/List` **только на `messenger-attachments`**. API этими ключами не может создать/удалить bucket, выставить публичную policy или ACL (`AccessDenied`). Анонимные GET/LIST → 403. Порты master 9333, filer 8888, volume 8080 изнутри сети недоступны (слушают loopback), портов на хосте у `s3` нет, `lb` (сеть `edge`) до S3 не достаёт (`evidence/s3-security.txt`). +9. **Bootstrap** (`app/s3_bootstrap.py`): создание bucket идемпотентно (`BucketAlreadyOwnedByYou` = успех), затем проверка, что анонимный доступ запрещён (иначе exit 1 и API не стартует), затем **прогрев записи**: put/get/delete пробного объекта `health/…`, пока операция не займёт < 0.5 с. Без прогрева первый PUT после рестарта SeaweedFS ждал 2.5–5 с, и публичный failover-тест с таймаутом клиента 2 с падал сразу после persistence-теста. +10. **Ограничение памяти и back-pressure:** `UPLOAD_CONCURRENCY=16` одновременных загрузок на реплику (остальные ждут, их тело остаётся в сокете), spool 1 MiB, `DECODE_CONCURRENCY=1`, `S3_PUT_CONCURRENCY=4`. У SeaweedFS: `-s3/-filer/-volume.concurrentUploadLimitMB=128`, `GOMEMLIMIT=450MiB`, лимит 768M. Зачем — см. эксперимент ниже (OOM S3). +11. **LB:** `timeout server` увеличен с 10 до 30 с (это таймаут бездействия, а декодирование 40 Мпикс занимает до ~1 с; запас для медленного S3). + +**Транзакционные границы:** pending-строка, S3 PUT и `ready` — три отдельных шага (распределённой транзакции между PostgreSQL и S3 нет; согласованность обеспечивает порядок записи + компенсация + sweeper). Сообщение + ключ идемпотентности + ссылки на вложения — одна транзакция PostgreSQL. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make test` | PASS lab4 (upload 201 ready, message с `attachment_ids`, download bob, SHA-256, 404 eve) | `evidence/smoke.txt` | +| (3) Возможности lab3 сохранены, S3 в общем стенде, метаданные в PG, байты в приватном bucket | `docker compose ps`, `make test-failover`, `chaos`, `idempotency_test.py` | 2 API, LB, PG, Valkey, SeaweedFS; failover/идемпотентность PASS; `pgdata` + `s3data` | `evidence/topology.txt`, `failover.txt`, `idempotency.txt`, `failover-load/*.json` | +| (3) 201 ready после сохранения, побайтное скачивание, чужой не видит | `make test-attachments` | png+jpeg byte-identical, metadata стабильны, outsider 404 на metadata/content/upload | `evidence/attachments.txt` | +| (3) Реальные PNG/JPEG ≤10 MiB, ≤40 Мпикс, MIME по содержимому, опасные filename, лимиты до неограниченного чтения | `make test-attachments` | ровно 40 000 000 px → 201; 8000×5001 → 413; заголовок 20000×20000 → 413 без декодирования; 10 MiB+1 → 413 (и с Content-Length, и chunked) | `evidence/attachments.txt` | +| (3) Upload → остановка **обслужившей** реплики → download через другую | `make test-upload-failover` | 2 раунда: остановлены api-1, затем api-2 — каждая сразу после того, как сама приняла файл; 10/10 скачиваний через другую реплику, SHA-256 совпал | `evidence/upload-failover.txt`, `evidence/stop-one.txt` | +| (3) Restart всего стенда с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS (тест сверяет и файл); StartedAt сменился у api, lb, sweeper, postgres, valkey, **s3**; volumes те же | `evidence/persistence.txt` | +| (3) На API нет локального файла оригинала | `docker compose ps` / `compose.yaml` | у `api` нет volumes; после остановки загрузившей реплики файл отдаёт другая | `evidence/topology.txt`, `upload-failover.txt` | +| (4) Слишком большой файл, повреждённое изображение, подмена MIME, чужой attachment_id, файл другого чата; ошибки не 500 | `make test-attachments` (47 случаев) | 413 / 415 (обрезанный PNG/JPEG, повреждённый IDAT, PNG как jpeg, JPEG как png, текст, GIF, SVG, пустое тело, octet-stream, multipart) / 404 (чужой чат, чужой/неизвестный id в сообщении) / 400; ни одного 500; составной FK отверг привязку через psql | `evidence/attachments.txt` | +| (4) Прерывание загрузки и отказ между S3/БД без вечных объектов | `make test-consistency` | обрыв на 1.1 из 5 MiB — ни объекта, ни строки; S3 down — 503, pending не остаётся; внедрены: orphan-объект, pending+объект, pending без объекта (старше grace) и «свежие» — dry-run нашёл 2+1 и ничего не тронул, реальный проход удалил 2+1, свежие и ready целы, повтор — 0 | `evidence/consistency.txt` | +| (4) Недоступность S3 | `make test-consistency` | upload 503 за 0.8 с, content 503, metadata 200, сообщения 201, ready 200; после старта S3 — файл цел | `evidence/consistency.txt` | +| (4) Потоковая обработка, лимиты; ≥10 одновременных загрузок с пиком памяти | `student-tests/upload_memory.py` | см. таблицу ниже: прирост памяти API не зависит от числа параллельных загрузок | `evidence/memory/*.json` | +| (4) Идемпотентный bucket bootstrap | повторный `s3-init` и `migrate` | «already exists», «0 applied now» | `evidence/bootstrap-idempotent.txt` | +| (4) Отдельные ограниченные S3 credentials; консоль не открыта; проверка private bucket вне API | `make test-s3-security` | anonymous GET/LIST 403; API: CreateBucket/PutBucketPolicy/PutObjectAcl/DeleteBucket → AccessDenied; 9333/8888/8080 refused; host-портов нет; `lb` до S3 не достаёт | `evidence/s3-security.txt` | +| Базовая нагрузка (регрессия lab3) | `tests/load.py --duration 30 --concurrency 4` | 471 успешных RPS, 0 ошибок, p50/p95/p99 = 7.4/13.0/32.7 мс | `evidence/load.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Память при параллельных загрузках + +**Гипотеза:** при потоковом приёме (spool 1 MiB) и одном декодировании за раз пик анонимной памяти реплики не растёт с числом одновременных загрузок; если держать тело целиком в RAM, рост линейный. + +**Методика** (`student-tests/upload_memory.py`): N потоков одновременно загружают через LB один и тот же PNG 9.6 MiB из случайного шума (несжимаемый, близко к лимиту); профиль «40 Мпикс» — PNG 7000×5714 RGB (117 KB файл, 120 MB после декодирования). Каждый контейнер опрашивается 4–5 раз в секунду: `anon` из `memory.stat` cgroup v2 (память процесса; `memory.current` включает ещё и reclaimable page cache временных файлов). API пересоздаётся перед каждым прогоном (одинаковая база ≈ 64 MiB). Контраст «тело в RAM» — тот же код с `UPLOAD_SPOOL_BYTES=16 MiB` (spool не сбрасывается на диск). Лимиты: api 0.5 CPU / 384M, s3 1 CPU / 768M. Один прогон на точку — демонстрация, не статистика. + +| Профиль | N | Результат | Время, с | API anon: прирост (api-1 / api-2), MiB | S3 anon пик, MiB | +| --- | --- | --- | --- | --- | --- | +| поток, шум 9.6 MiB | 10 | 10×201 | 2.7 | +38.5 / +32.4 | 380 | +| поток, шум 9.6 MiB | 20 | 20×201 | 5.2 | +32.1 / +37.3 | 422 | +| поток, шум 9.6 MiB | 40 | 40×201 | 9.2 | +49.1 / +45.9 | 403 | +| поток, 40 Мпикс | 10 | 10×201 | 3.2 | +161.8 / +163.7 | 325 | +| тело в RAM, шум | 10 | 10×201 | 3.9 | +83.1 / +77.3 | 426 | +| тело в RAM, шум | 20 | 20×201 | 6.0 | +137.6 / +142.5 | 449 | +| тело в RAM, шум | 40 | 40×201 | 9.9 | +194.0 / +220.8 | 434 | + +**Вывод:** гипотеза подтверждена. При потоковом приёме прирост ~32–49 MiB почти не меняется от 10 к 40 загрузкам (это одно декодирование ~9.6 MB RGB + чтение IDAT в `verify()` + буферы boto3/аллокатор); при теле в RAM рост линейный (~+5 MiB на каждую одновременную загрузку на реплике) и ограничен лишь `UPLOAD_CONCURRENCY`. Худший случай — изображение на границе 40 Мпикс: +162 MiB, но это **одно** декодирование (семафор), десять таких параллельно памяти не умножают. Верхняя оценка на реплику: база 64 + 16×1 MiB spool + 1 декодирование ≤ 160 MiB (RGBA 40 Мпикс) ≈ 240 MiB < 384M. + +### Найденный отказ: OOM SeaweedFS (до исправления) + +Первая версия без `S3_PUT_CONCURRENCY` и лимитов SeaweedFS при 20 параллельных 10 MiB загрузках дала **16 из 20 ответов 503**: `memory.current` S3 упирался в 512 MiB, процесс `weed` убивался без panic, контейнер перезапускался и при старте «re-indexed» хвосты volume-файлов (`evidence/s3-oom-before-fix.txt`). Потерь не было: клиенты получили 503 (не 201), компенсация/sweeper убрали pending-строки. Исправление в две стороны: API ограничивает параллельные PUT (4 на реплику, остальные ждут), SeaweedFS получил `concurrentUploadLimitMB=128` и `GOMEMLIMIT=450MiB` (без него Go-heap доходил до 726 из 768 MiB). После: 20 и 40 параллельных — все 201, пик S3 anon 403–422 MiB, рестартов 0. + +### Отказы | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка API-реплики, принявшей файл | файл отдаёт другая | 2/2 раунда, 10/10 скачиваний, SHA-256 совпал | ≤ 3 с (вывод из LB); потерь нет | +| Публичный failover ×2 (сразу после persistence) | запись ≤ 10 с | 0 failed attempts, recovery 3.3 / 1.8 с | данные, файл и токен сохранены | +| `docker stop` api-1 / api-2 под нагрузкой (`make chaos`) | без ошибок | 8858 и 6790 запросов, **0 ошибок**, p99 96 / 149 мс, max 1.03 с | ≤ 3 с; потерь и дублей нет | +| Restart всего стенда (`restart.sh`) | метаданные, байты, сессия живы | PASS | ≈ 27 с от старта S3 до готовности API (ожидание volume-сервера + прогрев записи в s3-init); потерь нет | +| S3 остановлен | 503 только на файлах | upload 503 за 0.8 с, content 503, остальное работает | сразу после `start` S3 | +| Обрыв клиента посреди тела | ничего не сохранено | 0 объектов, 0 строк | — | +| Внедрённые orphan/pending на обеих сторонах | обнаружение и безопасная очистка | dry-run: 2 pending + 1 orphan, ничего не изменено; реальный проход удалил их, свежие/ready не тронуты; повтор — 0; проход ~0.2 с на 27 объектов | — | +| 20 параллельных 10 MiB до исправления | — | OOM SeaweedFS, 16/20 → 503 | см. выше; исправлено | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается отказ **одной API-реплики** и кратковременная недоступность S3 (файлы — 503, остальное работает). Не выдерживаются: отказ PostgreSQL, LB, Docker-host; SeaweedFS — один узел без репликации (`replication 000`), потеря volume = потеря байтов. +- Sweeper обрабатывает только незавершённые загрузки и orphan-объекты. «Missing» (ready-строка без объекта) он не ищет и ничего не чинит — скачивание такого файла даёт 503 и ошибку в логе. Полный reconciler с отчётом о missing — это трек 5.2, не заявлен. Обход листинга линейный по числу объектов (~0.2 с на десятки объектов, на миллионах нужен инкрементальный обход). +- Вложения, загруженные, но так и не привязанные к сообщению, не удаляются: контракт не задаёт срок жизни непривязанных файлов. +- Временный spool пишется в writable-слой контейнера и удаляется в конце запроса; при жёстком убийстве процесса файл в `/tmp` контейнера остаётся до пересоздания контейнера (это не оригинал и не источник истины). +- Декодирование 40 Мпикс PNG (до 160 MiB для RGBA) выполняется по одному на реплику; при массовой загрузке таких файлов они ждут друг друга (латентность растёт, память — нет). Асинхронная обработка — lab5. +- Повтор загрузки после неопределённого ответа может создать второе вложение (по контракту идемпотентности для загрузки нет); HAProxy не повторяет POST с большим телом. +- SeaweedFS предупреждает, что внутренний IAM gRPC на filer без JWT; он слушает только loopback внутри контейнера и недоступен из сети, но полноценная защита (`security.toml`/JWT, TLS) — задача lab7. +- 471 RPS в базовой нагрузке ниже 840 RPS из lab3 при том же профиле; прогон одиночный, на хосте одновременно работали sweeper и другие контейнеры, причину не исследовал — не делаю вывода о деградации. +- Тестовый порт у автора 8088 (8080 занят посторонним контейнером); по умолчанию — 8080. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..298a65d6 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import asyncio +import base64 +import binascii +import hashlib +import secrets +from datetime import datetime, timezone +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from starlette.concurrency import run_in_threadpool + +from app import config +from app.cache import REVOKED, SessionCache +from app.errors import BASIC_CHALLENGE, ApiError, unauthorized +from app.storage import Store, fmt_ts, user_row + +_hasher = PasswordHasher() +_hash_slots = asyncio.Semaphore(config.PASSWORD_HASH_CONCURRENCY) +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + + +async def hash_password(password: str) -> str: + async with _hash_slots: + return await run_in_threadpool(_hasher.hash, password) + + +async def verify_password(password_hash: str, password: str) -> bool: + def verify() -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerificationError, InvalidHashError): + return False + + async with _hash_slots: + return await run_in_threadpool(verify) + + +def token_digest(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header: Optional[str]) -> Tuple[str, str]: + bad = unauthorized("invalid credentials", BASIC_CHALLENGE) + if not header: + raise bad + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise bad + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + raise bad from None + username, sep, password = decoded.partition(":") + if not sep or not username.isascii() or not 12 <= len(password) <= 128: + raise bad + return username, password + + +def parse_bearer(header: Optional[str]) -> str: + if not header: + raise unauthorized("bearer token required") + scheme, _, token = header.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not token or not token.isascii() or len(token) > 256: + raise unauthorized("bearer token required") + return token + + +async def login(store: Store, header: Optional[str]) -> dict: + username, password = parse_basic(header) + row = await store.get_credentials(username) + if row is None: + await verify_password(_DUMMY_HASH, password) + raise unauthorized("invalid credentials", BASIC_CHALLENGE) + if not await verify_password(row["password_hash"], password): + raise unauthorized("invalid credentials", BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + expires_at = await store.create_session(token_digest(token), row["id"], config.SESSION_TTL_SECONDS) + return {"token": token, "token_type": "Bearer", "expires_at": fmt_ts(expires_at), "user": user_row(row)} + + +async def authenticate(store: Store, cache: SessionCache, header: Optional[str]) -> dict: + digest = token_digest(parse_bearer(header)) + cached = await cache.get(digest) + if cached == REVOKED: + raise unauthorized("session expired or revoked") + if isinstance(cached, dict): + expires_at = datetime.fromisoformat(cached["expires_at"].replace("Z", "+00:00")) + if expires_at > datetime.now(timezone.utc): + return cached["user"] + raise unauthorized("session expired or revoked") + + row = await store.find_session(digest) + if row is None: + raise unauthorized("session expired or revoked") + user = user_row(row) + remaining = int((row["expires_at"] - datetime.now(timezone.utc)).total_seconds()) + await cache.put(digest, {"user": user, "expires_at": fmt_ts(row["expires_at"])}, remaining) + return user + + +async def logout(store: Store, cache: SessionCache, header: Optional[str]) -> None: + await authenticate(store, cache, header) + digest = token_digest(parse_bearer(header)) + expires_at = await store.revoke_session(digest) + if expires_at is None: + raise unauthorized("session expired or revoked") + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + 1 + if not await cache.revoke(digest, remaining): + raise ApiError(503, "unavailable", "logout not confirmed by session cache, retry later") diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..02db961c --- /dev/null +++ b/app/cache.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import time +from typing import Dict, Optional, Set, Union + +import redis.asyncio as redis +from redis.exceptions import RedisError + +from app import config + +logger = logging.getLogger("messenger.cache") + +REVOKED = "revoked" + + +class SessionCache: + def __init__(self, url: Optional[str]): + self._client: Optional[redis.Redis] = None + self._down_until = 0.0 + self._pending_revokes: Dict[bytes, float] = {} + self._retry_tasks: Set[asyncio.Task] = set() + if url: + pool = redis.BlockingConnectionPool.from_url( + url, + max_connections=config.CACHE_POOL_MAX, + timeout=config.CACHE_TIMEOUT, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + self._client = redis.Redis(connection_pool=pool) + + @property + def enabled(self) -> bool: + return self._client is not None + + def _usable(self) -> bool: + return self._client is not None and time.monotonic() >= self._down_until + + def _failed(self, op: str, exc: Exception) -> None: + self._down_until = time.monotonic() + config.CACHE_RETRY_AFTER + logger.warning("cache %s failed (%s); bypassing cache for %.0fs", op, type(exc).__name__, config.CACHE_RETRY_AFTER) + + @staticmethod + def _key(token_hash: bytes) -> str: + return "session:" + token_hash.hex() + + def _revoke_pending(self, token_hash: bytes) -> bool: + deadline = self._pending_revokes.get(token_hash) + if deadline is None: + return False + if time.monotonic() >= deadline: + self._pending_revokes.pop(token_hash, None) + return False + return True + + async def get(self, token_hash: bytes) -> Union[dict, str, None]: + if not self._usable() or self._revoke_pending(token_hash): + return None + try: + raw = await self._client.get(self._key(token_hash)) + except (RedisError, OSError) as exc: + self._failed("get", exc) + return None + if raw is None or raw == REVOKED: + return raw + try: + return json.loads(raw) + except ValueError: + return None + + async def put(self, token_hash: bytes, entry: dict, ttl_seconds: int) -> None: + ttl = min(ttl_seconds, config.SESSION_CACHE_MAX_SECONDS) + if ttl <= 0 or not self._usable(): + return + try: + await self._client.set(self._key(token_hash), json.dumps(entry), ex=ttl, nx=True) + except (RedisError, OSError) as exc: + self._failed("set", exc) + + async def _write_tombstone(self, token_hash: bytes, ttl_seconds: int) -> bool: + try: + await self._client.set(self._key(token_hash), REVOKED, ex=max(1, min(ttl_seconds, config.SESSION_CACHE_MAX_SECONDS))) + return True + except (RedisError, OSError) as exc: + self._failed("revoke", exc) + return False + + async def revoke(self, token_hash: bytes, ttl_seconds: int) -> bool: + if self._client is None: + return True + if await self._write_tombstone(token_hash, ttl_seconds): + return True + deadline = time.monotonic() + config.SESSION_CACHE_MAX_SECONDS + 1 + self._pending_revokes[token_hash] = deadline + task = asyncio.create_task(self._retry_revoke(token_hash, ttl_seconds, deadline)) + self._retry_tasks.add(task) + task.add_done_callback(self._retry_tasks.discard) + return False + + async def _retry_revoke(self, token_hash: bytes, ttl_seconds: int, deadline: float) -> None: + while time.monotonic() < deadline: + await asyncio.sleep(1) + remaining = int(deadline - time.monotonic()) + if remaining > 0 and await self._write_tombstone(token_hash, min(ttl_seconds, remaining)): + logger.info("revocation tombstone written after cache recovery") + break + self._pending_revokes.pop(token_hash, None) + + async def ping(self) -> bool: + if self._client is None: + return False + try: + return bool(await self._client.ping()) + except (RedisError, OSError): + return False + + async def close(self) -> None: + for task in list(self._retry_tasks): + task.cancel() + if self._client is not None: + await self._client.aclose() diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..c85acef7 --- /dev/null +++ b/app/config.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +import secrets +from typing import Optional + + +def _secret(name: str) -> Optional[str]: + value = os.environ.get(name) + if value: + return value.strip() + path = os.environ.get(name + "_FILE") + if path and os.path.exists(path): + with open(path, encoding="utf-8") as fh: + return fh.read().strip() or None + return None + + +def _int(name: str, default: int) -> int: + return int(os.environ.get(name, default)) + + +def _float(name: str, default: float) -> float: + return float(os.environ.get(name, default)) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:8333") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 5) +S3_POOL_MAX = _int("S3_POOL_MAX", 20) +# Without this cap 20 parallel 10 MiB PUTs drove SeaweedFS (512M limit) into an OOM kill. +S3_PUT_CONCURRENCY = _int("S3_PUT_CONCURRENCY", 4) + +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_MAX_SECONDS = _int("SESSION_CACHE_MAX_SECONDS", 300) + +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 3) +DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 3) +DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5) + +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.3) +CACHE_POOL_MAX = _int("CACHE_POOL_MAX", 20) +CACHE_RETRY_AFTER = _float("CACHE_RETRY_AFTER", 5) + +PASSWORD_HASH_CONCURRENCY = _int("PASSWORD_HASH_CONCURRENCY", 4) + +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1024 * 1024) +UPLOAD_TMP_DIR = os.environ.get("UPLOAD_TMP_DIR") or None +UPLOAD_TIMEOUT_SECONDS = _float("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 16) +DECODE_CONCURRENCY = _int("DECODE_CONCURRENCY", 1) + +ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) +SWEEP_INTERVAL_SECONDS = _int("SWEEP_INTERVAL_SECONDS", 60) + +INSTANCE_ID = "api-" + secrets.token_hex(4) +INSTANCE_ID_FILE = os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id") diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..4d38adf8 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Dict, Optional + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(message: str, challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def unavailable() -> ApiError: + return ApiError(503, "unavailable", "service temporarily unavailable") + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..2e57a94e --- /dev/null +++ b/app/images.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import unicodedata +import warnings +from typing import BinaryIO, Optional, Tuple + +from PIL import Image + +from app import config +from app.errors import ApiError, payload_too_large, unsupported_media_type + +# Pillow itself refuses > 2x this value; our own check below rejects anything above it before decoding. +Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS +Image.LOAD_TRUNCATED_IMAGES = False + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +SNIFF_BYTES = 8 + +# Bidi overrides/isolates make "gpj.exe" look like "exe.jpg"; reject them together with control characters. +_BIDI_CONTROLS = set("‪‫‬‭‮⁦⁧⁨⁩‎‏") + + +def validate_filename(name: Optional[str]) -> str: + bad = ApiError(400, "invalid_request", "filename must be 1-255 characters without path separators or control characters") + if name is None or not 1 <= len(name) <= 255 or not name.strip() or name in (".", ".."): + raise bad + for ch in name: + if ch in "/\\" or ch in _BIDI_CONTROLS or unicodedata.category(ch) in ("Cc", "Zl", "Zp"): + raise bad + return name + + +def declared_type(header: Optional[str]) -> str: + media = (header or "").split(";", 1)[0].strip().lower() + if media not in FORMATS: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return media + + +def sniff(head: bytes) -> Optional[str]: + if head.startswith(PNG_MAGIC): + return "image/png" + if head.startswith(JPEG_MAGIC): + return "image/jpeg" + return None + + +def _open(fp: BinaryIO, fmt: str) -> Image.Image: + fp.seek(0) + return Image.open(fp, formats=[fmt]) + + +def inspect(fp: BinaryIO, content_type: str) -> Tuple[int, int]: + """Check pixel count from the header, then fully decode to reject corrupted files. + + Runs in a worker thread under a semaphore: decoding is the only step whose memory grows with pixels. + """ + fmt = FORMATS[content_type] + corrupt = unsupported_media_type("file is not a valid " + fmt + " image") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with _open(fp, fmt) as im: + width, height = im.size + if width < 1 or height < 1: + raise corrupt + if width * height > config.MAX_IMAGE_PIXELS: + raise payload_too_large("image exceeds %d pixels" % config.MAX_IMAGE_PIXELS) + if fmt == "PNG": + im.verify() # chunk CRCs; the object is unusable afterwards + with _open(fp, fmt) as im: + if fmt == "JPEG": + # DCT scaling still walks the whole entropy-coded stream but allocates 1/64 of the pixels. + im.draft(None, (max(1, width // 8), max(1, height // 8))) + im.load() + except ApiError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise payload_too_large("image exceeds %d pixels" % config.MAX_IMAGE_PIXELS) from None + except (OSError, SyntaxError, ValueError, EOFError, MemoryError): + raise corrupt from None + return width, height diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..97e093b4 --- /dev/null +++ b/app/main.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import re +import tempfile +import uuid +from contextlib import asynccontextmanager +from typing import Optional + +import asyncpg +from fastapi import Depends, FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect + +from app import auth, config, images +from app.cache import SessionCache +from app.errors import ApiError, payload_too_large, unsupported_media_type +from app.models import (Attachment, Chat, ChatPage, CreateChat, CreateMessage, CreateUser, Health, Message, MessagePage, + Session, User) +from app.objects import ObjectStore, S3Unavailable, storage_unavailable +from app.storage import Store, attachment_row + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("messenger") + +store = Store() +cache = SessionCache(config.CACHE_URL) +objects = ObjectStore() +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +decode_slots = asyncio.Semaphore(config.DECODE_CONCURRENCY) +s3_put_slots = asyncio.Semaphore(config.S3_PUT_CONCURRENCY) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required") + if not (config.S3_ACCESS_KEY_ID and config.S3_SECRET_ACCESS_KEY): + raise RuntimeError("S3_ACCESS_KEY_ID(_FILE) and S3_SECRET_ACCESS_KEY(_FILE) are required") + await store.open() + logger.info("session cache: %s", "valkey" if cache.enabled else "disabled") + logger.info("object storage: %s bucket=%s", config.S3_ENDPOINT, config.S3_BUCKET) + try: + with open(config.INSTANCE_ID_FILE, "w", encoding="ascii") as fh: + fh.write(config.INSTANCE_ID + "\n") + except OSError: + logger.warning("cannot write %s", config.INSTANCE_ID_FILE) + logger.info("instance id: %s", config.INSTANCE_ID) + yield + await cache.close() + await store.close() + + +app = FastAPI(title="MTUSI Messenger — lab4", lifespan=lifespan) + +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +DB_UNAVAILABLE = ( + OSError, + asyncpg.PostgresConnectionError, + asyncpg.InterfaceError, + asyncpg.TooManyConnectionsError, + asyncpg.exceptions.OperatorInterventionError, + asyncpg.exceptions.QueryCanceledError, +) + + +def error_body(code: str, message: str, request_id: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def error_response(request: Request, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") + return JSONResponse(status_code=status, content=error_body(code, message, request_id), headers=headers) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = uuid.uuid4().hex + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception("unhandled error for request_id=%s", request_id) + response = JSONResponse(status_code=500, content=error_body("internal_error", "internal error", request_id)) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.status_code, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "request failed validation") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + code = {404: "not_found", 405: "invalid_request", 413: "payload_too_large", 415: "unsupported_media_type"}.get( + exc.status_code, "invalid_request") + message = exc.detail if isinstance(exc.detail, str) else "request failed" + return error_response(request, exc.status_code, code, message) + + +async def db_unavailable_handler(request: Request, exc: Exception): + logger.warning("database unavailable for request_id=%s: %s", request.state.request_id, type(exc).__name__) + return error_response(request, 503, "unavailable", "service temporarily unavailable") + + +for _exc in DB_UNAVAILABLE: + app.add_exception_handler(_exc, db_unavailable_handler) + + +@app.exception_handler(S3Unavailable) +async def s3_unavailable_handler(request: Request, exc: S3Unavailable): + err = storage_unavailable() + return error_response(request, err.status_code, err.code, err.message) + + +async def current_user(request: Request) -> dict: + return await auth.authenticate(store, cache, request.headers.get("Authorization")) + + +def parse_uuid(value: str, field: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except ValueError: + raise ApiError(400, "invalid_request", f"{field} is not a valid UUID") from None + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "chat not found") + + +@app.get("/health/live", response_model=Health) +async def liveness(): + return Health() + + +@app.get("/health/ready", response_model=Health) +async def readiness(): + try: + await store.ping() + except DB_UNAVAILABLE: + raise ApiError(503, "unavailable", "database is not available") from None + return Health() + + +@app.post("/api/v1/users", response_model=User, status_code=201) +async def register_user(body: CreateUser): + if not body.display_name.strip() or not body.password.strip(): + raise ApiError(400, "invalid_request", "display_name and password must not be blank") + password_hash = await auth.hash_password(body.password) + return await store.create_user(body.username, body.display_name, password_hash) + + +@app.get("/api/v1/users/me", response_model=User) +async def current_user_endpoint(user: dict = Depends(current_user)): + return user + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +async def create_session(request: Request): + return await auth.login(store, request.headers.get("Authorization")) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def delete_session(request: Request): + await auth.logout(store, cache, request.headers.get("Authorization")) + return Response(status_code=204) + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +async def create_chat(body: CreateChat, user: dict = Depends(current_user)): + if not body.title.strip(): + raise ApiError(400, "invalid_request", "title must not be blank") + if len(set(body.member_ids)) != len(body.member_ids): + raise ApiError(400, "invalid_request", "member_ids must not contain duplicates") + if uuid.UUID(user["id"]) not in body.member_ids: + raise ApiError(400, "invalid_request", "caller must be a member of the chat") + return await store.create_chat(body.title, list(body.member_ids)) + + +@app.get("/api/v1/chats", response_model=ChatPage) +async def list_chats( + user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +): + return await store.list_chats(uuid.UUID(user["id"]), limit, cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +async def get_chat(chat_id: str, user: dict = Depends(current_user)): + chat = await store.get_chat_for_member(parse_uuid(chat_id, "chat_id"), uuid.UUID(user["id"])) + if chat is None: + raise not_found() + return chat + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: str, + body: CreateMessage, + user: dict = Depends(current_user), + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}") + if len(set(body.attachment_ids)) != len(body.attachment_ids): + raise ApiError(400, "invalid_request", "attachment_ids must be unique") + if not body.text.strip() and not body.attachment_ids: + if not await store.is_member(chat_uuid, uuid.UUID(user["id"])): + raise not_found() + raise ApiError(400, "invalid_request", "text must not be blank without attachments") + canonical = json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + request_hash = hashlib.sha256(canonical.encode("utf-8")).digest() + message = await store.create_message( + chat_uuid, uuid.UUID(user["id"]), body.text, list(body.attachment_ids), idempotency_key, request_hash) + if message is None: + raise not_found() + return message + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: str, + user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + if not await store.is_member(chat_uuid, uuid.UUID(user["id"])): + raise not_found() + return await store.list_messages(chat_uuid, limit, cursor) + + +VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +async def receive_body(request: Request, spool) -> tuple: + """Stream the body into a spooled temp file: RAM holds at most UPLOAD_SPOOL_BYTES per upload.""" + digest = hashlib.sha256() + size = 0 + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES) + if len(head) < images.SNIFF_BYTES: + head += chunk[:images.SNIFF_BYTES - len(head)] + digest.update(chunk) + spool.write(chunk) + return size, digest.hexdigest(), head + + +@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + user: dict = Depends(current_user), + filename: Optional[str] = Query(default=None), +): + chat_uuid = parse_uuid(chat_id, "chat_id") + owner_id = uuid.UUID(user["id"]) + if not await store.is_member(chat_uuid, owner_id): + raise not_found() + filename = images.validate_filename(filename) + declared = images.declared_type(request.headers.get("Content-Type")) + length = request.headers.get("Content-Length") + if length is not None: + if not length.isdigit(): + raise ApiError(400, "invalid_request", "invalid Content-Length") + if int(length) > config.MAX_UPLOAD_BYTES: + raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES) + + async with upload_slots: + spool = tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES, dir=config.UPLOAD_TMP_DIR) + try: + try: + size, sha256, head = await asyncio.wait_for(receive_body(request, spool), config.UPLOAD_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + raise ApiError(400, "invalid_request", "upload was not completed in time") from None + except ClientDisconnect: + logger.info("upload aborted by client after partial body; nothing stored") + raise ApiError(400, "invalid_request", "upload interrupted") from None + if size == 0: + raise unsupported_media_type("empty body is not an image") + detected = images.sniff(head) + if detected is None: + raise unsupported_media_type("content is neither PNG nor JPEG") + if detected != declared: + raise unsupported_media_type("content does not match Content-Type " + declared) + async with decode_slots: + width, height = await run_in_threadpool(images.inspect, spool, detected) + + attachment_id = uuid.uuid4() + key = ObjectStore.new_key(uuid.uuid4()) + # 1) intent row (invisible) -> 2) S3 PUT -> 3) mark ready. A crash leaves a pending row for the sweeper. + if not await store.create_pending_attachment( + attachment_id, chat_uuid, owner_id, filename, detected, size, sha256, width, height, key, + ): + raise not_found() + try: + async with s3_put_slots: + await objects.put(key, spool, size, detected, sha256) + except S3Unavailable: + await objects.delete_quietly(key) + await store.delete_pending_attachment(attachment_id) + raise + try: + ready = await store.mark_attachment_ready(attachment_id) + except DB_UNAVAILABLE: + await objects.delete_quietly(key) + raise + if ready is None: + await objects.delete_quietly(key) + raise storage_unavailable() + return ready + finally: + spool.close() + + +async def readable_attachment(attachment_id: str, user: dict): + row = await store.get_attachment_for_member(parse_uuid(attachment_id, "attachment_id"), uuid.UUID(user["id"])) + if row is None: + raise ApiError(404, "not_found", "attachment not found") + return row + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +async def get_attachment(attachment_id: str, user: dict = Depends(current_user)): + return attachment_row(await readable_attachment(attachment_id, user)) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: str, + user: dict = Depends(current_user), + variant: str = Query(default="original"), +): + if variant not in VARIANTS: + raise ApiError(400, "invalid_request", "unknown variant") + row = await readable_attachment(attachment_id, user) + if variant != "original": + raise ApiError(404, "not_found", "variant not found") + chunks, length, close = await objects.open(row["object_key"]) + if length != row["size_bytes"]: + close() + logger.error("object size differs from metadata for attachment %s", row["id"]) + raise storage_unavailable() + return StreamingResponse( + chunks, + media_type=row["content_type"], + headers={ + "Content-Length": str(length), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + background=BackgroundTask(close), + ) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..29df5991 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +from pathlib import Path + +import asyncpg + +from app import config + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("migrate") + +MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "migrations" +LOCK_ID = 2026_0002 + + +async def connect_with_retry(attempts: int = 30) -> asyncpg.Connection: + for attempt in range(1, attempts + 1): + try: + return await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError) as exc: + logger.warning("database not reachable (attempt %d/%d): %s", attempt, attempts, type(exc).__name__) + await asyncio.sleep(1) + raise SystemExit("database is not reachable") + + +async def migrate() -> int: + if not config.DATABASE_URL: + raise SystemExit("DATABASE_URL or DATABASE_URL_FILE is required") + conn = await connect_with_retry() + applied_now = 0 + try: + await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID) + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + done = {r["version"] for r in await conn.fetch("SELECT version FROM schema_migrations")} + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = path.stem + if version in done: + continue + logger.info("applying %s", version) + async with conn.transaction(): + await conn.execute(path.read_text(encoding="utf-8")) + await conn.execute("INSERT INTO schema_migrations (version) VALUES ($1)", version) + applied_now += 1 + logger.info("migrations up to date: %d applied now, %d before", applied_now, len(done)) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID) + await conn.close() + return applied_now + + +if __name__ == "__main__": + asyncio.run(migrate()) + sys.exit(0) diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..d1247bb2 --- /dev/null +++ b/app/models.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import uuid +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field + +USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$" + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + +class User(BaseModel): + id: uuid.UUID + username: str + display_name: str + created_at: str + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(min_length=1, max_length=100) + + +class Chat(BaseModel): + id: uuid.UUID + title: str + member_ids: List[uuid.UUID] + created_at: str + + +class ChatPage(BaseModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default_factory=list, max_length=10) + + +class Message(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + sender_id: uuid.UUID + text: str + created_at: str + attachment_ids: List[uuid.UUID] + + +class MessagePage(BaseModel): + items: List[Message] + next_cursor: Optional[str] = None + + +class Session(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: User + + +class Variant(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: uuid.UUID + chat_id: uuid.UUID + owner_id: uuid.UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: List[Variant] + error_code: Optional[str] + created_at: str + updated_at: str + + +class Health(BaseModel): + status: str = "ok" diff --git a/app/objects.py b/app/objects.py new file mode 100644 index 00000000..b8e9d80a --- /dev/null +++ b/app/objects.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from starlette.concurrency import run_in_threadpool + +from app import config +from app.errors import ApiError + +logger = logging.getLogger("messenger.s3") + +ORIGINALS_PREFIX = "originals/" +DOWNLOAD_CHUNK = 64 * 1024 + + +class S3Unavailable(Exception): + pass + + +def _client(access_key: Optional[str], secret_key: Optional[str]): + cfg = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=config.S3_CONNECT_TIMEOUT, + read_timeout=config.S3_READ_TIMEOUT, + retries={"max_attempts": 1, "mode": "standard"}, + max_pool_connections=config.S3_POOL_MAX, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ) + return boto3.session.Session().client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=cfg, + ) + + +def _code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "") + + +class ObjectStore: + """Synchronous boto3 client; async wrappers run it in the threadpool.""" + + def __init__(self, access_key: Optional[str] = None, secret_key: Optional[str] = None): + self.bucket = config.S3_BUCKET + self._s3 = _client(access_key or config.S3_ACCESS_KEY_ID, secret_key or config.S3_SECRET_ACCESS_KEY) + + @staticmethod + def new_key(attachment_id) -> str: + return ORIGINALS_PREFIX + attachment_id.hex + + def _put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + fileobj.seek(0) + try: + self._s3.put_object( + Bucket=self.bucket, + Key=key, + Body=fileobj, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + logger.warning("s3 put failed: %s %s", type(exc).__name__, _code(exc) if isinstance(exc, ClientError) else "") + raise S3Unavailable() from None + + async def put(self, key: str, fileobj: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + await run_in_threadpool(self._put, key, fileobj, size, content_type, sha256_hex) + + def _open(self, key: str): + try: + return self._s3.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + logger.error("s3 get failed for existing metadata: %s", _code(exc)) + raise S3Unavailable() from None + except BotoCoreError as exc: + logger.warning("s3 get failed: %s", type(exc).__name__) + raise S3Unavailable() from None + + async def open(self, key: str) -> Tuple[Iterator[bytes], int, Callable[[], None]]: + obj = await run_in_threadpool(self._open, key) + body = obj["Body"] + return body.iter_chunks(DOWNLOAD_CHUNK), int(obj["ContentLength"]), body.close + + def delete_sync(self, key: str) -> None: + try: + self._s3.delete_object(Bucket=self.bucket, Key=key) + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + + async def delete_quietly(self, key: str) -> bool: + try: + await run_in_threadpool(self.delete_sync, key) + return True + except S3Unavailable: + logger.warning("compensating delete failed; sweeper will retry later") + return False + + def list_originals(self) -> Iterator[Tuple[str, datetime]]: + paginator = self._s3.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=self.bucket, Prefix=ORIGINALS_PREFIX, PaginationConfig={"PageSize": 1000}): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + + def delete_many(self, keys: List[str]) -> int: + deleted = 0 + for i in range(0, len(keys), 1000): + batch = keys[i:i + 1000] + try: + out = self._s3.delete_objects(Bucket=self.bucket, Delete={"Objects": [{"Key": k} for k in batch], "Quiet": True}) + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable() from exc + deleted += len(batch) - len(out.get("Errors", [])) + return deleted + + def bucket_exists(self) -> bool: + try: + self._s3.head_bucket(Bucket=self.bucket) + return True + except ClientError as exc: + if _code(exc) in ("404", "NoSuchBucket", "NotFound"): + return False + raise S3Unavailable() from exc + except BotoCoreError as exc: + raise S3Unavailable() from exc + + def create_bucket(self) -> bool: + try: + self._s3.create_bucket(Bucket=self.bucket) + return True + except ClientError as exc: + if _code(exc) in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + return False + raise + + +def storage_unavailable() -> ApiError: + return ApiError(503, "unavailable", "attachment storage temporarily unavailable") diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 00000000..c81da8a1 --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,76 @@ +"""Idempotent bucket bootstrap with the admin identity (the API identity cannot create buckets). + +Checks afterwards that anonymous access to the bucket is denied, and warms up the write path: after a +SeaweedFS restart the first PUT waits ~2.5 s for the volume topology, which a client with a short timeout +would see as a failed upload. The API starts only after this one-shot service exits 0. +""" +from __future__ import annotations + +import io +import logging +import os +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.config import _secret +from app.objects import ObjectStore, S3Unavailable + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("s3-bootstrap") + + +def anonymous_status(path: str) -> int: + try: + with urllib.request.urlopen(config.S3_ENDPOINT.rstrip("/") + path, timeout=3) as r: + return r.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + admin = ObjectStore(_secret("S3_ADMIN_ACCESS_KEY_ID"), _secret("S3_ADMIN_SECRET_ACCESS_KEY")) + for attempt in range(1, 31): + try: + created = admin.create_bucket() + break + except (BotoCoreError, ClientError, S3Unavailable) as exc: + logger.warning("s3 not ready (attempt %d/30): %s", attempt, type(exc).__name__) + time.sleep(1) + else: + return 1 + logger.info("bucket %s %s", config.S3_BUCKET, "created" if created else "already exists") + + for path in ("/" + config.S3_BUCKET, "/" + config.S3_BUCKET + "/originals/probe"): + status = anonymous_status(path) + if status not in (401, 403): + logger.error("anonymous request %s returned %d: bucket is NOT private", path, status) + return 1 + logger.info("anonymous list/get denied: bucket is private") + + probe = "health/bootstrap-probe" # outside originals/, so the sweeper never looks at it + for attempt in range(1, 31): + started = time.monotonic() + try: + admin._s3.put_object(Bucket=admin.bucket, Key=probe, Body=io.BytesIO(b"probe")) + admin._s3.get_object(Bucket=admin.bucket, Key=probe)["Body"].read() + admin._s3.delete_object(Bucket=admin.bucket, Key=probe) + except (BotoCoreError, ClientError) as exc: + logger.warning("write probe failed (attempt %d/30): %s", attempt, type(exc).__name__) + time.sleep(1) + continue + elapsed = time.monotonic() - started + if elapsed < 0.5: + logger.info("write path warm: put/get/delete probe %.3fs", elapsed) + return 0 + logger.info("write probe slow (%.2fs), repeating", elapsed) + return 1 + + +if __name__ == "__main__": + os.umask(0o077) + sys.exit(main()) diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..a8141537 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +import base64 +import json +import uuid +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +import asyncpg + +from app import config +from app.errors import ApiError + + +def fmt_ts(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def user_row(row: asyncpg.Record) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": fmt_ts(row["created_at"]), + } + + +def chat_row(row: asyncpg.Record) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], + "created_at": fmt_ts(row["created_at"]), + } + + +def message_row(row: asyncpg.Record, attachment_ids: Optional[List[uuid.UUID]] = None) -> dict: + if attachment_ids is None: + attachment_ids = row["attachment_ids"] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": fmt_ts(row["created_at"]), + "attachment_ids": [str(a) for a in attachment_ids], + } + + +def attachment_row(row: asyncpg.Record) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": None, + "created_at": fmt_ts(row["created_at"]), + "updated_at": fmt_ts(row["updated_at"]), + } + + +ATTACHMENT_COLUMNS = "id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, object_key, created_at, updated_at" + + +def encode_cursor(resource: str, created_at: str, item_id: str) -> str: + payload = json.dumps({"r": resource, "t": created_at, "i": item_id}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_cursor(resource: str, cursor: str) -> Tuple[datetime, uuid.UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")) + if payload.get("r") != resource: + raise ValueError("cursor belongs to a different resource") + created_at = datetime.fromisoformat(payload["t"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise ValueError("cursor time has no timezone") + return created_at, uuid.UUID(payload["i"]) + except Exception as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + + +def page_of(items: List[dict], resource: str, limit: int) -> dict: + page = items[:limit] + next_cursor = None + if len(items) > limit: + last = page[-1] + next_cursor = encode_cursor(resource, last["created_at"], last["id"]) + return {"items": page, "next_cursor": next_cursor} + + +class Store: + def __init__(self): + self.pool: Optional[asyncpg.Pool] = None + + async def open(self) -> None: + self.pool = await asyncpg.create_pool( + config.DATABASE_URL, + min_size=0, + max_size=config.DB_POOL_MAX, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT, + max_inactive_connection_lifetime=60, + ) + + async def close(self) -> None: + if self.pool is not None: + await self.pool.close() + + def acquire(self): + return self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) + + async def ping(self) -> bool: + async with self.acquire() as conn: + return await conn.fetchval("SELECT 1") == 1 + + + async def create_user(self, username: str, display_name: str, password_hash: str) -> dict: + async with self.acquire() as conn: + try: + row = await conn.fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at + """, + uuid.uuid4(), username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError: + raise ApiError(409, "conflict", "username already taken") from None + return user_row(row) + + async def get_credentials(self, username: str) -> Optional[asyncpg.Record]: + async with self.acquire() as conn: + return await conn.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + + + async def create_session(self, token_hash: bytes, user_id: uuid.UUID, ttl_seconds: int) -> datetime: + async with self.acquire() as conn: + return await conn.fetchval( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, clock_timestamp() + make_interval(secs => $3)) + RETURNING expires_at + """, + token_hash, user_id, float(ttl_seconds), + ) + + async def find_session(self, token_hash: bytes) -> Optional[asyncpg.Record]: + async with self.acquire() as conn: + return await conn.fetchrow( + """ + SELECT s.expires_at, u.id, u.username, u.display_name, u.created_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp() + """, + token_hash, + ) + + async def revoke_session(self, token_hash: bytes) -> Optional[datetime]: + async with self.acquire() as conn: + return await conn.fetchval( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp() + RETURNING expires_at + """, + token_hash, + ) + + + async def create_chat(self, title: str, member_ids: List[uuid.UUID]) -> dict: + chat_id = uuid.uuid4() + async with self.acquire() as conn: + try: + async with conn.transaction(): + row = await conn.fetchrow( + "INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at", + chat_id, title, + ) + await conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, m.user_id, m.ord - 1 + FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord) + """, + chat_id, member_ids, + ) + except asyncpg.ForeignKeyViolationError: + raise ApiError(400, "invalid_request", "unknown member in member_ids") from None + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in member_ids], + "created_at": fmt_ts(row["created_at"]), + } + + async def get_chat_for_member(self, chat_id: uuid.UUID, user_id: uuid.UUID) -> Optional[dict]: + async with self.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.position) AS member_ids + FROM chats c JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members me WHERE me.chat_id = c.id AND me.user_id = $2) + GROUP BY c.id + """, + chat_id, user_id, + ) + return chat_row(row) if row else None + + async def list_chats(self, user_id: uuid.UUID, limit: int, cursor: Optional[str]) -> dict: + after_t, after_i = decode_cursor("chats", cursor) if cursor else (None, None) + async with self.acquire() as conn: + rows = await conn.fetch( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.position) AS member_ids + FROM chat_members me + JOIN chats c ON c.id = me.chat_id + JOIN chat_members m ON m.chat_id = c.id + WHERE me.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + GROUP BY c.id + ORDER BY c.created_at, c.id + LIMIT $4 + """, + user_id, after_t, after_i, limit + 1, + ) + return page_of([chat_row(r) for r in rows], "chats", limit) + + async def is_member(self, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool: + async with self.acquire() as conn: + return bool(await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)", + chat_id, user_id, + )) + + + async def create_message( + self, chat_id: uuid.UUID, sender_id: uuid.UUID, text: str, attachment_ids: List[uuid.UUID], + key: str, request_hash: bytes, + ) -> Optional[dict]: + async with self.acquire() as conn: + for _ in range(3): + try: + async with conn.transaction(): + await conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + AND created_at < clock_timestamp() - make_interval(secs => $4) + """, + sender_id, chat_id, key, float(config.IDEMPOTENCY_TTL_SECONDS), + ) + message_id = uuid.uuid4() + claimed = await conn.fetchval( + """ + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) + SELECT $1, $2, $3, $4, $5 + WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1) + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING message_id + """, + sender_id, chat_id, key, request_hash, message_id, + ) + if claimed is not None: + if attachment_ids: + ready = await conn.fetchval( + """ + SELECT count(*) FROM attachments + WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'ready' + """, + attachment_ids, chat_id, + ) + if ready != len(attachment_ids): + raise ApiError(404, "not_found", "attachment not found") + row = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at + """, + message_id, chat_id, sender_id, text, + ) + if attachment_ids: + await conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) + SELECT $1, a.id, $2, a.ord - 1 + FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord) + """, + message_id, chat_id, attachment_ids, + ) + return message_row(row, attachment_ids) + except asyncpg.ForeignKeyViolationError: + return None + + existing = await conn.fetchrow( + """ + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + JOIN chat_members cm ON cm.chat_id = k.chat_id AND cm.user_id = k.user_id + WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3 + """, + sender_id, chat_id, key, + ) + if existing is None: + is_member = await conn.fetchval( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)", + chat_id, sender_id, + ) + if not is_member: + return None + continue + if existing["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key was already used with a different request") + return message_row(existing) + raise ApiError(503, "unavailable", "could not settle idempotent request, retry with the same key") + + async def list_messages(self, chat_id: uuid.UUID, limit: int, cursor: Optional[str]) -> dict: + resource = str(chat_id) + async with self.acquire() as conn: + if cursor: + after_t, after_i = decode_cursor(resource, cursor) + rows = await conn.fetch( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM messages m + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) + ORDER BY m.created_at, m.id + LIMIT $4 + """, + chat_id, after_t, after_i, limit + 1, + ) + else: + rows = await conn.fetch( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids + FROM messages m + WHERE m.chat_id = $1 + ORDER BY m.created_at, m.id + LIMIT $2 + """, + chat_id, limit + 1, + ) + return page_of([message_row(r) for r in rows], resource, limit) + + + async def create_pending_attachment( + self, attachment_id: uuid.UUID, chat_id: uuid.UUID, owner_id: uuid.UUID, filename: str, + content_type: str, size_bytes: int, sha256: str, width: int, height: int, object_key: str, + ) -> bool: + async with self.acquire() as conn: + try: + await conn.execute( + """ + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'pending') + """, + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, + ) + except asyncpg.ForeignKeyViolationError: + return False + return True + + async def mark_attachment_ready(self, attachment_id: uuid.UUID) -> Optional[dict]: + async with self.acquire() as conn: + row = await conn.fetchrow( + f""" + UPDATE attachments SET status = 'ready', updated_at = clock_timestamp() + WHERE id = $1 AND status = 'pending' + RETURNING {ATTACHMENT_COLUMNS} + """, + attachment_id, + ) + return attachment_row(row) if row else None + + async def delete_pending_attachment(self, attachment_id: uuid.UUID) -> None: + async with self.acquire() as conn: + await conn.execute("DELETE FROM attachments WHERE id = $1 AND status = 'pending'", attachment_id) + + async def get_attachment_for_member(self, attachment_id: uuid.UUID, user_id: uuid.UUID) -> Optional[asyncpg.Record]: + async with self.acquire() as conn: + return await conn.fetchrow( + f""" + SELECT {ATTACHMENT_COLUMNS} FROM attachments a + WHERE a.id = $1 AND a.status = 'ready' + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2) + """, + attachment_id, user_id, + ) diff --git a/app/sweeper.py b/app/sweeper.py new file mode 100644 index 00000000..044cfd89 --- /dev/null +++ b/app/sweeper.py @@ -0,0 +1,130 @@ +"""Cleanup of attachments that never became ready. + +Invariant kept by the upload path: a metadata row (status=pending) is committed BEFORE the S3 PUT and +switched to ready AFTER it. Therefore, after a crash/abort/failed COMMIT there can only be + * a stale pending row (object may or may not exist), or + * an object without any row (row already removed, compensating delete failed). +Both are deleted only when older than ORPHAN_GRACE_SECONDS, far longer than any live upload +(UPLOAD_TIMEOUT_SECONDS + S3 timeouts), so an in-flight upload is never touched. Ready rows and objects +referenced by any row are never deleted. Every step is idempotent; repeating a run is safe. + + python -m app.sweeper --once --dry-run # report only + python -m app.sweeper --once # one real pass + python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import time +from datetime import datetime, timedelta, timezone + +import asyncpg +from starlette.concurrency import run_in_threadpool + +from app import config +from app.objects import ObjectStore, S3Unavailable + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("sweeper") + +LOCK_ID = 2026_0004 +HEARTBEAT_FILE = "/tmp/sweeper-heartbeat" +BATCH = 1000 + + +async def stale_pending(conn: asyncpg.Connection, grace: int): + return await conn.fetch( + """ + SELECT id, object_key FROM attachments + WHERE status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $1) + ORDER BY created_at + """, + float(grace), + ) + + +async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, dry_run: bool) -> dict: + started = time.monotonic() + report = {"dry_run": dry_run, "grace_seconds": grace, "pending_found": 0, "pending_deleted": 0, + "objects_scanned": 0, "orphans_found": 0, "orphans_deleted": 0, "orphan_keys_sample": []} + + rows = await stale_pending(conn, grace) + report["pending_found"] = len(rows) + if not dry_run: + for row in rows: + # Object first, then the row: if we crash in between, the row remains and the next pass repeats. + await run_in_threadpool(objects.delete_sync, row["object_key"]) + deleted = await conn.execute( + """ + DELETE FROM attachments + WHERE id = $1 AND status = 'pending' AND created_at < clock_timestamp() - make_interval(secs => $2) + """, + row["id"], float(grace), + ) + report["pending_deleted"] += int(deleted.split()[-1]) + + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace) + + def old_keys(): + for key, modified in objects.list_originals(): + report["objects_scanned"] += 1 + if modified < cutoff: + yield key + + keys = await run_in_threadpool(lambda: list(old_keys())) + for i in range(0, len(keys), BATCH): + batch = keys[i:i + BATCH] + known = {r["object_key"] for r in await conn.fetch( + "SELECT object_key FROM attachments WHERE object_key = ANY($1::text[])", batch)} + orphans = [k for k in batch if k not in known] + report["orphans_found"] += len(orphans) + report["orphan_keys_sample"].extend(orphans[:5 - len(report["orphan_keys_sample"])]) + if orphans and not dry_run: + report["orphans_deleted"] += await run_in_threadpool(objects.delete_many, orphans) + + report["duration_ms"] = round((time.monotonic() - started) * 1000, 1) + return report + + +async def run_once(grace: int, dry_run: bool) -> dict: + conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT) + try: + if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID): + return {"skipped": "another sweeper holds the lock"} + try: + return await sweep(conn, ObjectStore(), grace, dry_run) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID) + finally: + await conn.close() + + +async def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--once", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS) + p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS) + args = p.parse_args() + if args.grace < 60: + p.error("grace below 60 s could remove an upload that is still in progress") + while True: + try: + report = await run_once(args.grace, args.dry_run) + print(json.dumps(report, ensure_ascii=False), flush=True) + with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh: + fh.write(str(time.time())) + except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc: + logger.warning("sweep failed (%s); will retry", type(exc).__name__) + if args.once: + return 1 + if args.once: + return 0 + await asyncio.sleep(args.interval) + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..64a960d0 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,242 @@ +name: messenger-lab4 + +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + stop_grace_period: 30s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.4-alpine3.22 + command: ["valkey-server", "/run/secrets/valkey_conf"] + secrets: [valkey_conf] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "valkey-cli ping 2>&1 | grep -qE 'PONG|NOAUTH'"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "0.25", memory: 96M} + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + # master/volume/filer (and their web UIs) listen only on loopback inside the container; + # only the S3 gateway is reachable, and only on the internal data network. No published ports. + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + - -s3.ip.bind=0.0.0.0 + - -s3.port=9000 + - -s3.config=/run/secrets/seaweedfs_s3_config + - -master.volumeSizeLimitMB=1024 + - -volume.max=0 + # back-pressure instead of unbounded buffering of in-flight uploads + - -s3.concurrentUploadLimitMB=128 + - -filer.concurrentUploadLimitMB=128 + - -volume.concurrentUploadLimitMB=128 + environment: + # Go GC soft limit below the cgroup limit; without it the heap grew to ~95% of 768M under 40 uploads + GOMEMLIMIT: 450MiB + secrets: [seaweedfs_s3_config] + volumes: + - s3data:/data + networks: [data] + healthcheck: + # /healthz alone turns green before the volume server re-registers after a restart (reads then hang), + # so also require a data node in the master topology (loopback-only endpoint inside the container). + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/healthz && wget -q -O - http://127.0.0.1:9333/dir/status | grep -q '\"DataNodes\":\\[{'"] + interval: 2s + timeout: 3s + start_period: 5s + retries: 30 + deploy: + resources: + limits: {cpus: "1.0", memory: 768M} + stop_grace_period: 20s + restart: unless-stopped + + s3-init: + build: . + image: mtusi-messenger:lab4 + command: ["python", "-m", "app.s3_bootstrap"] + environment: + <<: *s3-env + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [s3_admin_access_key_id, s3_admin_secret_access_key] + networks: [data] + depends_on: + s3: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: "no" + + migrate: + build: . + image: mtusi-messenger:lab4 + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: "no" + + api: + build: . + image: mtusi-messenger:lab4 + environment: + <<: *s3-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: postgres + SESSION_CACHE_MAX_SECONDS: ${SESSION_CACHE_MAX_SECONDS:-300} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MAX: ${DB_POOL_MAX:-10} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-16} + DECODE_CONCURRENCY: ${DECODE_CONCURRENCY:-1} + UPLOAD_SPOOL_BYTES: ${UPLOAD_SPOOL_BYTES:-1048576} + S3_PUT_CONCURRENCY: ${S3_PUT_CONCURRENCY:-4} + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"] + interval: 5s + timeout: 3s + start_period: 5s + retries: 3 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.5", memory: 384M} + stop_signal: SIGTERM + stop_grace_period: 10s + restart: unless-stopped + + sweeper: + build: . + image: mtusi-messenger:lab4 + command: ["python", "-m", "app.sweeper"] + healthcheck: + # no HTTP server: healthy while a pass finished recently (one JSON line per pass in the logs) + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/sweeper-heartbeat') < 3*int(os.environ.get('SWEEP_INTERVAL_SECONDS','60'))+30 else 1)"] + interval: 10s + timeout: 3s + start_period: 10s + retries: 3 + environment: + <<: *s3-env + DATABASE_URL_FILE: /run/secrets/database_url + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-900} + SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-60} + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: unless-stopped + + lb: + image: haproxy:3.2.4-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + networks: [edge] + depends_on: + api: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8405/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: unless-stopped + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: + s3data: + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_conf: + file: ./secrets/valkey.conf + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + seaweedfs_s3_config: + file: ./secrets/seaweedfs_s3.json diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..232780ff --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,50 @@ +# 2026-10-28T12:44:18Z +$ make test-attachments +PASS png+jpeg upload, member download is byte-identical, metadata stable +PASS message with only attachments, order kept, history shows links, re-use by other member +PASS idempotency: missing attachment_ids == [], other attachments -> 409 +PASS exactly 40 000 000 px accepted +PASS 10 MiB + 1 byte with Content-Length -> 413 +PASS 10 MiB + 1 byte chunked (no Content-Length) -> 413 while streaming +PASS decompression bomb 8000x5001 px (~1 KB/40 MB raw) -> 413 +PASS RGB 20000x20000 header -> 413 before decode +PASS truncated PNG -> 415 +PASS PNG with damaged IDAT byte (CRC) -> 415 +PASS truncated JPEG -> 415 +PASS PNG bytes declared image/jpeg -> 415 +PASS JPEG bytes declared image/png -> 415 +PASS text declared image/png -> 415 +PASS GIF declared image/png -> 415 +PASS SVG/HTML declared image/png -> 415 +PASS PNG magic + garbage -> 415 +PASS empty body -> 415 +PASS Content-Type application/octet-stream -> 415 +PASS Content-Type multipart/form-data -> 415 +PASS filename missing -> 400 +PASS filename path ../ -> 400 +PASS filename slash -> 400 +PASS filename backslash -> 400 +PASS filename NUL -> 400 +PASS filename newline -> 400 +PASS filename bidi RLO -> 400 +PASS filename dot-dot -> 400 +PASS filename blank -> 400 +PASS filename 256 chars -> 400 +PASS outsider uploads into chat -> 404 +PASS outsider metadata -> 404 +PASS outsider content -> 404 +PASS unknown attachment -> 404 +PASS invalid attachment UUID -> 400 +PASS no session -> 401 +PASS variant thumbnail not produced in lab4 -> 404 +PASS unknown variant -> 400 +PASS variant=original explicit -> 200 +PASS attachment of another chat (same author) -> 404 +PASS outsider links chat attachment into own chat -> 404 +PASS unknown attachment id in message -> 404 +PASS 11 attachment ids -> 400 +PASS duplicate attachment ids -> 400 +PASS blank text without attachments -> 400 +PASS attachment_ids wrong type -> 400 +PASS DB composite FK rejects cross-chat link even bypassing API +PASS: 47 attachment cases diff --git a/evidence/bootstrap-idempotent.txt b/evidence/bootstrap-idempotent.txt new file mode 100644 index 00000000..3cbe946b --- /dev/null +++ b/evidence/bootstrap-idempotent.txt @@ -0,0 +1,6 @@ +# 2026-10-28T12:44:12Z repeat bootstrap on existing volumes +$ docker compose run --rm --no-deps s3-init +INFO s3-bootstrap: bucket messenger-attachments already exists +INFO s3-bootstrap: anonymous list/get denied: bucket is private +$ docker compose run --rm --no-deps migrate +INFO migrate: migrations up to date: 0 applied now, 3 before diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 00000000..97400f83 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,10 @@ +# 2026-10-28T12:51:16Z +$ make test-consistency +PASS S3 down: upload 503 in 0.79s, content 503, metadata 200 (ready), messages 201, ready 200, no pending row left +PASS S3 back (healthy per compose): first download byte-identical after 0.0s +PASS aborted upload (1.1 of 5 MiB sent): no S3 object, no metadata row +injected: orphan object, stale pending row + object, stale pending row without object; waiting 63s +dry-run: {"pending_found": 2, "orphans_found": 1, "objects_scanned": 28, "duration_ms": 186.1} +real: {"pending_deleted": 2, "orphans_deleted": 1, "objects_scanned": 27, "duration_ms": 203.2} +repeat: {"pending_found": 0, "orphans_found": 0} +PASS sweeper: dry-run reports only; real pass removes stale pending rows/objects and orphans; fresh and ready kept; repeat is a no-op diff --git a/evidence/failover-load/stop-api-1.json b/evidence/failover-load/stop-api-1.json new file mode 100644 index 00000000..b298fdac --- /dev/null +++ b/evidence/failover-load/stop-api-1.json @@ -0,0 +1,153 @@ +{ + "profile": "closed loop, 8 workers, each: POST message (retry same key) + GET history; client timeout 2 s", + "action": "scripts/stop-container.sh messenger-lab4-api-1", + "action_started_s": 5.03, + "action_finished_s": 7.7, + "action_returncode": 0, + "action_output": [ + "2026-10-28T12:54:50Z stopped messenger-lab4-api-1 instance=api-a442bb8f state=exited" + ], + "elapsed_s": 25.2, + "requests": 8858, + "successful": 8858, + "errors": 0, + "error_ratio": 0.0, + "successful_rps": 351.5, + "latency_ms_success": { + "p50": 14.3, + "p95": 60.7, + "p99": 96.4, + "max": 306.4 + }, + "slow_over_500ms_after_action": 0, + "error_window_s": null, + "recovery_s_after_action": 0.0, + "instances_before": { + "api-e3ad2cfc": 1089, + "api-a442bb8f": 1090 + }, + "instances_last_5s": { + "api-e3ad2cfc": 1508 + }, + "timeline_ok_err_per_s": { + "0": [ + 372, + 0 + ], + "1": [ + 319, + 0 + ], + "2": [ + 489, + 0 + ], + "3": [ + 497, + 0 + ], + "4": [ + 488, + 0 + ], + "5": [ + 276, + 0 + ], + "6": [ + 256, + 0 + ], + "7": [ + 266, + 0 + ], + "8": [ + 261, + 0 + ], + "9": [ + 383, + 0 + ], + "10": [ + 365, + 0 + ], + "11": [ + 354, + 0 + ], + "12": [ + 433, + 0 + ], + "13": [ + 451, + 0 + ], + "14": [ + 293, + 0 + ], + "15": [ + 424, + 0 + ], + "16": [ + 401, + 0 + ], + "17": [ + 365, + 0 + ], + "18": [ + 282, + 0 + ], + "19": [ + 309, + 0 + ], + "20": [ + 351, + 0 + ], + "21": [ + 266, + 0 + ], + "22": [ + 180, + 0 + ], + "23": [ + 302, + 0 + ], + "24": [ + 471, + 0 + ], + "25": [ + 4, + 0 + ] + }, + "docker_stats_mid": [ + "messenger-lab4-api-2\t50.60%\t146.7MiB / 384MiB", + "messenger-lab4-lb-1\t12.87%\t16.84MiB / 128MiB", + "messenger-lab4-postgres-1\t22.94%\t59.46MiB / 512MiB", + "messenger-lab4-s3-1\t0.59%\t172.8MiB / 768MiB", + "messenger-lab4-sweeper-1\t0.00%\t58.7MiB / 128MiB", + "messenger-lab4-valkey-1\t1.58%\t10.67MiB / 96MiB" + ], + "integrity": { + "stored_messages": 4429, + "confirmed_writes": 4429, + "duplicate_ids": 0, + "duplicate_texts": 0, + "confirmed_missing": 0 + } +} diff --git a/evidence/failover-load/stop-api-2.json b/evidence/failover-load/stop-api-2.json new file mode 100644 index 00000000..ebef6853 --- /dev/null +++ b/evidence/failover-load/stop-api-2.json @@ -0,0 +1,153 @@ +{ + "profile": "closed loop, 8 workers, each: POST message (retry same key) + GET history; client timeout 2 s", + "action": "scripts/stop-container.sh messenger-lab4-api-2", + "action_started_s": 5.0, + "action_finished_s": 6.07, + "action_returncode": 0, + "action_output": [ + "2026-10-28T12:55:28Z stopped messenger-lab4-api-2 instance=api-e3ad2cfc state=exited" + ], + "elapsed_s": 25.02, + "requests": 6790, + "successful": 6790, + "errors": 0, + "error_ratio": 0.0, + "successful_rps": 271.4, + "latency_ms_success": { + "p50": 16.1, + "p95": 69.7, + "p99": 149.3, + "max": 1033.6 + }, + "slow_over_500ms_after_action": 16, + "error_window_s": null, + "recovery_s_after_action": 2.97, + "instances_before": { + "api-e3ad2cfc": 867, + "api-f80d6006": 869 + }, + "instances_last_5s": { + "api-f80d6006": 1379 + }, + "timeline_ok_err_per_s": { + "0": [ + 396, + 0 + ], + "1": [ + 364, + 0 + ], + "2": [ + 369, + 0 + ], + "3": [ + 295, + 0 + ], + "4": [ + 309, + 0 + ], + "5": [ + 300, + 0 + ], + "6": [ + 15, + 0 + ], + "7": [ + 21, + 0 + ], + "8": [ + 332, + 0 + ], + "9": [ + 285, + 0 + ], + "10": [ + 175, + 0 + ], + "11": [ + 297, + 0 + ], + "12": [ + 329, + 0 + ], + "13": [ + 324, + 0 + ], + "14": [ + 295, + 0 + ], + "15": [ + 246, + 0 + ], + "16": [ + 287, + 0 + ], + "17": [ + 184, + 0 + ], + "18": [ + 341, + 0 + ], + "19": [ + 246, + 0 + ], + "20": [ + 111, + 0 + ], + "21": [ + 324, + 0 + ], + "22": [ + 276, + 0 + ], + "23": [ + 323, + 0 + ], + "24": [ + 343, + 0 + ], + "25": [ + 3, + 0 + ] + }, + "docker_stats_mid": [ + "messenger-lab4-api-1\t46.83%\t85.9MiB / 384MiB", + "messenger-lab4-lb-1\t11.21%\t16.87MiB / 128MiB", + "messenger-lab4-postgres-1\t19.60%\t62.48MiB / 512MiB", + "messenger-lab4-s3-1\t0.63%\t182.8MiB / 768MiB", + "messenger-lab4-sweeper-1\t0.00%\t58.7MiB / 128MiB", + "messenger-lab4-valkey-1\t1.64%\t10.69MiB / 96MiB" + ], + "integrity": { + "stored_messages": 3395, + "confirmed_writes": 3395, + "duplicate_ids": 0, + "duplicate_texts": 0, + "confirmed_missing": 0 + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..0f1a5433 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,14 @@ +# 2026-10-28T12:50:25Z (immediately after the persistence restart) +## run 1 +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 0, "first_recovery_seconds": 3.254} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +## run 2 +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 1.805} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +## stop-one.sh log +2026-10-28T12:50:30Z stopped messenger-lab4-api-1 (1765636c87f6) instance=api-32eb21c0 state=exited +2026-10-28T12:50:54Z stopped messenger-lab4-api-1 (1765636c87f6) instance=api-53bf8995 state=exited diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..72f0398f --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,13 @@ +# 2026-10-28T12:44:24Z +$ student-tests/idempotency_test.py +ok - round 1: 32 parallel requests, 1 message, replicas {'api-657e97b1': 16, 'api-4d7d8490': 16} +ok - round 2: 32 parallel requests, 1 message, replicas {'api-657e97b1': 16, 'api-4d7d8490': 16} +ok - round 3: 32 parallel requests, 1 message, replicas {'api-657e97b1': 16, 'api-4d7d8490': 16} +ok - round 4: 32 parallel requests, 1 message, replicas {'api-657e97b1': 16, 'api-4d7d8490': 16} +ok - round 5: 32 parallel requests, 1 message, replicas {'api-657e97b1': 16, 'api-4d7d8490': 16} +ok - different body with same key -> 409, original replays unchanged +ok - same key independent per user and per chat +ok - missing/invalid Idempotency-Key -> 400, max-length key accepted +ok - replay needs a valid session and chat access (404 outsider, 401 revoked) +ok - DB rows in chat: 8, history items: 8, no duplicate texts +PASS: shared idempotency diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..57a3f96c --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.033, + "concurrency": 4, + "successful_requests": 14146, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 471.014, + "successful_latency_ms": { + "p50": 7.387, + "p95": 12.989, + "p99": 32.668 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/memory/ram-10-noise.json b/evidence/memory/ram-10-noise.json new file mode 100644 index 00000000..3e9041c5 --- /dev/null +++ b/evidence/memory/ram-10-noise.json @@ -0,0 +1,49 @@ +{ + "label": "body-in-RAM, 10x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 10, + "statuses": { + "201": 10 + }, + "per_instance": { + "api-41dde5be": 5, + "api-1312fe88": 5 + }, + "wall_seconds": 3.89, + "latency_s": { + "p50": 3.8085, + "max": 3.883 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 63.9, + "peak_anon": 147.0, + "delta_anon": 83.1, + "baseline_current": 69.1, + "peak_current": 154.1, + "peak_file_cache": 0.0 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.0, + "peak_anon": 141.3, + "delta_anon": 77.3, + "baseline_current": 69.5, + "peak_current": 149.8, + "peak_file_cache": 0.0 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 324.5, + "peak_anon": 426.0, + "delta_anon": 101.5, + "baseline_current": 700.3, + "peak_current": 768.0, + "peak_file_cache": 368.7 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "16777216", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/ram-20-noise.json b/evidence/memory/ram-20-noise.json new file mode 100644 index 00000000..1bf73aa5 --- /dev/null +++ b/evidence/memory/ram-20-noise.json @@ -0,0 +1,49 @@ +{ + "label": "body-in-RAM, 20x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 20, + "statuses": { + "201": 20 + }, + "per_instance": { + "api-7a770f14": 10, + "api-d37e4357": 10 + }, + "wall_seconds": 6.04, + "latency_s": { + "p50": 5.1419999999999995, + "max": 6.039 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 63.9, + "peak_anon": 201.6, + "delta_anon": 137.6, + "baseline_current": 69.3, + "peak_current": 212.7, + "peak_file_cache": 0.0 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.1, + "peak_anon": 206.6, + "delta_anon": 142.5, + "baseline_current": 69.2, + "peak_current": 218.6, + "peak_file_cache": 0.0 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 326.7, + "peak_anon": 449.2, + "delta_anon": 122.5, + "baseline_current": 688.2, + "peak_current": 768.0, + "peak_file_cache": 373.1 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "16777216", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/ram-40-noise.json b/evidence/memory/ram-40-noise.json new file mode 100644 index 00000000..872904b9 --- /dev/null +++ b/evidence/memory/ram-40-noise.json @@ -0,0 +1,49 @@ +{ + "label": "body-in-RAM, 40x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 40, + "statuses": { + "201": 40 + }, + "per_instance": { + "api-3c932187": 20, + "api-a0b8ffd2": 20 + }, + "wall_seconds": 9.86, + "latency_s": { + "p50": 7.7555, + "max": 9.859 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 64.0, + "peak_anon": 257.9, + "delta_anon": 194.0, + "baseline_current": 69.3, + "peak_current": 270.4, + "peak_file_cache": 0.0 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.0, + "peak_anon": 284.8, + "delta_anon": 220.8, + "baseline_current": 69.6, + "peak_current": 296.2, + "peak_file_cache": 0.0 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 333.1, + "peak_anon": 433.5, + "delta_anon": 100.4, + "baseline_current": 736.0, + "peak_current": 768.0, + "peak_file_cache": 387.7 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "16777216", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/stream-10-noise.json b/evidence/memory/stream-10-noise.json new file mode 100644 index 00000000..03d5e34d --- /dev/null +++ b/evidence/memory/stream-10-noise.json @@ -0,0 +1,49 @@ +{ + "label": "stream, 10x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 10, + "statuses": { + "201": 10 + }, + "per_instance": { + "api-4c952443": 5, + "api-6a870ec1": 5 + }, + "wall_seconds": 2.73, + "latency_s": { + "p50": 2.5810000000000004, + "max": 2.729 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 64.0, + "peak_anon": 102.5, + "delta_anon": 38.5, + "baseline_current": 69.4, + "peak_current": 155.3, + "peak_file_cache": 45.9 + }, + "messenger-lab4-api-2": { + "baseline_anon": 63.9, + "peak_anon": 96.3, + "delta_anon": 32.4, + "baseline_current": 69.2, + "peak_current": 147.2, + "peak_file_cache": 45.9 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 324.8, + "peak_anon": 379.5, + "delta_anon": 54.7, + "baseline_current": 741.6, + "peak_current": 768.0, + "peak_file_cache": 384.9 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "1048576", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/stream-10-pixels.json b/evidence/memory/stream-10-pixels.json new file mode 100644 index 00000000..728c7850 --- /dev/null +++ b/evidence/memory/stream-10-pixels.json @@ -0,0 +1,49 @@ +{ + "label": "stream, 10x 40Mpx", + "kind": "pixels", + "file_bytes": 116709, + "concurrency": 10, + "statuses": { + "201": 10 + }, + "per_instance": { + "api-6d8b2e85": 5, + "api-982fda78": 5 + }, + "wall_seconds": 3.17, + "latency_s": { + "p50": 2.0375, + "max": 3.167 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 63.9, + "peak_anon": 225.8, + "delta_anon": 161.8, + "baseline_current": 69.9, + "peak_current": 233.3, + "peak_file_cache": 0.0 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.0, + "peak_anon": 227.8, + "delta_anon": 163.7, + "baseline_current": 68.7, + "peak_current": 235.2, + "peak_file_cache": 0.0 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 323.6, + "peak_anon": 324.7, + "delta_anon": 1.1, + "baseline_current": 698.3, + "peak_current": 700.4, + "peak_file_cache": 346.3 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "1048576", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/stream-20-noise.json b/evidence/memory/stream-20-noise.json new file mode 100644 index 00000000..df481f72 --- /dev/null +++ b/evidence/memory/stream-20-noise.json @@ -0,0 +1,49 @@ +{ + "label": "stream, 20x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 20, + "statuses": { + "201": 20 + }, + "per_instance": { + "api-0b7cc085": 10, + "api-bc5d6770": 10 + }, + "wall_seconds": 5.2, + "latency_s": { + "p50": 4.5794999999999995, + "max": 5.193 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 63.9, + "peak_anon": 96.0, + "delta_anon": 32.1, + "baseline_current": 69.1, + "peak_current": 196.8, + "peak_file_cache": 91.7 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.0, + "peak_anon": 101.4, + "delta_anon": 37.3, + "baseline_current": 69.4, + "peak_current": 200.3, + "peak_file_cache": 91.7 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 323.7, + "peak_anon": 421.9, + "delta_anon": 98.2, + "baseline_current": 733.9, + "peak_current": 768.0, + "peak_file_cache": 399.3 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "1048576", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/memory/stream-40-noise.json b/evidence/memory/stream-40-noise.json new file mode 100644 index 00000000..756dca1e --- /dev/null +++ b/evidence/memory/stream-40-noise.json @@ -0,0 +1,49 @@ +{ + "label": "stream, 40x noise", + "kind": "noise", + "file_bytes": 9617088, + "concurrency": 40, + "statuses": { + "201": 40 + }, + "per_instance": { + "api-426f6ce1": 20, + "api-a3f1971e": 20 + }, + "wall_seconds": 9.24, + "latency_s": { + "p50": 6.92, + "max": 9.231 + }, + "memory_mib": { + "messenger-lab4-api-1": { + "baseline_anon": 63.9, + "peak_anon": 113.0, + "delta_anon": 49.1, + "baseline_current": 69.6, + "peak_current": 268.4, + "peak_file_cache": 146.8 + }, + "messenger-lab4-api-2": { + "baseline_anon": 64.0, + "peak_anon": 110.0, + "delta_anon": 45.9, + "baseline_current": 69.4, + "peak_current": 269.5, + "peak_file_cache": 146.8 + }, + "messenger-lab4-s3-1": { + "baseline_anon": 330.5, + "peak_anon": 403.0, + "delta_anon": 72.5, + "baseline_current": 759.4, + "peak_current": 768.0, + "peak_file_cache": 403.0 + } + }, + "api_env": { + "UPLOAD_SPOOL_BYTES": "1048576", + "UPLOAD_CONCURRENCY": "16", + "DECODE_CONCURRENCY": "1" + } +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..3424c91e --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,31 @@ +# 2026-10-28T12:49:29Z +## before +/messenger-lab4-api-1 started=2026-10-28T12:49:07.33200605Z status=running +/messenger-lab4-api-2 started=2026-10-28T12:49:07.446368967Z status=running +/messenger-lab4-lb-1 started=2026-10-28T12:49:07.548902801Z status=running +/messenger-lab4-migrate-1 started=2026-10-28T12:48:43.194072928Z status=exited +/messenger-lab4-postgres-1 started=2026-10-28T12:48:40.625567385Z status=running +/messenger-lab4-s3-1 started=2026-10-28T12:48:40.624710052Z status=running +/messenger-lab4-s3-init-1 started=2026-10-28T12:48:54.221527128Z status=exited +/messenger-lab4-sweeper-1 started=2026-10-28T12:49:07.330576425Z status=running +/messenger-lab4-valkey-1 started=2026-10-28T12:48:40.623424302Z status=running +messenger-lab4_pgdata +messenger-lab4_s3data +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +## after +/messenger-lab4-api-1 started=2026-10-28T12:50:19.211923042Z status=running +/messenger-lab4-api-2 started=2026-10-28T12:50:19.32544125Z status=running +/messenger-lab4-lb-1 started=2026-10-28T12:50:19.431133834Z status=running +/messenger-lab4-migrate-1 started=2026-10-28T12:49:55.030515461Z status=exited +/messenger-lab4-postgres-1 started=2026-10-28T12:49:52.442082085Z status=running +/messenger-lab4-s3-1 started=2026-10-28T12:49:52.439265918Z status=running +/messenger-lab4-s3-init-1 started=2026-10-28T12:50:06.033067342Z status=exited +/messenger-lab4-sweeper-1 started=2026-10-28T12:50:19.21224375Z status=running +/messenger-lab4-valkey-1 started=2026-10-28T12:49:52.44208621Z status=running +messenger-lab4_pgdata +messenger-lab4_s3data +## s3-init on restart +WARNING s3-bootstrap: write probe failed (attempt 1/30): ReadTimeoutError +INFO s3-bootstrap: write path warm: put/get/delete probe 0.008s diff --git a/evidence/s3-oom-before-fix.txt b/evidence/s3-oom-before-fix.txt new file mode 100644 index 00000000..f848fd3a --- /dev/null +++ b/evidence/s3-oom-before-fix.txt @@ -0,0 +1,23 @@ +# 2026-10-28, найдено экспериментом upload_memory.py (до исправления; тексты выводов сокращены вручную) +# Конфигурация ДО: s3 memory limit 512M, без -*.concurrentUploadLimitMB, без GOMEMLIMIT, +# в API нет ограничения параллельных S3 PUT (UPLOAD_CONCURRENCY=16 на реплику). + +$ python3 student-tests/upload_memory.py --concurrency 20 # 20 x 9.6 MiB noise PNG + "statuses": {"201": 4, "503": 16} + +API logs: + 8 WARNING:messenger.s3:s3 put failed: EndpointConnectionError + 8 WARNING:messenger.s3:compensating delete failed; sweeper will retry later + +S3 memory.current (cgroup), выборка каждые 0.25 с во время прогона: + 511MiB ... 511MiB (упор в лимит 512M), затем процесс weed исчезает без panic, + контейнер перезапускается (RestartCount растёт), в логе при старте: + W volume_checking.go:399 volume 6: repaired the unindexed tail of /data/messenger-attachments_6.dat: re-indexed 1 needle(s) + +Вывод: OOM kill SeaweedFS под 20 параллельными 10 MiB PUT. Данных не потеряно: API ответил 503, +pending-строки удалены компенсацией, а оставшиеся объекты/строки подбирает sweeper. + +# ПОСЛЕ исправления (S3_PUT_CONCURRENCY=4 в каждой реплике, -s3/-filer/-volume.concurrentUploadLimitMB=128, +# GOMEMLIMIT=450MiB, limit 768M) — см. evidence/memory/stream-{20,40}-noise.json: + 20 параллельно: 201 x 20, S3 anon peak 421.9 MiB + 40 параллельно: 201 x 40, S3 anon peak 403.0 MiB, RestartCount s3 = 0 diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 00000000..bc44300f --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,18 @@ +# 2026-10-28T12:44:21Z +$ make test-s3-security +object key: originals/, unrelated to attachment id and filename + anonymous_get_object 403 + anonymous_list_bucket 403 + anonymous_list_all 403 + api_create_bucket AccessDenied + api_put_bucket_policy_public AccessDenied + api_put_object_acl_public AccessDenied + api_delete_bucket AccessDenied + api_get_own_bucket_object ALLOWED + s3:9333 master UI ConnectionRefusedError + s3:8888 filer UI ConnectionRefusedError + s3:8080 volume ConnectionRefusedError + s3:9000 S3 API OPEN + s3 published host ports: none ({}) + lb -> s3:9000 (edge network): unreachable (exit status 1) +PASS: private bucket, least-privilege API credentials, no exposed admin interfaces diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..9399c28f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-10-28T12:44:14Z +$ make test +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stop-one.txt b/evidence/stop-one.txt new file mode 100644 index 00000000..bb1277cd --- /dev/null +++ b/evidence/stop-one.txt @@ -0,0 +1,4 @@ +2026-10-28T12:53:49Z stopped messenger-lab4-api-1 (1765636c87f6) instance=api-a882f11d state=exited +2026-10-28T12:53:59Z stopped messenger-lab4-api-2 (2977947aaf31) instance=api-04d7b1b1 state=exited +2026-10-28T12:54:50Z stopped messenger-lab4-api-1 instance=api-a442bb8f state=exited +2026-10-28T12:55:28Z stopped messenger-lab4-api-2 instance=api-e3ad2cfc state=exited diff --git a/evidence/topology.txt b/evidence/topology.txt new file mode 100644 index 00000000..bf4aefb1 --- /dev/null +++ b/evidence/topology.txt @@ -0,0 +1,45 @@ +# 2026-10-28T12:43:46Z clean volumes +$ docker compose --env-file .env config --quiet && docker compose --env-file .env up -d --build --wait + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-valkey-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-valkey-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-sweeper-1 Healthy + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-lb-1 Healthy +$ docker compose ps +NAME IMAGE STATUS PORTS +messenger-lab4-api-1 mtusi-messenger:lab4 Up 5 seconds (healthy) 8080/tcp +messenger-lab4-api-2 mtusi-messenger:lab4 Up 5 seconds (healthy) 8080/tcp +messenger-lab4-lb-1 haproxy:3.2.4-alpine Up 5 seconds (healthy) 127.0.0.1:8088->8080/tcp +messenger-lab4-migrate-1 mtusi-messenger:lab4 Exited (0) 12 seconds ago +messenger-lab4-postgres-1 postgres:17.6-alpine3.22 Up 15 seconds (healthy) +messenger-lab4-s3-1 chrislusf/seaweedfs:4.51 Up 15 seconds (healthy) +messenger-lab4-s3-init-1 mtusi-messenger:lab4 Exited (0) 6 seconds ago +messenger-lab4-sweeper-1 mtusi-messenger:lab4 Up 5 seconds (healthy) +messenger-lab4-valkey-1 valkey/valkey:8.1.4-alpine3.22 Up 15 seconds (healthy) +$ docker compose logs migrate s3-init +INFO migrate: applying 0001_init +INFO migrate: applying 0002_idempotency +INFO migrate: applying 0003_attachments +INFO migrate: migrations up to date: 3 applied now, 0 before +INFO s3-bootstrap: bucket messenger-attachments created +INFO s3-bootstrap: anonymous list/get denied: bucket is private +$ limits +/messenger-lab4-api-1 cpus=500000000 mem=402653184 networks=messenger-lab4_data messenger-lab4_edge +/messenger-lab4-api-2 cpus=500000000 mem=402653184 networks=messenger-lab4_data messenger-lab4_edge +/messenger-lab4-lb-1 cpus=500000000 mem=134217728 networks=messenger-lab4_edge +/messenger-lab4-postgres-1 cpus=1000000000 mem=536870912 networks=messenger-lab4_data +/messenger-lab4-s3-1 cpus=1000000000 mem=805306368 networks=messenger-lab4_data +/messenger-lab4-sweeper-1 cpus=250000000 mem=134217728 networks=messenger-lab4_data +/messenger-lab4-valkey-1 cpus=250000000 mem=100663296 networks=messenger-lab4_data diff --git a/evidence/upload-failover.txt b/evidence/upload-failover.txt new file mode 100644 index 00000000..d3c3f90e --- /dev/null +++ b/evidence/upload-failover.txt @@ -0,0 +1,7 @@ +# 2026-10-28T12:53:44Z +$ make test-upload-failover +round 1: uploaded via api-a882f11d (messenger-lab4-api-1); 2026-10-28T12:53:49Z stopped messenger-lab4-api-1 (1765636c87f6) instance=api-a882f11d state=exited +round 1: 10/10 downloads byte-identical via ['api-04d7b1b1'] +round 2: uploaded via api-04d7b1b1 (messenger-lab4-api-2); 2026-10-28T12:53:59Z stopped messenger-lab4-api-2 (2977947aaf31) instance=api-04d7b1b1 state=exited +round 2: 10/10 downloads byte-identical via ['api-a442bb8f'] +PASS: attachment downloaded through another replica after its uploader stopped (2 rounds) diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..e510c5ad --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,52 @@ +global + log stdout format raw local0 info + maxconn 2048 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold other 2s + hold refused 2s + hold nx 2s + hold timeout 2s + hold obsolete 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s + timeout client 30s + # uploads up to 10 MiB + image decode; inactivity timeouts, not total request time + timeout server 30s + timeout http-request 10s + timeout queue 5s + timeout check 1s + retries 3 + option redispatch 1 + retry-on conn-failure empty-response + unique-id-format %[uuid()] + http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API replica failed\",\"request_id\":\"%[unique-id]\"}}" + http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no healthy API replica\",\"request_id\":\"%[unique-id]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API replica timeout\",\"request_id\":\"%[unique-id]\"}}" + +frontend public + bind :8080 + default_backend api + +frontend monitor + bind 127.0.0.1:8405 + monitor-uri /lb-health + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 8 api:8080 diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..a46a618c --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,49 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); + +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..b58ca4a8 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$') +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..a5efcfe9 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,49 @@ +-- lab4: message text may be empty when the message carries attachments (checked in the API). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); + +-- Metadata only; bytes live in the private S3 bucket under object_key (assigned by the server). +-- 'pending' = intent recorded before S3 PUT, invisible to the API; the sweeper removes stale ones. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + object_key text NOT NULL, + status text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000) +); + +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +-- chat_id is repeated so that both composite FKs guarantee: an attachment is linked only within its own chat. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, position), + CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id), + CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cf6d418a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +pydantic==2.13.5 +asyncpg==0.32.0 +redis==8.1.0 +argon2-cffi==25.1.0 +boto3==1.43.120 +pillow==12.0.0 diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..754e71dd --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +mkdir -p secrets + +gen() { python3 -c 'import secrets; print(secrets.token_hex(24))'; } +keyid() { python3 -c 'import secrets,string; a=string.ascii_uppercase+string.digits; print("".join(secrets.choice(a) for _ in range(20)))'; } +once() { [ -f "secrets/$1" ] || { "$2" > "secrets/$1"; echo "created secrets/$1"; }; } + +once postgres_password gen +PG_PASS=$(cat secrets/postgres_password) +printf 'postgresql://messenger:%s@postgres:5432/messenger\n' "$PG_PASS" > secrets/database_url + +if [ ! -f secrets/valkey.conf ]; then + { + printf 'requirepass %s\n' "$(gen)" + printf 'save ""\nappendonly no\n' + printf 'maxmemory 64mb\nmaxmemory-policy volatile-lru\n' + } > secrets/valkey.conf + echo "created secrets/valkey.conf" +fi +VK_PASS=$(sed -n 's/^requirepass //p' secrets/valkey.conf) +printf 'redis://:%s@valkey:6379/0\n' "$VK_PASS" > secrets/cache_url + +# S3 (SeaweedFS): admin identity only for bucket bootstrap; API identity limited to Read/Write/List of one bucket. +BUCKET=${S3_BUCKET:-messenger-attachments} +once s3_admin_access_key_id keyid +once s3_admin_secret_access_key gen +once s3_access_key_id keyid +once s3_secret_access_key gen +python3 - "$BUCKET" > secrets/seaweedfs_s3.json <<'PY' +import json, sys +r = lambda n: open("secrets/" + n).read().strip() +b = sys.argv[1] +print(json.dumps({"identities": [ + {"name": "bootstrap-admin", + "credentials": [{"accessKey": r("s3_admin_access_key_id"), "secretKey": r("s3_admin_secret_access_key")}], + "actions": ["Admin", "Read", "List", "Write"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": r("s3_access_key_id"), "secretKey": r("s3_secret_access_key")}], + "actions": ["Read:" + b, "Write:" + b, "List:" + b]}, +]}, indent=1)) +PY + +chmod 644 secrets/* +echo "secrets ready in ./secrets (not tracked by git)" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f1127649 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose ps --format 'table {{.Name}}\t{{.Status}}' +docker compose stop lb api sweeper +docker compose stop valkey postgres s3 +docker compose up -d --no-build --wait api lb sweeper +docker compose ps --format 'table {{.Name}}\t{{.Status}}' diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..4f8a3207 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-build --no-recreate --wait api +docker compose ps api --format 'table {{.Name}}\t{{.Status}}' diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..eaf0d0be --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +N=$(docker compose ps -q api | wc -l | tr -d ' ') +[ "$N" -ge 1 ] || { echo "api is not running" >&2; exit 1; } +OLD=$(docker compose ps -q api) + +docker compose build api +docker compose run --rm --no-deps migrate + +wait_healthy() { + i=0 + until [ "$(docker inspect -f '{{.State.Health.Status}}' "$1")" = healthy ]; do + i=$((i + 1)); [ "$i" -le 60 ] || { echo "replica $1 not healthy" >&2; exit 1; } + sleep 1 + done +} + +for old in $OLD; do + before=$(docker compose ps -q api | sort) + docker compose up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1 + new=$(docker compose ps -q api | sort | grep -vxF "$before" || true) + [ -n "$new" ] || { echo "no new replica started" >&2; exit 1; } + wait_healthy "$new" + sleep 2 + printf '%s new %s healthy; stopping old %s\n' "$(date -u +%H:%M:%S)" \ + "$(docker inspect -f '{{.Name}}' "$new")" "$(docker inspect -f '{{.Name}}' "$old")" + docker stop --time 8 "$old" >/dev/null + docker rm "$old" >/dev/null +done +docker compose ps api --format 'table {{.Name}}\t{{.Image}}\t{{.Status}}' diff --git a/scripts/stop-container.sh b/scripts/stop-container.sh new file mode 100755 index 00000000..eb5b312f --- /dev/null +++ b/scripts/stop-container.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +name=${1:?usage: stop-container.sh -api-N} +case "$name" in *-api-*) ;; *) echo "refusing: $name is not an API replica" >&2; exit 2;; esac +alias=$(docker exec "$name" cat /tmp/instance-id 2>/dev/null || echo "?") +docker stop --time 8 "$name" >/dev/null +printf '%s stopped %s instance=%s state=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$name" "$alias" \ + "$(docker inspect -f '{{.State.Status}}' "$name")" | tee -a evidence/stop-one.txt diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8d979ae3 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +log=evidence/stop-one.txt +for cid in $(docker compose ps -q api); do + alias=$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null || true) + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + name=$(docker inspect -f '{{.Name}}' "$cid" | sed 's#^/##') + docker stop --time 8 "$cid" >/dev/null + state=$(docker inspect -f '{{.State.Status}}' "$cid") + printf '%s stopped %s (%s) instance=%s state=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$name" "$(printf %.12s "$cid")" "$alias" "$state" | tee -a "$log" + exit 0 + fi +done +echo "no running api container reports instance id $TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/student-tests/attachments_test.py b/student-tests/attachments_test.py new file mode 100644 index 00000000..b5fb6d35 --- /dev/null +++ b/student-tests/attachments_test.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Negative/positive suite for attachments (lab4, level 4). User errors must never become 500.""" +import hashlib +import os +import uuid +from urllib.parse import quote + +from stack import SAMPLE_JPG, SAMPLE_PNG, png, png_header_only, psql +from client import Client, arguments, check, run + +LIMIT = 10 * 1024 * 1024 + + +def main(): + args = arguments("Attachment validation, ACL and linking through the LB") + c = Client(args.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + other = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "Другой чат", "member_ids": [alice["id"], eve["id"]]}, schema="Chat")["json"] + results = [] + + def case(name, fn): + fn() + results.append(name) + print("PASS", name) + + def upload(data, ctype="image/png", filename="f.png", actor=alice, target=chat, expected=201): + path = "/api/v1/chats/" + target["id"] + "/attachments" + if filename is not None: + path += "?filename=" + quote(filename, safe="") + headers = {"Content-Type": ctype} if ctype else {} + r = c.call("POST", path, expected, actor=actor, raw=data, headers=headers, schema="Attachment" if expected == 201 else None) + return r["json"] + + def post(body, actor=alice, target=chat, expected=201): + return c.call("POST", "/api/v1/chats/" + target["id"] + "/messages", expected, actor=actor, body=body, + headers={"Idempotency-Key": uuid.uuid4().hex}, schema="Message" if expected == 201 else None)["json"] + + def download(att_id, actor=bob, suffix="/content"): + return c.call("GET", "/api/v1/attachments/" + att_id + suffix, actor=actor) + + # --- positive ----------------------------------------------------------------------------- + att_png = upload(SAMPLE_PNG, filename="фото 😀.png") + att_jpg = upload(SAMPLE_JPG, "image/jpeg", "sample.jpg") + + def roundtrip(): + check(att_png["filename"] == "фото 😀.png" and att_png["status"] == "ready" and att_png["variants"] == [], "png metadata") + check(att_jpg["content_type"] == "image/jpeg" and att_jpg["sha256"] == hashlib.sha256(SAMPLE_JPG).hexdigest(), "jpeg metadata") + for att, data, ctype in ((att_png, SAMPLE_PNG, "image/png"), (att_jpg, SAMPLE_JPG, "image/jpeg")): + d = download(att["id"]) + check(d["bytes"] == data and d["headers"].get_content_type() == ctype, "download differs") + check(d["headers"].get("X-Content-Type-Options") == "nosniff", "nosniff missing") + meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + check(meta == att, "metadata via GET differs from upload response") + case("png+jpeg upload, member download is byte-identical, metadata stable", roundtrip) + + def message_links(): + m = post({"text": "", "attachment_ids": [att_jpg["id"], att_png["id"]]}) + check(m["attachment_ids"] == [att_jpg["id"], att_png["id"]] and m["text"] == "", "order/text of attachment message") + plain = post({"text": "без вложений"}) + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + hist = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]["items"] + by_id = {x["id"]: x for x in hist} + check(by_id[m["id"]]["attachment_ids"] == m["attachment_ids"] and by_id[plain["id"]]["attachment_ids"] == [], "history lost links") + post({"text": "второй раз то же вложение", "attachment_ids": [att_png["id"]]}, actor=bob) + case("message with only attachments, order kept, history shows links, re-use by other member", message_links) + + def idempotency(): + key = uuid.uuid4().hex + h = {"Idempotency-Key": key} + path = "/api/v1/chats/" + chat["id"] + "/messages" + a = c.call("POST", path, 201, actor=alice, body={"text": "x"}, headers=h, schema="Message")["json"] + b = c.call("POST", path, 201, actor=alice, body={"attachment_ids": [], "text": "x"}, headers=h, schema="Message")["json"] + check(a == b, "missing attachment_ids and [] must be the same request") + c.call("POST", path, 409, actor=alice, body={"text": "x", "attachment_ids": [att_png["id"]]}, headers=h) + case("idempotency: missing attachment_ids == [], other attachments -> 409", idempotency) + + def boundary_pixels(): + att = upload(png(8000, 5000, 0), filename="40mpx.png") # exactly 40 000 000 px, gray + check(att["size_bytes"] > 0, "boundary image rejected") + case("exactly 40 000 000 px accepted", boundary_pixels) + + # --- size / pixels ------------------------------------------------------------------------ + big = b"\x89PNG\r\n\x1a\n" + os.urandom(LIMIT + 1 - 8) + case("10 MiB + 1 byte with Content-Length -> 413", lambda: upload(big, expected=413)) + + def chunked(): + def gen(): + for i in range(0, len(big), 256 * 1024): + yield big[i:i + 256 * 1024] + upload(gen(), expected=413) + case("10 MiB + 1 byte chunked (no Content-Length) -> 413 while streaming", chunked) + case("decompression bomb 8000x5001 px (~1 KB/40 MB raw) -> 413", lambda: upload(png(8000, 5001, 0), expected=413)) + case("RGB 20000x20000 header -> 413 before decode", lambda: upload(png_header_only(20000, 20000), expected=413)) + + # --- corrupted / MIME --------------------------------------------------------------------- + case("truncated PNG -> 415", lambda: upload(SAMPLE_PNG[:len(SAMPLE_PNG) // 2], expected=415)) + flipped = bytearray(SAMPLE_PNG) + flipped[len(flipped) // 2] ^= 0xFF + case("PNG with damaged IDAT byte (CRC) -> 415", lambda: upload(bytes(flipped), expected=415)) + case("truncated JPEG -> 415", lambda: upload(SAMPLE_JPG[:len(SAMPLE_JPG) // 2], "image/jpeg", expected=415)) + case("PNG bytes declared image/jpeg -> 415", lambda: upload(SAMPLE_PNG, "image/jpeg", expected=415)) + case("JPEG bytes declared image/png -> 415", lambda: upload(SAMPLE_JPG, "image/png", expected=415)) + case("text declared image/png -> 415", lambda: upload(b"" * 10, expected=415)) + case("GIF declared image/png -> 415", lambda: upload(b"GIF89a" + bytes(100), expected=415)) + case("SVG/HTML declared image/png -> 415", lambda: upload(b"", expected=415)) + case("PNG magic + garbage -> 415", lambda: upload(b"\x89PNG\r\n\x1a\n" + os.urandom(2000), expected=415)) + case("empty body -> 415", lambda: upload(b"", expected=415)) + case("Content-Type application/octet-stream -> 415", lambda: upload(SAMPLE_PNG, "application/octet-stream", expected=415)) + case("Content-Type multipart/form-data -> 415", lambda: upload(SAMPLE_PNG, "multipart/form-data; boundary=x", expected=415)) + + # --- filename ----------------------------------------------------------------------------- + for label, name in (("missing", None), ("path ../", "../etc/passwd.png"), ("slash", "a/b.png"), ("backslash", "a\\b.png"), + ("NUL", "a\x00.png"), ("newline", "a\nb.png"), ("bidi RLO", "gpj‮.exe"), ("dot-dot", ".."), + ("blank", " "), ("256 chars", "a" * 252 + ".png")): + case("filename %s -> 400" % label, lambda name=name: upload(SAMPLE_PNG, filename=name, expected=400)) + + # --- access ------------------------------------------------------------------------------- + case("outsider uploads into chat -> 404", lambda: upload(SAMPLE_PNG, actor=eve, expected=404)) + case("outsider metadata -> 404", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"], 404, actor=eve)) + case("outsider content -> 404", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"] + "/content", 404, actor=eve)) + case("unknown attachment -> 404", lambda: c.call("GET", "/api/v1/attachments/" + str(uuid.uuid4()), 404, actor=alice)) + case("invalid attachment UUID -> 400", lambda: c.call("GET", "/api/v1/attachments/not-a-uuid", 400, actor=alice)) + case("no session -> 401", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"] + "/content", 401)) + case("variant thumbnail not produced in lab4 -> 404", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"] + "/content?variant=thumbnail", 404, actor=bob)) + case("unknown variant -> 400", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"] + "/content?variant=raw", 400, actor=bob)) + case("variant=original explicit -> 200", lambda: c.call("GET", "/api/v1/attachments/" + att_png["id"] + "/content?variant=original", 200, actor=bob)) + + # --- linking ------------------------------------------------------------------------------ + foreign = upload(SAMPLE_PNG, actor=alice, target=other) + case("attachment of another chat (same author) -> 404", lambda: post({"text": "x", "attachment_ids": [foreign["id"]]}, expected=404)) + case("outsider links chat attachment into own chat -> 404", lambda: post({"text": "x", "attachment_ids": [att_png["id"]]}, actor=eve, target=other, expected=404)) + case("unknown attachment id in message -> 404", lambda: post({"text": "x", "attachment_ids": [str(uuid.uuid4())]}, expected=404)) + case("11 attachment ids -> 400", lambda: post({"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, expected=400)) + case("duplicate attachment ids -> 400", lambda: post({"text": "x", "attachment_ids": [att_png["id"], att_png["id"]]}, expected=400)) + case("blank text without attachments -> 400", lambda: post({"text": " "}, expected=400)) + case("attachment_ids wrong type -> 400", lambda: post({"text": "x", "attachment_ids": "abc"}, expected=400)) + + def db_guard(): + msg = post({"text": "db guard"}) + sql = ("BEGIN; INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) VALUES ('%s','%s','%s',0); ROLLBACK;" + % (msg["id"], foreign["id"], chat["id"])) + try: + psql(sql) + except RuntimeError as exc: + check("foreign key" in str(exc), "unexpected DB error") + return + check(False, "DB accepted a cross-chat attachment link") + case("DB composite FK rejects cross-chat link even bypassing API", db_guard) + + print("PASS: %d attachment cases" % len(results)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/chaos_load.py b/student-tests/chaos_load.py new file mode 100644 index 00000000..882ffc96 --- /dev/null +++ b/student-tests/chaos_load.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import json +import math +import shlex +import subprocess +import sys +import threading +import time +import uuid +from collections import Counter, defaultdict +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def pct(values, p): + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) if values else None + + +def docker_stats(): + ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split() + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}", *ids], + capture_output=True, text=True) + return sorted(out.stdout.splitlines()) + + +def main(): + def configure(p): + p.add_argument("--duration", type=float, default=25) + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--action", default=None, help="command run once at --action-at seconds") + p.add_argument("--action-at", type=float, default=5) + p.add_argument("--output", default=None) + args = arguments("Load with an operator action in the middle", configure) + boot = Client(args.base_url, timeout=10) + alice, bob, _eve, chat = boot.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + + events = [] + confirmed = [] + lock = threading.Lock() + t0 = time.monotonic() + stop_at = t0 + args.duration + action_info = {} + + def worker(wid): + c = Client(args.base_url, timeout=2) + key, text, n = None, None, 0 + while time.monotonic() < stop_at: + if key is None: + key = uuid.uuid4().hex + text = "chaos-%d-%d-%s" % (wid, n, key[:8]) + for kind in ("write", "read"): + start = time.monotonic() + try: + if kind == "write": + r = c.call("POST", path, 201, actor=alice, body={"text": text}, headers={"Idempotency-Key": key}, schema="Message") + else: + r = c.call("GET", path + "?limit=20", actor=bob, schema="MessagePage") + ok, inst = True, r["headers"].get("X-Instance-Id") + except Failure: + ok, inst = False, None + end = time.monotonic() + with lock: + events.append((start - t0, kind, ok, end - start, inst)) + if ok and kind == "write": + confirmed.append(r["json"]) + if kind == "write" and ok: + key, n = None, n + 1 + if not ok: + time.sleep(0.05) + + def action(): + time.sleep(args.action_at) + action_info["started"] = time.monotonic() - t0 + proc = subprocess.run(shlex.split(args.action), cwd=ROOT, capture_output=True, text=True) + action_info["finished"] = time.monotonic() - t0 + action_info["returncode"] = proc.returncode + action_info["output_tail"] = proc.stdout.strip().splitlines()[-6:] + + stats_mid = [] + threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] + if args.action: + threads.append(threading.Thread(target=action)) + for t in threads: + t.start() + time.sleep(min(args.duration / 2, args.action_at + 3)) + stats_mid = docker_stats() + for t in threads: + t.join() + elapsed = time.monotonic() - t0 + + ok_lat = sorted(e[3] for e in events if e[2]) + errors = [e for e in events if not e[2]] + a0 = action_info.get("started") + timeline = defaultdict(lambda: [0, 0]) + for e in events: + timeline[int(e[0])][0 if e[2] else 1] += 1 + report = { + "profile": "closed loop, %d workers, each: POST message (retry same key) + GET history; client timeout 2 s" % args.concurrency, + "action": args.action, "action_started_s": a0 and round(a0, 2), "action_finished_s": action_info.get("finished") and round(action_info["finished"], 2), + "action_returncode": action_info.get("returncode"), "action_output": action_info.get("output_tail"), + "elapsed_s": round(elapsed, 2), "requests": len(events), "successful": len(ok_lat), "errors": len(errors), + "error_ratio": round(len(errors) / len(events), 5) if events else None, + "successful_rps": round(len(ok_lat) / elapsed, 1), + "latency_ms_success": {"p50": pct(ok_lat, .50), "p95": pct(ok_lat, .95), "p99": pct(ok_lat, .99), "max": pct(ok_lat, 1)}, + "slow_over_500ms_after_action": sum(1 for e in events if e[2] and e[3] > .5 and a0 is not None and e[0] >= a0), + "error_window_s": [round(errors[0][0] - (a0 or 0), 2), round(max(e[0] + e[3] for e in errors) - (a0 or 0), 2)] if errors else None, + "recovery_s_after_action": None, + "instances_before": dict(Counter(e[4] for e in events if e[2] and (a0 is None or e[0] < a0))), + "instances_last_5s": dict(Counter(e[4] for e in events if e[2] and e[0] > elapsed - 5)), + "timeline_ok_err_per_s": {k: v for k, v in sorted(timeline.items())}, + "docker_stats_mid": stats_mid, + } + if a0 is not None: + bad_ends = [e[0] + e[3] for e in events if e[0] + e[3] >= a0 and (not e[2] or e[3] > 1)] + report["recovery_s_after_action"] = round(max(bad_ends) - a0, 2) if bad_ends else 0.0 + + items, cursor = [], None + while True: + q = path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else "") + page = boot.call("GET", q, actor=bob, schema="MessagePage")["json"] + items += page["items"] + cursor = page["next_cursor"] + if cursor is None: + break + ids = Counter(m["id"] for m in items) + texts = Counter(m["text"] for m in items) + stored = {m["id"]: m for m in items} + report["integrity"] = { + "stored_messages": len(items), "confirmed_writes": len(confirmed), + "duplicate_ids": sum(1 for n in ids.values() if n > 1), "duplicate_texts": sum(1 for n in texts.values() if n > 1), + "confirmed_missing": sum(1 for m in confirmed if stored.get(m["id"]) != m), + } + text = json.dumps(report, ensure_ascii=False, indent=1) + print(text) + if args.output: + Path(args.output).write_text(text + "\n") + check(report["integrity"]["duplicate_texts"] == 0 and report["integrity"]["confirmed_missing"] == 0, "data integrity violated") + check(action_info.get("returncode", 0) == 0, "action failed") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/consistency_test.py b/student-tests/consistency_test.py new file mode 100644 index 00000000..668427ac --- /dev/null +++ b/student-tests/consistency_test.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Failures between S3 and PostgreSQL: S3 outage, aborted upload, and the sweeper with injected faults. + +Modifies the compose stack: stops/starts the s3 service and writes synthetic rows/objects. Takes ~1.5 min +(the sweeper refuses grace < 60 s, so injected "old" objects must really age). +""" +import json +import socket +import time +import uuid +from urllib.parse import urlparse + +from stack import SAMPLE_PNG, api_python, count_objects, dc, psql +from client import Client, Failure, arguments, check, run + +GRACE = 60 + + +def pending_count(): + return int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")) + + +def put_object(key): + api_python("import io; from app.objects import ObjectStore; s=ObjectStore(); " + "s._s3.put_object(Bucket=s.bucket, Key=%r, Body=io.BytesIO(b'injected'))" % key) + + +def object_exists(key): + return api_python("from app.objects import ObjectStore; s=ObjectStore(); " + "print(any(k == %r for k, _ in s.list_originals()))" % key) == "True" + + +def insert_pending(chat_id, owner_id, key, age_seconds): + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + "object_key, status, created_at, updated_at) VALUES ('%s','%s','%s','injected.png','image/png',8,'%s',1,1,'%s'," + "'pending', now() - make_interval(secs => %d), now() - make_interval(secs => %d))" + % (uuid.uuid4(), chat_id, owner_id, "0" * 64, key, age_seconds, age_seconds)) + + +def row_exists(key): + return psql("SELECT count(*) FROM attachments WHERE object_key = '%s'" % key) == "1" + + +def sweep(*flags): + out = dc("run", "--rm", "--no-deps", "-T", "sweeper", "python", "-m", "app.sweeper", "--once", "--grace", str(GRACE), *flags).stdout + report = json.loads(out.strip().splitlines()[-1]) + check("skipped" not in report, "sweeper lock is held") + return report + + +def main(): + args = arguments("S3/PostgreSQL consistency under failures") + c = Client(args.base_url, timeout=15) + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=x.png" + ready = c.call("POST", path, 201, actor=alice, raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment")["json"] + ready_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % ready["id"]) + + # 1. S3 outage ------------------------------------------------------------------------------- + pend0 = pending_count() + dc("stop", "s3") + try: + t = time.monotonic() + c.call("POST", path, 503, actor=alice, raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}) + upload_s = time.monotonic() - t + c.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", 503, actor=bob) + meta = c.call("GET", "/api/v1/attachments/" + ready["id"], actor=bob, schema="Attachment")["json"] + c.message(alice, chat, "S3 лежит, а сообщения работают") + c.call("GET", "/health/ready", schema="Health") + check(pending_count() == pend0, "failed upload left a pending row") + print("PASS S3 down: upload 503 in %.2fs, content 503, metadata 200 (%s), messages 201, ready 200, no pending row left" + % (upload_s, meta["status"])) + finally: + dc("start", "s3") + dc("up", "-d", "--no-build", "--wait", "s3") + t = time.monotonic() + while True: + try: + r = c.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", [200, 503], actor=bob) + except Failure: + r = {"status": None} + if r["status"] == 200: + check(r["bytes"] == SAMPLE_PNG, "bytes changed after S3 restart") + break + check(time.monotonic() - t < 30, "S3 did not recover") + time.sleep(1) + print("PASS S3 back (healthy per compose): first download byte-identical after %.1fs" % (time.monotonic() - t)) + + # 2. Client aborts in the middle of the body ---------------------------------------------------- + objects0, pend0 = count_objects(), pending_count() + u = urlparse(c.base) + s = socket.create_connection((u.hostname, u.port or 80), timeout=5) + s.sendall(("POST /api/v1/chats/%s/attachments?filename=abort.png HTTP/1.1\r\nHost: %s\r\nAuthorization: Bearer %s\r\n" + "Content-Type: image/png\r\nContent-Length: %d\r\n\r\n" % (chat["id"], u.hostname, alice["token"], 5 * 1024 * 1024)).encode()) + s.sendall(SAMPLE_PNG + bytes(1024 * 1024)) + s.close() + time.sleep(2) + check(count_objects() == objects0 and pending_count() == pend0, "aborted upload left an object or a row") + print("PASS aborted upload (1.1 of 5 MiB sent): no S3 object, no metadata row") + + # 3. Sweeper: inject inconsistencies on both sides ----------------------------------------------- + # The background sweeper (grace 900 s) would clean the injected 1-hour-old rows during the wait; pause it + # and run single passes with grace 60 s instead. + dc("stop", "sweeper") + try: + sweeper_scenario(c, alice, bob, chat, ready, ready_key) + finally: + dc("start", "sweeper") + + +def sweeper_scenario(c, alice, bob, chat, ready, ready_key): + old_orphan = "originals/" + uuid.uuid4().hex # object without row (failed compensation) + old_pending_obj = "originals/" + uuid.uuid4().hex # crash after PUT, before COMMIT ready + old_pending_noobj = "originals/" + uuid.uuid4().hex # crash before PUT + put_object(old_orphan) + put_object(old_pending_obj) + insert_pending(chat["id"], alice["id"], old_pending_obj, 3600) + insert_pending(chat["id"], alice["id"], old_pending_noobj, 3600) + print("injected: orphan object, stale pending row + object, stale pending row without object; waiting %ds" % (GRACE + 3)) + time.sleep(GRACE + 3) + fresh_orphan = "originals/" + uuid.uuid4().hex # look like uploads still in progress + fresh_pending = "originals/" + uuid.uuid4().hex + put_object(fresh_orphan) + put_object(fresh_pending) + insert_pending(chat["id"], alice["id"], fresh_pending, 0) + + dry = sweep("--dry-run") + print("dry-run:", json.dumps({k: dry[k] for k in ("pending_found", "orphans_found", "objects_scanned", "duration_ms")})) + check(dry["pending_found"] >= 2 and dry["orphans_found"] >= 1, "dry-run did not detect injected faults") + check(old_orphan in dry["orphan_keys_sample"] or dry["orphans_found"] > 5, "orphan key not reported") + check(object_exists(old_orphan) and row_exists(old_pending_obj), "dry-run changed something") + + real = sweep() + print("real:", json.dumps({k: real[k] for k in ("pending_deleted", "orphans_deleted", "objects_scanned", "duration_ms")})) + check(not object_exists(old_orphan), "orphan object survived") + check(not object_exists(old_pending_obj) and not row_exists(old_pending_obj), "stale pending row/object survived") + check(not row_exists(old_pending_noobj), "stale pending row without object survived") + check(object_exists(fresh_orphan) and object_exists(fresh_pending) and row_exists(fresh_pending), "fresh upload was deleted") + check(object_exists(ready_key), "ready object was deleted") + d = c.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob) + check(d["bytes"] == SAMPLE_PNG, "ready attachment damaged") + + again = sweep() + print("repeat:", json.dumps({k: again[k] for k in ("pending_found", "orphans_found")})) + check(again["pending_found"] == 0 and again["orphans_found"] == 0, "second pass is not a no-op") + print("PASS sweeper: dry-run reports only; real pass removes stale pending rows/objects and orphans; " + "fresh and ready kept; repeat is a no-op") + # fresh_* are left on purpose: the background sweeper removes them after ORPHAN_GRACE_SECONDS. + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1fef4570428b19332d6f2b004ada77f0a412078d GIT binary patch literal 3973 zcmc&$2Ut^C8ohzgQH&%+5t5N2ZN`R-AV^du6j=>T0a0TC3yvZRB2pekB~BC~Dk6%6 zB4WcRs6hlHA|QwaM5QXd2&jZm5=h?LhqLaC?0h?A_uIYs?z_pmfA0Iw`O7;<9nu1( zt+aQt2N(MqmdNXON1O`@KG9?3w5L?%rff zAcw=q<1olmKt}ykz$dv(ZQH(Mr~m$--~%BCLk}G}dMuiG{3q7wGiT$^C7e&Zkd}Tq z<4We$YuB@L?%uo4%YBgdr08jJN$In)@|QKQUcafWt8Zv+Ywzgn>h5{pD;yLJ4Ude9 zC1Vq~FhKriSm^a*{a-onGm_!V)f zcM4VX&D{nFTlQ6}Y8fnQH55)j`;6?50b~3nWd8*A3$6yBE{{QjCyxUZAi8dx#@}I! zHBMuJ|7_+pi`T-75RkGMUS)@X;#x&(i^rW(zE75Fcr9)?!Or#m2hA=1U&1w zdW#1W_%W28DF`^@f&gI<^wmkjH}p_<8tCGM@T$$^J0I38mM;!6Y+ozl&eRZT{M&#AS8QxtwLNs{rSj1Ih58P99lAF& zvA%ml32zyd3#H*nd*fSOvm|~qecrq{?^)P3|NWA?O#yE$B&VvWgr1~UD&7V2WzxNC z%i@PGMXI$!(#HsBqRC1kKFK5CIReH*5U@uTtplvB`1fkStc#)zJ!{W`>Q+Fq8Lx*r z!%N#~5}jb~xH1g^>Dv5V1MaeM0<0k6XqGfT&gez6MvrC8?MWA_m&v`^*HJ!J z$yX5a;Z4v-Be>mm^e_TC5g>br0Gs@((QIR}UZFR=uw1 zWH2Om!4v$}?fmXyx>nE)rQEyKSsa%KF|2PZmhmbzDwM7EPF0+b>7G)RJg<$-_7mb? zT-g~b+==5EovQs-L%k?F_PIEDFOk8${oL|^&GjoL{$mhFLZ(HP>6d37Z{|rZJxUeQ zN{H?%-#omZ?UK8oa8HI)S$B8SesO#3+2<4%X(6N2Fu^7IJ<*EgfQb#Bd>t_T8 zc~(K&Hc3Jw&PH~GQCy1Fy?YVyY=k}~eAC@{E%G$4rTH0Aq7U{x15Rb8t2@Eg)GeVc zL0no+-u;66&KIrE@7Wao;hfRbtRqJlmNpEtj=Wx2_u$JrOx*jyiYl0r{-Jv8fKAc?ClzXCG@G}-r`PmRV#LmCb#~C^y$M=1b9*{A z9)(scnHGxO^qM7pKrOES7@o7rdVU(qM0p|!{`Y|@DQIPlWUDXao?PGP zK=11kxErNhZ;KyUmqbM~DUGw|=S-r4L$!~|)8=33E2<4yy(XbDJLXx#Wz8gdmgnfW$+6MsW23N0C#<%{VnD|s+T%{RaC{{Kma=06 z#-;~j{YK*9%~ZUnAbGwk6%RZsBGK1CTFLe)ehauztro*`X_4EteYEipr%l$I^yl_zrGZIN9V|c z6<7hU;-u9OH+5Gz0(MNn`Q6SDVvW1JX86$VIgz}})*4y5%+k*^EVWS`wLFxU9^)j- z2&f%e`s2CmOxDErzK=HkXz^AF?xF1=YGN*qB>T`p+GDiLg5LFo>Rk5Kh@W9iiXb>i z&EA&_X;Dk`-tHo<>+c!=G2wYs{OrHQDsP!(X)q7Ff3&cjHIgTuYk2GWmb1iq`iWX8 zn=RO7(uPNW9CxJ>M487YfzCi@Q zZxPVQK!Fu%Jo*fbHPWEgXwv`Dy@GMD!Nk*G&)OOK`f-IH+^Sc7f2nFtO`dI__(aP? zzAF8r+o-nhixXiCE2UedrS{}FdHr0SjEJq*>KgnHXnN5bGm7s%zwg+XO0J4ciCsRE z&`B~C+Zl(Q;A?D5H`!8MzgpQe*T}!@%$a1x7E2r6Ov>(afm5ss-Jd>m<~zW&0oxqR zr4@lLxBAD2piT+NnHr?cWaN8qogW<`C#$>FvWZ=}-j|)92!u6OSG>j6#hxK10Yv67 zA*53M_9ExUf^WsHH4nHcV{es6dx8Tz!YJP9yF@n*rK|G739S)!#Y=AIRD%dF||%j z*R=Y(tRiB=<*qd5%$lyl58v6a`{Ps9*IBh{ou z%U)(JT1skMdB;<0q<6)q-4#v-r)k#=II9nWHND2%=%frQE2pbEa1Kz;WzsRw+4!n zbY(}I0t)iksG2-h&_)I$*7?F+Kkc~a4-Q558{juqh(9n7)mAg zlBXh|%78)^X?*A}g(34Q1bl7fApP#sbOZ=uQB;xW1Kaup#_2M<*5)xRvmF(71?SPt zu0!O>W)t%D*93)j;-RbI)~MDzC50*|yd1JbD)<8|{z4*iWMnIv$ca7BB(Bq8tSmQRW}<&`ZeG*cWSDP82>j#lo5 z)q@-C-17K|u6_>^yY@yMQ?GC;3ERKsD%ahtwJFCWDU*?w(wWZl*w>b$v-UgV+KZ<* SJV9I9j= 2, "all requests hit one replica; LB not balancing") + stored = [m for m in history(c, bob, chat) if m["text"] == text] + check(len(stored) == 1, "history has %d copies" % len(stored)) + print("ok - round %d: %d parallel requests, 1 message, replicas %s" % (rnd + 1, args.parallel, dict(instances))) + + key = uuid.uuid4().hex + first = post(c, alice, chat, "оригинал", key)["json"] + for _ in range(4): + post(c, alice, chat, "другой текст", key, expected=409) + check(post(c, alice, chat, "оригинал", key)["json"] == first, "replay changed") + print("ok - different body with same key -> 409, original replays unchanged") + + other = post(c, bob, chat, "оригинал", key)["json"] + check(other["id"] != first["id"] and other["sender_id"] == bob["id"], "key leaked across users") + chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "второй", "member_ids": [alice["id"], bob["id"]]}, schema="Chat")["json"] + third = post(c, alice, chat2, "оригинал", key)["json"] + check(third["id"] != first["id"] and third["chat_id"] == chat2["id"], "key leaked across chats") + print("ok - same key independent per user and per chat") + + for bad in ("", "with space", "кириллица", "x" * 129, "a/b"): + try: + post(c, alice, chat, "bad key", bad, expected=400) + except UnicodeEncodeError: + pass + c.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 400, actor=alice, body={"text": "без ключа"}) + post(c, alice, chat, "ok", "A-Za-z0-9._:-" + "k" * 115) + print("ok - missing/invalid Idempotency-Key -> 400, max-length key accepted") + + post(c, eve, chat, "оригинал", key, expected=404) + sid = Client(args.base_url) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=bob) + post(sid, bob, chat, "оригинал", key, expected=401) + print("ok - replay needs a valid session and chat access (404 outsider, 401 revoked)") + + count = db_count(chat["id"]) + texts = Counter(m["text"] for m in history(c, alice, chat)) + check(all(n == 1 for t, n in texts.items() if t.startswith("одновременно")), "duplicates in history") + print("ok - DB rows in chat: %s, history items: %d, no duplicate texts" % (count, sum(texts.values()))) + print("PASS: shared idempotency") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/s3_security_test.py b/student-tests/s3_security_test.py new file mode 100644 index 00000000..2a4b0497 --- /dev/null +++ b/student-tests/s3_security_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Private bucket checked OUTSIDE the API, least-privilege API credentials, no exposed admin interfaces.""" +import json +import subprocess + +from stack import SAMPLE_PNG, api_python, dc, psql +from client import Client, arguments, check, run + +PROBE = r''' +import json, socket, urllib.request, urllib.error +from app import config +from app.objects import ObjectStore +from botocore.exceptions import ClientError +key = %r +s = ObjectStore() +out = {} +def anon(path): + try: + return urllib.request.urlopen(config.S3_ENDPOINT + path, timeout=3).status + except urllib.error.HTTPError as e: + return e.code +out["anonymous_get_object"] = anon("/" + s.bucket + "/" + key) +out["anonymous_list_bucket"] = anon("/" + s.bucket + "?list-type=2") +out["anonymous_list_all"] = anon("/") +def try_(name, fn): + try: + fn(); out[name] = "ALLOWED" + except ClientError as e: + out[name] = e.response["Error"]["Code"] +pub = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%%s/*"}]}' %% s.bucket +try_("api_create_bucket", lambda: s._s3.create_bucket(Bucket="evil-bucket")) +try_("api_put_bucket_policy_public", lambda: s._s3.put_bucket_policy(Bucket=s.bucket, Policy=pub)) +try_("api_put_object_acl_public", lambda: s._s3.put_object_acl(Bucket=s.bucket, Key=key, ACL="public-read")) +try_("api_delete_bucket", lambda: s._s3.delete_bucket(Bucket=s.bucket)) +try_("api_get_own_bucket_object", lambda: s._s3.head_object(Bucket=s.bucket, Key=key)) +def port(p): + try: + socket.create_connection(("s3", p), timeout=2).close(); return "OPEN" + except OSError as e: + return type(e).__name__ +out["s3:9333 master UI"] = port(9333) +out["s3:8888 filer UI"] = port(8888) +out["s3:8080 volume"] = port(8080) +out["s3:9000 S3 API"] = port(9000) +print(json.dumps(out)) +''' + + +def main(): + args = arguments("S3 privacy and least privilege") + c = Client(args.base_url, timeout=10) + alice, bob, eve, chat = c.bootstrap() + att = c.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=p.png", 201, actor=alice, raw=SAMPLE_PNG, + headers={"Content-Type": "image/png"}, schema="Attachment")["json"] + key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att["id"]) + check(key.startswith("originals/") and att["id"].replace("-", "") not in key and "p.png" not in key, + "object key must be server-generated, unrelated to id/filename") + print("object key: originals/, unrelated to attachment id and filename") + + r = json.loads(api_python(PROBE % key)) + for k, v in r.items(): + print(" %-32s %s" % (k, v)) + check(r["anonymous_get_object"] == 403 and r["anonymous_list_bucket"] == 403, "bucket is not private") + check(r["anonymous_list_all"] in (401, 403), "anonymous ListBuckets allowed") + for k in ("api_create_bucket", "api_put_bucket_policy_public", "api_put_object_acl_public", "api_delete_bucket"): + check(r[k] != "ALLOWED", k + " must be denied for API credentials") + check(r["api_get_own_bucket_object"] == "ALLOWED", "API cannot read its bucket") + for k in ("s3:9333 master UI", "s3:8888 filer UI", "s3:8080 volume"): + check(r[k] != "OPEN", k + " reachable from the network") + + ports = subprocess.run(["docker", "inspect", "-f", "{{json .NetworkSettings.Ports}}", dc("ps", "-q", "s3").stdout.strip()], + capture_output=True, text=True).stdout.strip() + check("HostPort" not in ports, "s3 publishes ports on the host") + print(" s3 published host ports: none (%s)" % ports) + lb = dc("exec", "-T", "lb", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://s3:9000/", check=False) + check(lb.returncode != 0, "LB (edge network) can reach S3") + print(" lb -> s3:9000 (edge network): unreachable (%s)" % (lb.stderr.strip().splitlines() or ["refused"])[-1]) + print("PASS: private bucket, least-privilege API credentials, no exposed admin interfaces") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/stack.py b/student-tests/stack.py new file mode 100644 index 00000000..f732b8a0 --- /dev/null +++ b/student-tests/stack.py @@ -0,0 +1,61 @@ +"""Helpers for student tests: synthetic images (stdlib only) and read-only inspection of the compose stack.""" +import os +import struct +import subprocess +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +SAMPLE_PNG = (ROOT / "tests/fixtures/sample.png").read_bytes() +SAMPLE_JPG = (FIXTURES / "sample.jpg").read_bytes() + + +def _chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, color_type=2, row=None): + """PNG writer: color_type 0 = gray (1 byte/px), 2 = RGB (3 bytes/px). row(y) returns raw row bytes.""" + bpp = {0: 1, 2: 3}[color_type] + blank = bytes(width * bpp) + comp = zlib.compressobj(6) + parts = [] + for y in range(height): + parts.append(comp.compress(b"\x00" + (row(y) if row else blank))) + parts.append(comp.flush()) + ihdr = struct.pack(">IIBBBBB", width, height, 8, color_type, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", b"".join(parts)) + _chunk(b"IEND", b"") + + +def png_header_only(width, height): + """Valid signature + IHDR claiming width x height, tiny IDAT: the pixel limit must fire before decoding.""" + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(bytes(1000))) + _chunk(b"IEND", b"") + + +def noise_png(width, height): + return png(width, height, 2, row=lambda y: os.urandom(width * 3)) + + +def dc(*args, check=True, timeout=120): + out = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, timeout=timeout) + if check and out.returncode != 0: + raise RuntimeError("docker compose %s failed: %s" % (args[0], out.stderr.strip()[-300:])) + return out + + +def psql(sql): + return dc("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAX", "-c", sql).stdout.strip() + + +def api_python(code, service="api"): + """Run Python inside an API container: it has the API's own (restricted) S3 credentials as secrets.""" + return dc("exec", "-T", service, "python", "-c", code).stdout.strip() + + +def count_objects(): + return int(api_python("from app.objects import ObjectStore; print(sum(1 for _ in ObjectStore().list_originals()))")) diff --git a/student-tests/upload_failover.py b/student-tests/upload_failover.py new file mode 100644 index 00000000..a2c260b0 --- /dev/null +++ b/student-tests/upload_failover.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Upload -> stop exactly the API replica that served the upload -> download through the other one. + +Repeats so that each replica is once the "uploader" that dies; restores the replica in between. +""" +import hashlib +import os +import subprocess +import time + +from stack import ROOT, SAMPLE_PNG, dc +from client import Client, Failure, arguments, check, run + + +def container_of(instance_id): + for cid in dc("ps", "-q", "api").stdout.split(): + alias = subprocess.run(["docker", "exec", cid, "cat", "/tmp/instance-id"], capture_output=True, text=True).stdout.strip() + if alias == instance_id: + return subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True).stdout.strip("/\n") + return None + + +def main(): + args = arguments("Attachment survives the loss of the replica that stored it") + c = Client(args.base_url, timeout=10) + alice, bob, eve, chat = c.bootstrap() + stopped = set() + for rnd in range(1, 3): + # Upload until the serving replica is one we have not stopped yet. + for _ in range(10): + r = c.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=r%d.png" % rnd, 201, actor=alice, + raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment") + uploader = r["headers"]["X-Instance-Id"] + name = container_of(uploader) + if name not in stopped: + break + att = r["json"] + env = dict(os.environ, TARGET_INSTANCE_ID=uploader) + out = subprocess.run([str(ROOT / "scripts/stop-one.sh")], env=env, capture_output=True, text=True) + check(out.returncode == 0, "stop-one.sh failed") + print("round %d: uploaded via %s (%s); %s" % (rnd, uploader, name, out.stdout.strip())) + served, ok = set(), 0 + deadline = time.monotonic() + 10 + while ok < 10: + try: + d = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob) + except Failure: + check(time.monotonic() < deadline, "download did not recover within 10 s") + time.sleep(0.2) + continue + check(hashlib.sha256(d["bytes"]).hexdigest() == att["sha256"], "bytes differ after failover") + served.add(d["headers"]["X-Instance-Id"]) + ok += 1 + check(uploader not in served, "stopped replica still served") + print("round %d: 10/10 downloads byte-identical via %s" % (rnd, sorted(served))) + stopped.add(name) + dc("up", "-d", "--no-build", "--no-recreate", "--wait", "api") + print("PASS: attachment downloaded through another replica after its uploader stopped (%d rounds)" % len(stopped)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..1379f7fd --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Experiment: N simultaneous uploads through the LB; peak memory of every API container (cgroup v2). + +Hypothesis: with streaming into a spooled temp file (UPLOAD_SPOOL_BYTES in RAM per upload) and DECODE_CONCURRENCY=1, +peak anonymous memory of a replica grows by O(spool x N + one decoded image), not O(file size x N). +""" +import argparse +import json +import statistics +import subprocess +import sys +import threading +import time +import uuid + +from stack import ROOT, dc, noise_png, png +from client import Client, Failure + +MB = 1024 * 1024 + + +def sample(cid): + out = subprocess.run(["docker", "exec", cid, "sh", "-c", + "cat /sys/fs/cgroup/memory.current; grep -E '^(anon|file) ' /sys/fs/cgroup/memory.stat"], + capture_output=True, text=True).stdout.split() + return {"current": int(out[0]), "anon": int(out[2]), "file": int(out[4])} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--base-url", default="http://localhost:8080") + p.add_argument("--concurrency", type=int, default=10) + p.add_argument("--kind", choices=["noise", "pixels"], default="noise", + help="noise: ~9.6 MiB incompressible PNG; pixels: 7000x5714 RGB (39.99 Mpx, tiny file, 120 MB decoded)") + p.add_argument("--label", default="") + p.add_argument("--output") + a = p.parse_args() + + data = noise_png(1790, 1790) if a.kind == "noise" else png(7000, 5714, 2) + c = Client(a.base_url, timeout=120) + alice, bob, eve, chat = c.bootstrap() + cids = dc("ps", "-q", "api").stdout.split() + dc("ps", "-q", "s3").stdout.split() + names = {cid: subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True).stdout.strip("/\n") + for cid in cids} + base = {cid: sample(cid) for cid in cids} + peaks = {cid: dict(base[cid]) for cid in cids} + stop = threading.Event() + + def poll(cid): + while not stop.is_set(): + s = sample(cid) + for k in s: + peaks[cid][k] = max(peaks[cid][k], s[k]) + + pollers = [threading.Thread(target=poll, args=(cid,), daemon=True) for cid in cids] + for t in pollers: + t.start() + time.sleep(1) + results = [] + lock = threading.Lock() + + def one(i): + t = time.monotonic() + try: + r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d.png" % (chat["id"], i), [201, 413, 503], actor=alice, + raw=data, headers={"Content-Type": "image/png"}) + status, inst = r["status"], r["headers"].get("X-Instance-Id") + except Failure as exc: + status, inst = str(exc)[:60], None + with lock: + results.append({"status": status, "seconds": round(time.monotonic() - t, 3), "instance": inst}) + + started = time.monotonic() + workers = [threading.Thread(target=one, args=(i,)) for i in range(a.concurrency)] + for w in workers: + w.start() + for w in workers: + w.join() + wall = time.monotonic() - started + time.sleep(1.5) + stop.set() + for t in pollers: + t.join() + + lat = sorted(r["seconds"] for r in results) + report = { + "label": a.label, "kind": a.kind, "file_bytes": len(data), "concurrency": a.concurrency, + "statuses": {str(s): sum(1 for r in results if r["status"] == s) for s in {r["status"] for r in results}}, + "per_instance": {i: sum(1 for r in results if r["instance"] == i) for i in {r["instance"] for r in results}}, + "wall_seconds": round(wall, 2), "latency_s": {"p50": statistics.median(lat), "max": lat[-1]}, + "memory_mib": {names[cid]: { + "baseline_anon": round(base[cid]["anon"] / MB, 1), "peak_anon": round(peaks[cid]["anon"] / MB, 1), + "delta_anon": round((peaks[cid]["anon"] - base[cid]["anon"]) / MB, 1), + "baseline_current": round(base[cid]["current"] / MB, 1), "peak_current": round(peaks[cid]["current"] / MB, 1), + "peak_file_cache": round(peaks[cid]["file"] / MB, 1)} for cid in cids}, + "api_env": {k: dc("exec", "-T", "api", "printenv", k, check=False).stdout.strip() or "default" + for k in ("UPLOAD_SPOOL_BYTES", "UPLOAD_CONCURRENCY", "DECODE_CONCURRENCY")}, + } + text = json.dumps(report, ensure_ascii=False, indent=1) + print(text) + if a.output: + (ROOT / a.output).write_text(text + "\n") + sys.exit(0 if all(r["status"] == 201 for r in results) else 1) + + +if __name__ == "__main__": + main() -- 2.54.0