forked from mtusi2026/mtusi2026lab4
Compare commits
5
Commits
main
...
lab4-verified
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48dea139a0 | ||
|
|
2c2f427750 | ||
|
|
1d8f8e6928 | ||
|
|
36d295e16c | ||
|
|
85433e20e4 |
@@ -17,4 +17,10 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
student-tests
|
||||
tests
|
||||
research
|
||||
REPORT.md
|
||||
SCHEMA.md
|
||||
requirements-dev.txt
|
||||
.DS_Store
|
||||
|
||||
@@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
PUBLIC_PORT=8084
|
||||
DATABASE_READ_AUTOCOMMIT=1
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Горбанюк / БВТ2301.
|
||||
Целевая оценка: 5. Трек: 2 — согласованность двух хранилищ.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Реализация и воспроизводимая последовательность запуска: [REPORT.md](../REPORT.md).
|
||||
Исходники приложения — src/, собственные проверки — student-tests/, результаты — evidence/.
|
||||
Сохранено 65 собственных проверочных случаев; условия и результаты конкретных запусков приведены в отчёте.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Контракт, исходное задание и публичные тесты сохранены. Секреты создаются локально и не входят в Git.
|
||||
Ограничения и использованные первичные источники перечислены в REPORT.md.
|
||||
|
||||
- [x] Реализован обязательный минимум и заявленный трек.
|
||||
- [x] Приведены команды запуска, проверок и безопасной остановки.
|
||||
- [x] Сохранены результаты проверок и ограничения выводов.
|
||||
- [x] Credentials и приватные ключи находятся вне репозитория.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Application integration
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.14.7'
|
||||
- run: python -m pip install uv==0.12.13
|
||||
- run: uv sync --locked
|
||||
- run: make bootstrap lint check
|
||||
- run: VCS_REF=$(git rev-parse HEAD) docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
|
||||
- run: PYTEST_ADDOPTS=--disable-warnings make test-student
|
||||
- run: make test
|
||||
- run: uv run --locked python scripts/verify_cluster.py --output /tmp/cluster-ci.json
|
||||
- if: always()
|
||||
run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml down
|
||||
@@ -0,0 +1 @@
|
||||
3.14.7
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
|
||||
WORKDIR /app
|
||||
RUN python -m pip install --no-cache-dir uv==0.12.13
|
||||
ENV UV_PYTHON_DOWNLOADS=never
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
|
||||
COPY src ./src
|
||||
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
|
||||
|
||||
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY migrations /app/migrations
|
||||
ARG VCS_REF=uncommitted
|
||||
LABEL org.opencontainers.image.revision=$VCS_REF
|
||||
ENV PATH="/app/.venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
APP_LOG_LEVEL=INFO
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://127.0.0.1:8084
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -28,3 +28,21 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down image lint test-student verify reconcile
|
||||
bootstrap:
|
||||
$(PYTHON) scripts/bootstrap.py
|
||||
up:
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env up -d --build --wait
|
||||
down:
|
||||
docker compose --env-file .env down
|
||||
image:
|
||||
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env build
|
||||
lint:
|
||||
uv run --locked ruff check src student-tests scripts/bootstrap.py scripts/cluster.py scripts/stop-one.py scripts/rolling-update.py scripts/verify_cluster.py scripts/verify_storage.py scripts/storage_lab.py research
|
||||
test-student:
|
||||
uv run --locked pytest --disable-warnings
|
||||
verify:
|
||||
uv run --locked python scripts/verify_cluster.py
|
||||
reconcile:
|
||||
docker compose --env-file .env run --rm reconciler
|
||||
|
||||
@@ -1,37 +1,387 @@
|
||||
# Отчёт — лабораторная 4
|
||||
# Лабораторная 4 — Горбанюк, БВТ2301
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
|
||||
## Запуск из чистого clone
|
||||
Целевая оценка: **5**, трек 2 «Согласованность двух хранилищ». Работа подготовлена
|
||||
для проверки. Название МР:
|
||||
`ЛР 4 — БВТ2301 — Горбанюк — на 5`. Проверенная версия отмечается `lab4-verified`.
|
||||
Окончательную оценку выставляет преподаватель, публичный smoke — только минимум.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Перенесена собственная реализация lab3: слои HTTP/domain/service/repository,
|
||||
сессии PostgreSQL, Argon2id, кэш Valkey, keyset-пагинация, две API за HAProxy,
|
||||
идемпотентные сообщения, отдельный migrate, action hooks. README, COURSE,
|
||||
CONTRIBUTING, contracts и публичные tests именно lab4 сохранены без изменений.
|
||||
Все заявленные результаты подтверждаются командами и evidence.
|
||||
|
||||
## Архитектура и решения
|
||||
Публичный форк: <https://gitea.privcloud.tech/Wemiliam/mtusi2026lab4>. Ветка решения: `solution/bvt2301-gorbanyuk`, целевая ветка преподавателя — `main`. Ветка опубликована; МР преподавателю: <https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/36>.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
## Запуск из отдельного чистого clone
|
||||
|
||||
## Доказательства критериев
|
||||
Нужны Docker Engine/Desktop + Compose v2, Git, Python 3.14.7 и uv 0.12.13.
|
||||
Go на host не нужен: S3-сервер и клиент собираются внутри Docker. Первая сборка
|
||||
требует доступа к реестрам, Python packages и Go modules и занимает заметное время.
|
||||
Повторные сборки используют слои/cache. Нужны свободные RAM/диск для компиляции
|
||||
и контейнеров; испытанный Docker-host: ARM64, 18 CPU, около 7.75 GiB RAM.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
```sh
|
||||
make bootstrap
|
||||
uv sync --locked
|
||||
make check
|
||||
uv run --locked python scripts/validate_openapi.py
|
||||
make up
|
||||
make test
|
||||
```
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Адрес клиента: **http://127.0.0.1:8084**, только локальные синтетические данные.
|
||||
bootstrap создаёт новый случайный пароль PostgreSQL и отдельные S3 credentials
|
||||
для root/API/reconciler/tests в игнорируемом secrets/, создаёт .env; существующие
|
||||
значения не заменяет. Каталог 0700, файлы 0644 для non-root Compose secret mounts.
|
||||
Не переносить чужие .env/secrets из lab3: у этой работы собственные volumes/проект.
|
||||
|
||||
## Эксперименты и отказы
|
||||
`make up` собирает образы, поднимает PostgreSQL и S3, выполняет отдельные migrate
|
||||
и bucket-init, затем API ×2/Valkey/HAProxy и ждёт readiness. Bucket-init идемпотентен:
|
||||
создание bucket с ignore-existing, запрет anonymous, повторное применение scoped
|
||||
policies/users с теми же локальными ключами. Все зависимости поднимаются здесь.
|
||||
Ни S3 API/console, ни PostgreSQL/Valkey, ни API напрямую в обычном compose
|
||||
не публикуются; единственный host-port — loopback HAProxy.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Для собственных тестов используется override. Он открывает только loopback
|
||||
PostgreSQL 55435, Valkey 56382 и S3 API 59004 и создаёт отдельные test bucket/identity:
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
```sh
|
||||
docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
|
||||
make test-student
|
||||
make lint
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# Вернуть остановленную реплику:
|
||||
docker compose --env-file .env up -d --no-build --wait
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
```
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Тестовая БД создаётся/удаляется отдельно; test bucket не совпадает с основным.
|
||||
API основной системы не имеет credentials тестового bucket. Не запускать
|
||||
разрушительные ручные тесты/эксперименты против данных вне этого учебного проекта.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Полная проверка реальных контейнеров создаёт новые синтетические объекты:
|
||||
|
||||
```sh
|
||||
make verify
|
||||
```
|
||||
|
||||
По умолчанию получается новое evidence/cluster-<timestamp>.json. Для сохранённого
|
||||
запуска использовалось `--output evidence/cluster.json`. Проверяется stop каждой
|
||||
API по очереди, реальный CID/PID, сообщения, rolling replacement и полный restart
|
||||
включая S3. Восстанавливается остановленная реплика. Hook stop-one.sh требует
|
||||
TARGET_INSTANCE_ID, публичный resilience.py передаёт его сам.
|
||||
|
||||
Обычная остановка сохраняет PostgreSQL/S3 volumes:
|
||||
|
||||
```sh
|
||||
make down
|
||||
```
|
||||
|
||||
Намеренно удалить **только учебные данные lab4**, если они больше не нужны:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down --volumes
|
||||
```
|
||||
|
||||
Скрипты не используют общий Docker prune и не останавливают другие лабораторные.
|
||||
|
||||
Дополнительно проверен отдельный чистый clone коммита `900182c`: новые credentials,
|
||||
новые PostgreSQL/S3 volumes, сборка образа с OCI revision этого коммита, все 65
|
||||
тестов, публичные smoke и persistence. Хеши всех runtime-модулей в образе совпали
|
||||
с исходниками clone и версией измерений. Временный стенд и только его новые
|
||||
volumes удалены; основные данные lab4 сохранены. Доказательства:
|
||||
[clean-clone.txt](evidence/clean-clone.txt), [clean-clone.json](evidence/clean-clone.json).
|
||||
Workflow application.yml подготовлен, удалённый runner здесь не запускался.
|
||||
|
||||
## Архитектура и версии
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[Client] --> LB[HAProxy :8084]
|
||||
LB --> A1[API 1]
|
||||
LB --> A2[API 2]
|
||||
A1 --> PG[(PostgreSQL: metadata, sessions, messages)]
|
||||
A2 --> PG
|
||||
A1 --> S3[(Private S3: original bytes)]
|
||||
A2 --> S3
|
||||
A1 --> V[Valkey: disposable User cache]
|
||||
A2 --> V
|
||||
R[Reconciler: dry-run/apply] --> PG
|
||||
R --> S3
|
||||
```
|
||||
|
||||
Детальная схема/протокол и ограничения — [SCHEMA.md](SCHEMA.md). Новая миграция 003:
|
||||
Attachment, внутренний storage_state, связи с message; составные FK запрещают
|
||||
вложение из другого чата. Сервер назначает UUID key originals/<UUID>, filename
|
||||
остаётся отображаемой строкой. Идентификация/ACL проверяются при upload, metadata,
|
||||
content и отправке сообщения. Участник может прикрепить вложение другого участника
|
||||
этого же чата. Посторонний получает 404. Публичный ответ не раскрывает object key.
|
||||
|
||||
Используется реляционная модель для связей/транзакций, объектное хранилище для
|
||||
байтов, stateless API и явный протокол восстановления. PostgreSQL — источник
|
||||
сессий, членства и результатов повторов. Кэш не даёт право доступа. Sticky
|
||||
sessions нет. Идемпотентность сообщения включает text и упорядоченные attachment_ids;
|
||||
отсутствующий список равен []. Message + links + результат ключа коммитятся вместе.
|
||||
|
||||
Версии: Python 3.14.7 (digest в Dockerfile), FastAPI 0.143.3, Pydantic 2.14.1,
|
||||
psycopg 3.3.6, Argon2id через argon2-cffi 25.1.0, boto3 1.43.109, Pillow 13.0.0.
|
||||
Точные транзитивные версии/hashes — uv.lock. PostgreSQL 18.3, Valkey 8.1.3-alpine,
|
||||
HAProxy 3.2.1-alpine. Образы S3 собираются из официального исходного релиза MinIO
|
||||
**RELEASE.2025-10-15T17-29-55Z** с исправлением CVE, mc — RELEASE.2025-08-13T08-35-41Z.
|
||||
Готовые upstream Docker Hub теги оказались недоступны, поэтому используется
|
||||
infra/s3.Dockerfile. Компилятор Go 1.26.7-alpine3.23 и runtime Alpine 3.23 закреплены
|
||||
digest. Community-репозиторий MinIO архивный: это воспроизводимая учебная версия,
|
||||
поддержка production здесь не заявляется. Go module source/tag и локальный image ID
|
||||
фиксируются в доказательствах, а не подменяются обещанием latest.
|
||||
Собранный через go install бинарник сообщает DEVELOPMENT.GOGET; это build label,
|
||||
а точная исходная pseudo-version/commit и SumDB hash извлечены из Go build info
|
||||
в [s3-source-version.txt](evidence/s3-source-version.txt).
|
||||
|
||||
Миграции выполняются одной отдельной службой под advisory lock с checksum.
|
||||
API сам DDL не запускает. Миграции 001/002 не менялись. 003 добавляет схему вложений,
|
||||
но внешние wire-ответы lab4 изменились: старые lab3 API нельзя смешивать с lab4 за LB.
|
||||
Rolling здесь — последовательное пересоздание процессов тем же образом, с
|
||||
ожиданием готовности между шагами, не произвольное обновление бизнес-схемы.
|
||||
|
||||
## Поток, безопасность и ограничение ресурсов
|
||||
|
||||
Upload — raw PNG/JPEG, не multipart. Auth/ACL, filename и Content-Length проверяются
|
||||
до чтения body. Опасные разделители, NUL/управляющие символы/суррогаты отклоняются.
|
||||
Внешний MIME должен совпасть с PNG/JPEG, обнаруженным Pillow. До полной распаковки
|
||||
проверяется площадь; verify и повторное load отсекают повреждённый поток. Байты
|
||||
никогда не перекодируются, SHA-256 считается инкрементально.
|
||||
|
||||
На API одновременно обрабатывается **один upload**, остальные ждут до 5 s,
|
||||
потом 429. Сам поток ограничен 15 s и фактическими **10 485 760 bytes**; объявленному
|
||||
Content-Length без проверки не доверяем. **40 000 000 пикселей** проверяются до load.
|
||||
Анонимный временный файл находится в tmpfs 32 MiB и закрывается при успехе,
|
||||
ошибке или отключении клиента. Это scratch, от него не зависит доступность файла
|
||||
после 201. RAM декодера зависит от ограниченной площади, не только сжатых bytes;
|
||||
один слот не даёт десяти декодерам одновременно занять память одной API.
|
||||
|
||||
Boto3 выполняет один PutObject ≤10 MiB: multipart transfer/автоматических retries
|
||||
нет, connect timeout 0.5 s, socket read timeout 2 s. Повтор сообщения защищён
|
||||
Idempotency-Key, повтор upload может создать новое вложение. SDK/SQL/декодирование
|
||||
выполняются вне event loop. Download передаёт 64 KiB chunks через API с ACL,
|
||||
Content-Length, истинным Content-Type, безопасным Content-Disposition и nosniff;
|
||||
redirect в S3 нет. Клиент проверяет итоговый SHA. Разрыв stream после начала ответа
|
||||
нельзя превратить в JSON-ошибку задним числом: такой GET нужно повторить.
|
||||
|
||||
API: 1 CPU / 512 MiB, readonly filesystem, UID 10001, cap_drop ALL, tmpfs 32 MiB,
|
||||
один Uvicorn worker. PostgreSQL: 1 CPU / 768 MiB; S3: 1 CPU / 768 MiB;
|
||||
Valkey: 0.5 CPU / 128 MiB; LB: 0.5 CPU / 128 MiB. API swap сверх лимита запрещён.
|
||||
API не публикует порт и не имеет container_name. Пулы/SQL timeout сохранены из lab3.
|
||||
SIGTERM: Uvicorn перестаёт принимать соединения, grace 5 s, Compose ждёт 7 s.
|
||||
Не подтвердившийся upload оставляет pending, который восстановит сверка.
|
||||
|
||||
HAProxy: round-robin, readiness каждые 500 ms, fall 2/rise 1, connect/check 500 ms,
|
||||
Docker DNS обновляется с hold valid 1 s; адреса не зашиты. LB client timeout 20 s,
|
||||
server timeout 30 s учитывают bounded upload. Readiness требует PostgreSQL и S3;
|
||||
при отказе S3 LB может временно исключить все API. Valkey не критичен.
|
||||
|
||||
Приватность/права: API identity умеет Get/Put только в originals/ основного bucket,
|
||||
List/GetLocation только этого bucket для readiness. Нет DeleteObject, bucket/admin
|
||||
операций или доступа к чужому bucket. Reconciler имеет List/Get/Delete своего
|
||||
prefix, без Put. Root передан только S3/bootstrap. Консоль привязана к loopback
|
||||
внутри S3-контейнера и не опубликована. Отдельный test identity ограничен test bucket.
|
||||
Secret values отсутствуют в Git, образах, evidence и логах; локальные secret mounts
|
||||
не заменяют защиту Docker-host и secret manager удалённого окружения.
|
||||
|
||||
## Согласованность и сверка: трек 2
|
||||
|
||||
201 возвращается после PUT и COMMIT ready. До PUT уже есть устойчивый pending.
|
||||
Это не двухфазный COMMIT между S3 и PostgreSQL. При отключении клиента до конца
|
||||
body не создаются ни intent, ни S3 объект; после intent любой неясный исход остаётся
|
||||
учтённым. Сверка отдельно восстанавливает запись или удаляет брошенное намерение.
|
||||
|
||||
```sh
|
||||
make reconcile # только dry-run
|
||||
# Изменения включаются явно:
|
||||
docker compose --env-file .env run --rm reconciler python -m messenger.reconcile --apply --grace-seconds 300
|
||||
```
|
||||
|
||||
Grace не может быть меньше 300 s. CLI работает постранично (S3 до 1000 объектов,
|
||||
PostgreSQL до 100 строк), preview до 100 actions, counters полные. Один reconciler
|
||||
на БД благодаря advisory lock, исправляемые строки блокируются FOR UPDATE.
|
||||
|
||||
| Обнаружено | Безопасное исправление после grace |
|
||||
| --- | --- |
|
||||
| Старый orphan нашего UUID-prefix без metadata | Повторный HEAD, проверка ETag/size/возраста, DELETE только этого объекта |
|
||||
| Старый pending, PUT сохранил правильный объект | ready; checksum/size/MIME сверяются с намерением |
|
||||
| Старый pending, подтверждённое 404 объекта, нет links | Удаление брошенного намерения |
|
||||
| Ready, объект пропал/не соответствует metadata | missing, metadata и Message links сохраняются; API выдаёт 409 |
|
||||
| Missing, правильный оригинал восстановлен оператором | ready, тот же ID и связи |
|
||||
| Свежие объекты/намерения, ready с/без Message links | Не удаляются |
|
||||
| Сетевой сбой, 403, 5xx | Проход завершается ошибкой, это не отсутствие объекта |
|
||||
| Неожиданные bytes у pending | Review action, автоматическое удаление запрещено |
|
||||
|
||||
S3 LIST и HEAD имеют разную точность даты. Проверяется возраст обеих наблюдаемых
|
||||
записей и ETag/size, а не точное равенство LastModified. ETag используется как
|
||||
проверка неизменности, не как SHA оригинала. Приложение не переиспользует ключи.
|
||||
Сторонний writer с правом overwrite не должен менять originals/ во время сверки.
|
||||
|
||||
В dry-run нет изменений данных. Повтор apply сходится к отсутствию новых actions.
|
||||
При ошибке часть независимых исправлений могла уже закоммититься; запуск можно
|
||||
повторить. Потерянные bytes нельзя восстановить из SHA: для фактического восстановления
|
||||
нужна исходная копия/backup. Сверка проверяет HEAD metadata, а не перехеширует весь
|
||||
bucket; сквозные проверки отдельно скачивают оригинал и считают SHA.
|
||||
|
||||
## Команды исследований и критерии
|
||||
|
||||
Для проверки настоящего orphan нужен его **реальный** возраст ≥300 s, часы S3
|
||||
не подменяются. Сначала создать seed, затем выполнять прочие проверки:
|
||||
|
||||
```sh
|
||||
uv run --locked python scripts/verify_storage.py --seed evidence/storage-seed-new.json
|
||||
make test-student
|
||||
make verify
|
||||
# Когда LastModified seed старше 300 s:
|
||||
uv run --locked python scripts/verify_storage.py --seed-file evidence/storage-seed-new.json --output evidence/storage-new.json
|
||||
uv run --locked --group research python research/analyze.py evidence/storage-new.json
|
||||
```
|
||||
|
||||
Сохранённый запуск разбит на базовые проверки и продолжение после aging
|
||||
(`--resume-basics evidence/storage-basics.json`); memory/cost повторены отдельно
|
||||
на финальном образе без параллельного запуска тестов. Итоговый набор — storage.json.
|
||||
Скрипт отказывается перезаписывать output. Для графиков сохранённой серии
|
||||
analyze.py читает evidence/storage.json. Raw содержит cgroup readings, времена
|
||||
каждой загрузки, CID/alias и counters/время всех проходов сверки, без токенов.
|
||||
|
||||
| Критерий | Проверка | Доказательство |
|
||||
| --- | --- | --- |
|
||||
| Исходный skeleton/OpenAPI | make check; validate_openapi | [contracts.txt](evidence/contracts.txt) |
|
||||
| Весь API lab4 и прежние возможности | make test; make test-student | [smoke.txt](evidence/smoke.txt), [student-tests.txt](evidence/student-tests.txt) |
|
||||
| PNG/JPEG, MIME, лимиты, ACL, чужой chat/id | 65 собственных тестов с настоящими PG/S3 | [test_attachments.py](student-tests/test_attachments.py) |
|
||||
| Байты с другой API после stop загрузившей | cross_replica в verify_storage | [storage.json](evidence/storage.json) |
|
||||
| Сохранность metadata/bytes/session при restart | публичный persistence, StartedAt PG/Valkey/S3/LB | [persistence.txt](evidence/persistence.txt), [cluster.json](evidence/cluster.json) |
|
||||
| Каждая API, идемпотентность, rolling | make verify | [cluster.json](evidence/cluster.json) |
|
||||
| Приватный bucket и ограниченный API user | анонимный GET 403, Delete/other bucket 403 | [storage.json](evidence/storage.json) |
|
||||
| Прерванный body / отказ S3 | raw TCP abort; реальный stop/start S3 | [storage.json](evidence/storage.json) |
|
||||
| PUT прошёл, DB finalize упал; PUT не прошёл | injected failure между шагами, dry-run/apply/repeat | [test_attachments.py](student-tests/test_attachments.py) |
|
||||
| ≥10 загрузок, bounded RAM | барьер 10 клиентов, два размера, kernel memory.peak | [storage.json](evidence/storage.json) |
|
||||
| Orphan/missing, свежие/used защищены | реальный aged orphan, удаление used bytes, восстановление | [storage.json](evidence/storage.json) |
|
||||
| Стоимость обхода | 0/100/1100 дополнительных объектов, 3 dry-run повтора | [storage.json](evidence/storage.json) |
|
||||
|
||||
## Наблюдаемые результаты
|
||||
|
||||
65 собственных тестов и публичный smoke прошли. Публичные persistence/failover
|
||||
сохранили прежнюю сессию и оригинал; реальный S3 stop дал два 503, после start
|
||||
тот же файл снова скачался. Консоль недоступна из API, повторный bucket bootstrap
|
||||
дважды завершился успешно с теми же credentials и данными.
|
||||
|
||||
| Серия загрузок | Клиенты | Успешные RPS | p50/p95/p99, ms | Максимум memory.peak одной API, MiB | Ошибки |
|
||||
| --- | ---: | ---: | --- | ---: | ---: |
|
||||
| 1,081,015 bytes, 360,000 pixels | 10 | 58.07 | 121.6/159.4/159.4 | 162.77 | 0 |
|
||||
| 8,674,572 bytes, 2,890,000 pixels | 10 | 13.52 | 500.3/726.9/726.9 | 163.62 | 0 |
|
||||
|
||||
Средняя суммарная загрузка CPU двух API по delta cgroup cpu.stat: 0.58 ядра, 0.91 ядра
|
||||
для малого/большого payload. Окно включает probes чтения счётчиков и служебные процессы после readiness; это короткая усреднённая оценка, не CPU p95. Raw cpu.stat
|
||||
сохранён в before/after каждой серии.
|
||||
|
||||
Kernel peak около 163 MiB при лимите 512 MiB. Он включает startup (в том числе
|
||||
Argon2id) и является верхней границей за весь trial. Из близости двух peak
|
||||
не следует, что размер файла не влияет на RAM: эксперимент не отделяет пик
|
||||
startup от пика декодирования. Файлы больше проверенных размеров, но в пределах
|
||||
10 MiB/40 млн pixels, покрыты лимитами реализации; их полная матрица RAM не измерялась.
|
||||
|
||||
| Дополнительные fresh S3 objects | dry-run min / median / max, ms | LIST pages |
|
||||
| ---: | --- | --- |
|
||||
| 0 | 137.78 / 141.20 / 184.41 | 1,1,1 |
|
||||
| 100 | 150.60 / 163.01 / 166.15 | 1,1,1 |
|
||||
| 1100 | 189.81 / 216.18 / 233.53 | 2,2,2 |
|
||||
|
||||
Настоящий orphan был выдержан 503.0 s. Dry-run обнаружил
|
||||
его и брошенный intent, но не изменил S3/БД; apply удалил ровно эти объекты,
|
||||
повторный apply вернул пустой actions. Свежий orphan сохранён. Для used-вложения
|
||||
после удаления bytes dry-run показал mark_missing без изменения metadata;
|
||||
apply сохранил ID и Message link и перевёл запись в missing. После возврата
|
||||
правильных bytes следующий проход вернул ready, download снова совпал с fixture.
|
||||
Это подтверждённое улучшение протокола: сбой между двумя хранилищами больше не
|
||||
оставляет незаметный вечный orphan/pending; он обнаруживается и сходится к явному
|
||||
состоянию. Это не обещание восстановления без исходных bytes.
|
||||
|
||||
| Воздействие API | Успешные RPS | Ошибки/попытки | Успех после hook, s | p95, ms |
|
||||
| --- | ---: | --- | ---: | ---: |
|
||||
| Остановка API 1 | 421.2 | 167/2615 | 2.225 | 31.2 |
|
||||
| Остановка API 2 | 334.3 | 153/1964 | 1.819 | 44.8 |
|
||||
| Rolling replacement | 432.3 | 287/7903 | 13.610 | 47.9 |
|
||||
|
||||
Каждая остановка укладывается в 10 s; подтверждённые сообщения сохранены.
|
||||
first_success_after_action_seconds отсчитывается от старта hook до первого
|
||||
успеха после его завершения, это не длительность сплошного простоя. Другая API
|
||||
может отвечать в течение hook. В stopped хранится паспорт процесса до stop;
|
||||
скрипт отдельно проверяет running=false. Проверка cross_replica сохраняет также
|
||||
фактический after_stop с running=false и alias другой скачавшей API.
|
||||
|
||||

|
||||
|
||||
CPU/RAM лимиты проверялись inspect. Kernel memory.peak включает Python, decoder,
|
||||
tmpfs, page cache и startup; это не process RSS и не усреднение Docker stats.
|
||||
Каждый размер испытывался одной серией из 10 клиентов, API пересоздавались до
|
||||
серии. Десять клиентов не означают десять одновременных декодеров: именно gate
|
||||
ограничивает активную обработку, остальные ждут, все десять завершились успешно.
|
||||
Короткие локальные измерения не являются production capacity/SLO.
|
||||
|
||||
Стоимость сверки измерена при неизменном числе metadata и росте числа свежих
|
||||
S3 objects. Это проверяет paging и постоянные накладные расходы; масштабирование
|
||||
по числу metadata отдельно не измерялось. LIST стоит O(objects/1000), индексированный
|
||||
DB lookup нужен для старых объектов, HEAD — для старых metadata. Нет вывода, будто
|
||||
полное чтение/хеширование всех bytes столь же дёшево. Три повтора дают min/median/max,
|
||||
не статистический доверительный интервал. Обе API и health probes работают во время
|
||||
замеров, остальные тесты/сборки не запускаются.
|
||||
|
||||
## Вопросы на защите
|
||||
|
||||
**Почему Docker volume одной API не равен S3?** Локальный файл зависит от выбранного
|
||||
host/контейнера и не появляется у соседней реплики. S3 — общий адресуемый сервис
|
||||
с object key, API хранения и отдельными правами. Временный tmpfs здесь нужен только
|
||||
во время запроса; после 201 оригинал читается из общего S3, что проверено остановкой
|
||||
uploading process. S3 volume тоже нуждается в защите от отказа диска/host.
|
||||
|
||||
**Когда честно вернуть 201?** Когда bytes полностью приняты, проверены, PUT успешно
|
||||
подтверждён и metadata ready закоммичена. Возвращать 201 после помещения в локальную
|
||||
память нельзя. В lab5 будет другой контракт: 202 после устойчивого staging и
|
||||
надёжного сохранения намерения обработки; ready появится позже.
|
||||
|
||||
**PUT прошёл, а COMMIT — нет?** Два хранилища не откатываются одной SQL-транзакцией.
|
||||
До PUT уже сохранён pending, поэтому объект не теряется из учёта. После grace
|
||||
reconciler сверит key/checksum/size/MIME и восстановит ready. Если Put не дошёл,
|
||||
старый пустой intent удалится. Если metadata вообще нет, старый orphan удаляется
|
||||
после дополнительной проверки. HTTP timeout сам по себе не доказывает rollback.
|
||||
|
||||
**Почему случайный key/private bucket не заменяют ACL?** Случайность — защита от
|
||||
угадывания, приватность — от анонимного S3. Любой участник приложения всё равно
|
||||
может знать ID чужого вложения. Только проверка действительной сессии и членства
|
||||
в конкретном чате на metadata/content/message запрещает чужой доступ. Автор
|
||||
выводится из сессии, filename/key не используются как доказательство прав.
|
||||
|
||||
## Ограничения и продолжение
|
||||
|
||||
Защищён отказ одной API, но host/LB/PostgreSQL/MinIO и их диски остаются SPOF.
|
||||
Backup/PITR/репликация storage не заявляются. Это локальный HTTP-стенд с вымышленными
|
||||
данными; удалённая демонстрация потребует TLS. Presigned upload (трек 1) не реализован.
|
||||
Готовые неиспользованные вложения сохраняются; политики удаления истории нет.
|
||||
Для неожиданного несовпадения pending требуется оператор, оно явно видно в report.
|
||||
Reconciler запускается вручную по команде; для регулярной эксплуатации его нужно
|
||||
запускать по расписанию, вечное автоматическое исправление без запуска не обещается.
|
||||
|
||||
В lab5 появятся устойчивое staging, outbox/job, брокер и worker. Объекты очереди
|
||||
будут ссылками, не bytes. Потребуются lease/retry и защита активной обработки от GC;
|
||||
обязательный thumbnail будет отдельным вариантом. Lab4 останется независимым
|
||||
проверенным репозиторием, его контракт 201 ready не меняется будущей работой.
|
||||
|
||||
## Источники
|
||||
|
||||
- [README](README.md), [семантика](contracts/README.md), [COURSE](COURSE.md).
|
||||
- [Официальный релиз MinIO с исправлением CVE](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z).
|
||||
- [MinIO policy access](https://docs.min.io/aistor/administration/iam/access/).
|
||||
- [Pillow Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html): lazy open, verify, pixel guard.
|
||||
- [Boto3 retries](https://docs.aws.amazon.com/boto3/latest/guide/retries.html).
|
||||
- [AWS S3 object uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html).
|
||||
- [Linux cgroup v2 memory interface](https://docs.kernel.org/admin-guide/cgroup-v2.html): memory.current/peak/max.
|
||||
|
||||
## Соответствие ревизий
|
||||
|
||||
После редактуры отчёта, шаблона и служебных метаданных сохранено соответствие прежних и текущих Git-ревизий: [revision-map.json](evidence/revision-map.json). Код приложения, миграции, зависимости, Dockerfile, конфигурация сервисов и исходные публичные проверки не изменены. Численные результаты и исторические image/source identifiers в evidence сохранены в исходном виде; их не следует трактовать как новый запуск измерений.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
# Lab4: метаданные, байты и восстановление
|
||||
|
||||
Основа — собственная lab3. PostgreSQL хранит users/chats/sessions/messages и
|
||||
результаты Idempotency-Key. Valkey — восстанавливаемый кэш User. В lab4 добавлены
|
||||
Attachment и message_attachments; оригиналы находятся в приватном S3.
|
||||
|
||||
Миграции 001/002 перенесены без изменения checksum. 003 допускает пустой text
|
||||
для сообщения с вложением и добавляет таблицы. Обязательность содержимого
|
||||
проверяет модель HTTP; составные FK в message_attachments гарантируют один chat
|
||||
у сообщения и вложения. Position сохраняет порядок ID. До десяти уникальных ID,
|
||||
проверка доступности и вставка связей входят в транзакцию message + request result.
|
||||
Для текстового POST отсутствие attachment_ids и [] дают один hash.
|
||||
|
||||
Attachment имеет id/chat/owner, отображаемый filename, фактический MIME,
|
||||
size_bytes, SHA-256, серверный object_key, storage_state и timestamps.
|
||||
Object key: originals/<UUID>; имена не используются как пути.
|
||||
Storage_state — внутреннее состояние pending/ready/missing. Внешняя схема lab4
|
||||
позволяет только ready; для pending/missing API возвращает 409, а не выдумывает
|
||||
несуществующий статус. Чужой чат — 404, перед любым GET выполняется ACL.
|
||||
|
||||
## Загрузка
|
||||
|
||||
1. Auth/ACL и filename/MIME/Content-Length проверяются до чтения файла.
|
||||
2. Один активный upload на процесс. До входа очередь ждёт максимум 5 s.
|
||||
3. Request.stream читается с общим бюджетом 15 s и счётчиком фактических bytes.
|
||||
Временный анонимный файл — только ограниченная scratch-копия на tmpfs 32 MiB.
|
||||
4. Pillow читает заголовок и проверяет площадь до декодирования; verify + повторное
|
||||
load отклоняют повреждённые PNG/JPEG. Байты не перекодируются. SHA считается
|
||||
инкрементально по исходному потоку. Размер ≤10 MiB, площадь ≤40 млн пикселей.
|
||||
5. COMMIT записи pending с известными key/checksum **до** S3 PUT.
|
||||
6. PUT целого объекта; затем отдельный COMMIT перевода в ready; только затем 201.
|
||||
|
||||
Падение до шага 5 не создаёт объектов. После 5 любой исход PUT/COMMIT остаётся
|
||||
связан с устойчивым намерением. Состояние 201 не кэшируется до COMMIT.
|
||||
Повтор upload может создать новое вложение: Idempotency-Key гарантирует повтор
|
||||
сообщения, а не бинарного POST. Прямой загрузки и multipart upload здесь нет.
|
||||
|
||||
## Сверка
|
||||
|
||||
Отдельный CLI с отдельным S3 identity; dry-run по умолчанию. --apply разрешает
|
||||
исправления. Минимальный grace — 300 s, заметно больше штатной загрузки/остановки.
|
||||
Ключи не переиспользуются и не перезаписываются приложением. Сверка берёт advisory
|
||||
lock на весь запуск, обрабатывает списки постранично, строки — под FOR UPDATE.
|
||||
|
||||
- Orphan: старый объект строго нашего UUID-prefix без строки Attachment.
|
||||
Повторный HEAD проверяет ETag, размер и возраст непосредственно перед DELETE.
|
||||
LIST и HEAD имеют разную точность timestamp, точное равенство дат не требуется.
|
||||
Свежий объект пропускается, другие prefix/buckets не затрагиваются.
|
||||
- Старый pending + существующий совпадающий объект → ready. Это восстановление
|
||||
случая «PUT завершён, финальный COMMIT/HTTP не подтверждён».
|
||||
- Старый pending + подтверждённое отсутствие объекта → удаление намерения,
|
||||
только если нет message relation. Свежие намерения защищены.
|
||||
- Старый ready + отсутствующие/несогласованные bytes → missing; строка и ссылки
|
||||
сохраняются. GET возвращает 409. Сверка не способна восстановить утраченные байты
|
||||
из SHA; после восстановления правильного оригинала она возвращает ready.
|
||||
- Ready-объекты, даже не прикреплённые к сообщению, не удаляются. Действующие ссылки
|
||||
никогда не удаляются. Ошибки сети/403/5xx не трактуются как 404.
|
||||
- Несогласованный pending с чужими checksum/size/MIME требует оператора;
|
||||
report содержит review_invalid_pending. Автоматически удалять такие bytes нельзя.
|
||||
|
||||
Сравнение HeadObject проверяет size/MIME/метаданные SHA, а не повторно хеширует
|
||||
все bytes; полная проверка SHA выполняется клиентскими сценариями скачивания.
|
||||
Обычный проход: LIST O(objects/1000), индексированный lookup для старых объектов,
|
||||
HEAD для старых metadata, O(metadata) строк. Память ограничена страницами и
|
||||
100 записями preview, counters учитывают весь проход. При ошибке часть apply
|
||||
могла уже завершиться: повторный запуск безопасен и доводит сверку до результата.
|
||||
|
||||
## Границы гарантий
|
||||
|
||||
Это протокол восстановления двух независимых хранилищ, не распределённая ACID
|
||||
транзакция. PostgreSQL/S3/диск/LB и host остаются SPOF. Глобальные backup/PITR не
|
||||
реализованы. Clock skew предполагается значительно меньше grace. Поддерживаются
|
||||
только серверные неизменяемые UUID-ключи; сторонний writer с правом перезаписи
|
||||
не должен менять их во время сверки. Для lab5 придётся учесть lease активного worker
|
||||
и staging-prefix: этот CLI не должен удалять объекты по ещё активному заданию.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Only DB/cache test ports; API replicas always stay behind the LB.
|
||||
services:
|
||||
postgres:
|
||||
ports: ["127.0.0.1:${TEST_PG_PORT:-55435}:5432"]
|
||||
valkey:
|
||||
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56382}:6379"]
|
||||
s3:
|
||||
ports: ["127.0.0.1:${TEST_S3_PORT:-59004}:9000"]
|
||||
bucket-init:
|
||||
environment:
|
||||
CREATE_TEST_BUCKET: '1'
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
name: mtusi-lab4
|
||||
|
||||
x-api: &api
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab4}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
VCS_REF: ${VCS_REF:-uncommitted}
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
pids_limit: 96
|
||||
cpus: ${API_CPU:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 7s
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_api_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_api_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
APP_ENV: lab4
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
|
||||
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
networks: [backend]
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.3
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql]
|
||||
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
|
||||
-c, shared_preload_libraries=pg_stat_statements]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: ${POSTGRES_CPU:-1}
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
networks: [backend]
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-alpine
|
||||
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
|
||||
--maxmemory-policy, allkeys-lru]
|
||||
healthcheck:
|
||||
test: [CMD, valkey-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 20
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks: [backend]
|
||||
migrate:
|
||||
<<: *api
|
||||
command: [python, -m, messenger.migrate]
|
||||
secrets: [database_url]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
valkey: {condition: service_healthy}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
lb:
|
||||
image: haproxy:3.2.1-alpine
|
||||
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
ports: ["127.0.0.1:${PUBLIC_PORT:-8084}:8080"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
networks: [backend]
|
||||
|
||||
s3:
|
||||
image: mtusi-lab4-s3:2025-10-15
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
|
||||
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
|
||||
cap_drop: [ALL]
|
||||
cpus: 1
|
||||
mem_limit: 768m
|
||||
memswap_limit: 768m
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
|
||||
interval: 2s
|
||||
timeout: 1s
|
||||
retries: 30
|
||||
networks: [backend]
|
||||
bucket-init:
|
||||
image: mtusi-lab4-mc:2025-08-13
|
||||
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
|
||||
command: [/bootstrap-s3.sh]
|
||||
volumes:
|
||||
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
|
||||
- ./infra/api-policy.json:/policies/api-policy.json:ro
|
||||
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
|
||||
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
|
||||
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
|
||||
s3_test_access_key_id, s3_test_secret_access_key]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
reconciler:
|
||||
<<: *api
|
||||
profiles: [tools]
|
||||
command: [python, -m, messenger.reconcile]
|
||||
healthcheck: {disable: true}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_reconciler_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_reconciler_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
bucket-init: {condition: service_completed_successfully}
|
||||
|
||||
networks:
|
||||
backend: {}
|
||||
volumes:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
s3_test_access_key_id:
|
||||
file: ./secrets/s3_test_access_key_id
|
||||
s3_test_secret_access_key:
|
||||
file: ./secrets/s3_test_secret_access_key
|
||||
@@ -0,0 +1 @@
|
||||
PASS: research figure generated from storage.json
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: 103 candidate/versioned files; 129 reachable objects; no requested names or local credentials.
|
||||
PASS: instructor files unchanged, upstream push disabled.
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: private bucket and scoped identities ready
|
||||
PASS: private bucket and scoped identities ready
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"tested_commit": "900182cbb405d57b20c1b8bfd7cef21592ccf4cf",
|
||||
"image_revision": "900182cbb405d57b20c1b8bfd7cef21592ccf4cf",
|
||||
"runtime_sha256": {
|
||||
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
|
||||
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
|
||||
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
|
||||
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
|
||||
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
|
||||
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
|
||||
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
|
||||
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
|
||||
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
|
||||
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
|
||||
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
|
||||
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
|
||||
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
|
||||
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
|
||||
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298"
|
||||
},
|
||||
"runtime_matches_measurements": true,
|
||||
"fresh_secrets": true,
|
||||
"fresh_pg_and_s3_volumes": true,
|
||||
"student_tests_passed": 65,
|
||||
"public_smoke_passed": true,
|
||||
"public_persistence_passed": true,
|
||||
"api_instances": [
|
||||
{
|
||||
"id": "f43056b397aa3639d9fe0220734c61f2e66caa5d6ce4522c35a645933787204d",
|
||||
"started_at": "2026-10-17T17:22:12.959003129Z",
|
||||
"pid": 9135,
|
||||
"running": true,
|
||||
"image": "sha256:4c8bff2f05a461167d1921b16e64fbd46342daffcaed4f1e9486dadea0ea3416",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "c4a9f8fb8759467b83f84981872cc014"
|
||||
},
|
||||
{
|
||||
"id": "ca95b52ce4b1976a36b4f0e1aa32a7310cacee932f4f8090b410cf957dc931c3",
|
||||
"started_at": "2026-10-17T17:22:13.11128092Z",
|
||||
"pid": 9269,
|
||||
"running": true,
|
||||
"image": "sha256:4c8bff2f05a461167d1921b16e64fbd46342daffcaed4f1e9486dadea0ea3416",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "8f7362c97d3f40c48a6c72dd5504fb1e"
|
||||
}
|
||||
],
|
||||
"s3": {
|
||||
"id": "3f7a14116acfd92ef834a5c71f2571b08c297e3786f16a7789bae5d28e4e7fae",
|
||||
"started_at": "2026-10-17T17:22:12.732578545Z",
|
||||
"pid": 8874,
|
||||
"running": true,
|
||||
"image": "sha256:ca2f62967cfd3096ef07615e356e3e6909bb6885e1672999fc0e206b478ec904",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:28084"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
uv run --locked pytest --disable-warnings
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
|
||||
rootdir: <repository>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: anyio-4.15.1
|
||||
collected 65 items
|
||||
|
||||
student-tests/test_api.py .................................... [ 55%]
|
||||
student-tests/test_attachments.py ................. [ 81%]
|
||||
student-tests/test_cluster.py ............ [100%]
|
||||
|
||||
======================= 65 passed, 2 warnings in 21.77s ========================
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:28084"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,307 @@
|
||||
{
|
||||
"initial_instances": [
|
||||
{
|
||||
"id": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
|
||||
"started_at": "2026-10-17T16:54:30.42283897Z",
|
||||
"pid": 10590,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "fccb137fe0244ddf98a9e454eab19756"
|
||||
},
|
||||
{
|
||||
"id": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
|
||||
"started_at": "2026-10-17T16:54:30.345976304Z",
|
||||
"pid": 10534,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "1ee65c20c8df4f808677d83c6556e18d"
|
||||
}
|
||||
],
|
||||
"failures": [
|
||||
{
|
||||
"duration_seconds": 5.81228291700063,
|
||||
"workers": 6,
|
||||
"requests": 2615,
|
||||
"successes": 2448,
|
||||
"errors": 167,
|
||||
"successful_rps": 421.1770202788521,
|
||||
"error_fraction": 0.06386233269598471,
|
||||
"first_success_after_action_seconds": 2.225296332999278,
|
||||
"action_seconds": 1.7712559159990633,
|
||||
"p50_ms": 8.855707999828155,
|
||||
"p95_ms": 31.16029100056039,
|
||||
"p99_ms": 73.12783300039882,
|
||||
"instances": [
|
||||
"1ee65c20c8df4f808677d83c6556e18d",
|
||||
"fccb137fe0244ddf98a9e454eab19756"
|
||||
],
|
||||
"tail_instances": [
|
||||
"1ee65c20c8df4f808677d83c6556e18d"
|
||||
],
|
||||
"confirmed_writes": 243,
|
||||
"stopped": {
|
||||
"id": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
|
||||
"started_at": "2026-10-17T16:54:30.42283897Z",
|
||||
"pid": 10590,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "fccb137fe0244ddf98a9e454eab19756"
|
||||
}
|
||||
},
|
||||
{
|
||||
"duration_seconds": 5.416828084000372,
|
||||
"workers": 6,
|
||||
"requests": 1964,
|
||||
"successes": 1811,
|
||||
"errors": 153,
|
||||
"successful_rps": 334.32849850803495,
|
||||
"error_fraction": 0.07790224032586558,
|
||||
"first_success_after_action_seconds": 1.8193489170007524,
|
||||
"action_seconds": 1.3782874589996936,
|
||||
"p50_ms": 10.209916999883717,
|
||||
"p95_ms": 44.78274999928544,
|
||||
"p99_ms": 68.33825000103388,
|
||||
"instances": [
|
||||
"1ee65c20c8df4f808677d83c6556e18d",
|
||||
"ec543faccabb4b4d856cb21490da98d9"
|
||||
],
|
||||
"tail_instances": [
|
||||
"ec543faccabb4b4d856cb21490da98d9"
|
||||
],
|
||||
"confirmed_writes": 179,
|
||||
"stopped": {
|
||||
"id": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
|
||||
"started_at": "2026-10-17T16:54:30.345976304Z",
|
||||
"pid": 10534,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "1ee65c20c8df4f808677d83c6556e18d"
|
||||
}
|
||||
}
|
||||
],
|
||||
"concurrent_replays": 30,
|
||||
"concurrent_effects": 1,
|
||||
"rolling": {
|
||||
"duration_seconds": 17.618615958999726,
|
||||
"workers": 6,
|
||||
"requests": 7903,
|
||||
"successes": 7616,
|
||||
"errors": 287,
|
||||
"successful_rps": 432.27004991329574,
|
||||
"error_fraction": 0.03631532329495128,
|
||||
"first_success_after_action_seconds": 13.610468083001251,
|
||||
"action_seconds": 13.610214250000354,
|
||||
"p50_ms": 9.198334000757313,
|
||||
"p95_ms": 47.916500001520035,
|
||||
"p99_ms": 80.57516699955158,
|
||||
"instances": [
|
||||
"156f31907e404f4b82cd2ece01102a69",
|
||||
"5ae9d1ba363c48999bc872e0a1648271",
|
||||
"c8354af5dac443e3afdfc970ba76d70e",
|
||||
"ec543faccabb4b4d856cb21490da98d9"
|
||||
],
|
||||
"tail_instances": [
|
||||
"156f31907e404f4b82cd2ece01102a69",
|
||||
"5ae9d1ba363c48999bc872e0a1648271"
|
||||
],
|
||||
"confirmed_writes": 759,
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
|
||||
"old_instance": "ec543faccabb4b4d856cb21490da98d9",
|
||||
"live_instances": [
|
||||
"5ae9d1ba363c48999bc872e0a1648271",
|
||||
"c8354af5dac443e3afdfc970ba76d70e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"old_container": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
|
||||
"old_instance": "c8354af5dac443e3afdfc970ba76d70e",
|
||||
"live_instances": [
|
||||
"156f31907e404f4b82cd2ece01102a69",
|
||||
"5ae9d1ba363c48999bc872e0a1648271"
|
||||
]
|
||||
}
|
||||
],
|
||||
"confirmed_messages_preserved": true
|
||||
},
|
||||
"persistence": {
|
||||
"before": {
|
||||
"postgres": {
|
||||
"id": "a6e5f14597245ada8071635bf261c294e53045ee0d09bbea1cd682681cfacd9c",
|
||||
"started_at": "2026-10-17T16:54:25.66952526Z",
|
||||
"pid": 9916,
|
||||
"running": true,
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55435"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"valkey": {
|
||||
"id": "1899ba79d40145cd3ff95269e73076e3b2ba74a472e4427f204da8e9c52592b6",
|
||||
"started_at": "2026-10-17T16:54:25.484294468Z",
|
||||
"pid": 9764,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56382"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:54:25.588696677Z",
|
||||
"pid": 9841,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"lb": {
|
||||
"id": "f7affe8e1b5904cb78f8ac5d43bab11da4006dd9d933118be1696ec36b8fc1b3",
|
||||
"started_at": "2026-10-17T16:54:36.001647626Z",
|
||||
"pid": 10960,
|
||||
"running": true,
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8084"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"postgres": {
|
||||
"id": "a6e5f14597245ada8071635bf261c294e53045ee0d09bbea1cd682681cfacd9c",
|
||||
"started_at": "2026-10-17T16:58:16.715706381Z",
|
||||
"pid": 26327,
|
||||
"running": true,
|
||||
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"5432/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "55435"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"valkey": {
|
||||
"id": "1899ba79d40145cd3ff95269e73076e3b2ba74a472e4427f204da8e9c52592b6",
|
||||
"started_at": "2026-10-17T16:58:16.815795964Z",
|
||||
"pid": 26382,
|
||||
"running": true,
|
||||
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"6379/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "56382"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"s3": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:58:16.681019131Z",
|
||||
"pid": 26248,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"lb": {
|
||||
"id": "f7affe8e1b5904cb78f8ac5d43bab11da4006dd9d933118be1696ec36b8fc1b3",
|
||||
"started_at": "2026-10-17T16:58:16.575992006Z",
|
||||
"pid": 26158,
|
||||
"running": true,
|
||||
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
|
||||
"memory": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8084"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"same_token_public_test": true
|
||||
},
|
||||
"final_instances": [
|
||||
{
|
||||
"id": "ac20d4b45be85d82fe302e812b9b4c8d3645f68b126ef36b0aa1f0eae9f59a4d",
|
||||
"started_at": "2026-10-17T16:58:16.863466756Z",
|
||||
"pid": 26514,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "af0c8373068c4e0782fdb1af3602cac6"
|
||||
},
|
||||
{
|
||||
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
|
||||
"started_at": "2026-10-17T16:58:16.838496214Z",
|
||||
"pid": 26426,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "e96c5c7e0457467597c53e1af497f8ac"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
Verified failure and restoration of one API
|
||||
Verified failure and restoration of one API
|
||||
Verified sequential replacement under traffic
|
||||
PASS: real replica failures, replay, rolling replacement and persistence
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 3, "first_recovery_seconds": 0.769}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
uv run --locked ruff check src student-tests scripts/bootstrap.py scripts/cluster.py scripts/stop-one.py scripts/rolling-update.py scripts/verify_cluster.py scripts/verify_storage.py scripts/storage_lab.py research
|
||||
All checks passed!
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8084"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,3 @@
|
||||
Container mtusi-lab4-reconciler-run-de897097f2a4 Creating
|
||||
Container mtusi-lab4-reconciler-run-de897097f2a4 Created
|
||||
{"mode": "dry-run", "grace_seconds": 300, "elapsed_seconds": 0.10601412500000151, "counts": {"s3_list_pages": 1, "s3_objects": 50, "s3_head": 50, "delete_orphan": 2, "pg_metadata_pages": 2, "pg_attachments": 48, "ready_preserved": 48}, "actions": [{"action": "delete_orphan", "id": "2d392b8a-fbb1-444c-856e-f193ea279929"}, {"action": "delete_orphan", "id": "81c3c3e9-589f-4e4f-a130-c81082388565"}], "actions_preview_limit": 100}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"scope": "Редактура документации и метаданных рисунка",
|
||||
"runtime_sources_unchanged": true,
|
||||
"migrations_dependencies_and_infrastructure_unchanged": true,
|
||||
"instructor_contract_and_public_tests_unchanged": true,
|
||||
"measurement_values_unchanged": true,
|
||||
"original_head": "da6ca232c350186445c8531a0fae52d657985569",
|
||||
"rewritten_history_head": "36d295e16cc9477ce10dcf92aab2d80a6d3b826f",
|
||||
"commit_map": {
|
||||
"900182cbb405d57b20c1b8bfd7cef21592ccf4cf": "85433e20e4102202451b2358c4a65075b2964b27",
|
||||
"da6ca232c350186445c8531a0fae52d657985569": "36d295e16cc9477ce10dcf92aab2d80a6d3b826f",
|
||||
"ec540a1a725862a8619fcb758daec4ad197f0dd5": "ec540a1a725862a8619fcb758daec4ad197f0dd5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
|
||||
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
|
||||
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298",
|
||||
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
|
||||
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
|
||||
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
|
||||
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
|
||||
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
|
||||
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
|
||||
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
|
||||
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
|
||||
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
|
||||
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
|
||||
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
|
||||
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
/binary/minio: go1.26.7
|
||||
path github.com/minio/minio
|
||||
mod github.com/minio/minio v0.0.0-20251015172955-9e49d5e7a648 h1:6TdolSCLSs2nwm8i0PpWDqf9iX2Ty9WQK8wmr7dCnUM=
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:8084"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,249 @@
|
||||
{
|
||||
"privacy": {
|
||||
"anonymous_get_status": 403,
|
||||
"api_delete_denied": true,
|
||||
"other_bucket_denied": true,
|
||||
"attachment": "d8693ef0-f3bc-4206-988f-d1664afeb827"
|
||||
},
|
||||
"cross_replica": {
|
||||
"uploader": {
|
||||
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
|
||||
"started_at": "2026-10-17T16:58:16.838496214Z",
|
||||
"pid": 26426,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "e96c5c7e0457467597c53e1af497f8ac"
|
||||
},
|
||||
"after_stop": {
|
||||
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
|
||||
"started_at": "2026-10-17T16:58:16.838496214Z",
|
||||
"pid": 0,
|
||||
"running": false,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {}
|
||||
},
|
||||
"downloader_instance": "af0c8373068c4e0782fdb1af3602cac6",
|
||||
"original_preserved": true
|
||||
},
|
||||
"interruption": {
|
||||
"before_rows": 5,
|
||||
"after_rows": 5,
|
||||
"aborted_body_no_intent": true
|
||||
},
|
||||
"uploads": [
|
||||
{
|
||||
"clients": 10,
|
||||
"bytes_each": 1081015,
|
||||
"pixels": 360000,
|
||||
"duration_seconds": 0.16621475000101782,
|
||||
"successful_rps": 60.16313233295339,
|
||||
"errors": 0,
|
||||
"p50_ms": 114.14141599925642,
|
||||
"p95_ms": 153.93212499839137,
|
||||
"p99_ms": 153.93212499839137,
|
||||
"before": {
|
||||
"09da358ec38461f3ee21c4d1129697a9f1486323410ddde90b5716268ac9e07c": {
|
||||
"current_bytes": 95498240,
|
||||
"peak_bytes": 170278912
|
||||
},
|
||||
"c0356d1a469d326b73ef6a577338bffb3860ff6afcc646dd5bf1198a0c5c1792": {
|
||||
"current_bytes": 96903168,
|
||||
"peak_bytes": 165269504
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"09da358ec38461f3ee21c4d1129697a9f1486323410ddde90b5716268ac9e07c": {
|
||||
"current_bytes": 102379520,
|
||||
"peak_bytes": 170278912
|
||||
},
|
||||
"c0356d1a469d326b73ef6a577338bffb3860ff6afcc646dd5bf1198a0c5c1792": {
|
||||
"current_bytes": 101384192,
|
||||
"peak_bytes": 165269504
|
||||
}
|
||||
},
|
||||
"responses": [
|
||||
{
|
||||
"duration_ms": 76.5632500006177,
|
||||
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
|
||||
"id": "27d3163c-060f-4918-bed4-ec714a233595"
|
||||
},
|
||||
{
|
||||
"duration_ms": 95.78312500161701,
|
||||
"instance": "28dadbd8074d47a88abc712e0c76b41e",
|
||||
"id": "f7796135-d0d4-4486-8281-6947ddfa0dee"
|
||||
},
|
||||
{
|
||||
"duration_ms": 114.24379100026272,
|
||||
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
|
||||
"id": "fa09990f-7d53-4ee9-be56-e8153bd77eae"
|
||||
},
|
||||
{
|
||||
"duration_ms": 114.14141599925642,
|
||||
"instance": "28dadbd8074d47a88abc712e0c76b41e",
|
||||
"id": "ccdfddef-f5fb-4c40-8d88-7049d5812f12"
|
||||
},
|
||||
{
|
||||
"duration_ms": 133.79766599973664,
|
||||
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
|
||||
"id": "d3623f75-6e59-4f46-a809-a95590c89238"
|
||||
},
|
||||
{
|
||||
"duration_ms": 153.14370899977803,
|
||||
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
|
||||
"id": "efd8eaa0-c524-4d93-8c4f-4e212af9f524"
|
||||
},
|
||||
{
|
||||
"duration_ms": 94.01983300085703,
|
||||
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
|
||||
"id": "6810a6bc-af11-4d08-a007-d49908ce64df"
|
||||
},
|
||||
{
|
||||
"duration_ms": 153.93212499839137,
|
||||
"instance": "28dadbd8074d47a88abc712e0c76b41e",
|
||||
"id": "3ab6e0ac-cf3f-4bcb-80d0-faf404fcea3b"
|
||||
},
|
||||
{
|
||||
"duration_ms": 133.07204200100387,
|
||||
"instance": "28dadbd8074d47a88abc712e0c76b41e",
|
||||
"id": "f12f4260-b0fa-49aa-b4bc-2d0c6a4539a0"
|
||||
},
|
||||
{
|
||||
"duration_ms": 75.5845829989994,
|
||||
"instance": "28dadbd8074d47a88abc712e0c76b41e",
|
||||
"id": "fecbb057-308a-45ec-9bcf-95856be457d7"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"clients": 10,
|
||||
"bytes_each": 8674572,
|
||||
"pixels": 2890000,
|
||||
"duration_seconds": 0.7018210000005638,
|
||||
"successful_rps": 14.248647447129633,
|
||||
"errors": 0,
|
||||
"p50_ms": 485.1206250004907,
|
||||
"p95_ms": 683.7534580008651,
|
||||
"p99_ms": 683.7534580008651,
|
||||
"before": {
|
||||
"4e4fab2b66feb2efc1c03e276733a0cd367a1d71d4c01714da527e4cfaa0f3db": {
|
||||
"current_bytes": 95817728,
|
||||
"peak_bytes": 170561536
|
||||
},
|
||||
"3e96dd5071d136499279b012cab6415e2a1d16bccb0605ab0db81f4f0d89de18": {
|
||||
"current_bytes": 95109120,
|
||||
"peak_bytes": 167698432
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"4e4fab2b66feb2efc1c03e276733a0cd367a1d71d4c01714da527e4cfaa0f3db": {
|
||||
"current_bytes": 112578560,
|
||||
"peak_bytes": 170561536
|
||||
},
|
||||
"3e96dd5071d136499279b012cab6415e2a1d16bccb0605ab0db81f4f0d89de18": {
|
||||
"current_bytes": 112189440,
|
||||
"peak_bytes": 167698432
|
||||
}
|
||||
},
|
||||
"responses": [
|
||||
{
|
||||
"duration_ms": 650.3965840001911,
|
||||
"instance": "fded6353f47f48bda9a66715f6e739d8",
|
||||
"id": "d771f44e-cbcf-41de-9d6f-b3e8dbe6ac7d"
|
||||
},
|
||||
{
|
||||
"duration_ms": 607.8036670005531,
|
||||
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
|
||||
"id": "2bf2e458-909b-4aaa-b939-69ad893a2191"
|
||||
},
|
||||
{
|
||||
"duration_ms": 453.6054160016647,
|
||||
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
|
||||
"id": "26ad1005-42cc-486b-bdc6-f0f9a959d224"
|
||||
},
|
||||
{
|
||||
"duration_ms": 485.1206250004907,
|
||||
"instance": "fded6353f47f48bda9a66715f6e739d8",
|
||||
"id": "a077b398-0d5d-43b9-8c94-1e42f0c61591"
|
||||
},
|
||||
{
|
||||
"duration_ms": 328.2669590007572,
|
||||
"instance": "fded6353f47f48bda9a66715f6e739d8",
|
||||
"id": "7d293d77-2c70-4382-b1cd-14d318c33dde"
|
||||
},
|
||||
{
|
||||
"duration_ms": 528.5843330002535,
|
||||
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
|
||||
"id": "11795c80-f431-46de-b672-005870afb0e0"
|
||||
},
|
||||
{
|
||||
"duration_ms": 561.6339999996853,
|
||||
"instance": "fded6353f47f48bda9a66715f6e739d8",
|
||||
"id": "7d6915eb-e40d-4742-8590-1d6c482fc6a3"
|
||||
},
|
||||
{
|
||||
"duration_ms": 410.78812499836204,
|
||||
"instance": "fded6353f47f48bda9a66715f6e739d8",
|
||||
"id": "394c46d5-af43-49ea-a6b2-bec2e5405343"
|
||||
},
|
||||
{
|
||||
"duration_ms": 379.54966700090154,
|
||||
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
|
||||
"id": "40e7d89a-ad66-4f98-a5f6-076c25df99b9"
|
||||
},
|
||||
{
|
||||
"duration_ms": 683.7534580008651,
|
||||
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
|
||||
"id": "0994d2cb-53ee-45fe-a20d-66b165f528d1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"outage": {
|
||||
"before": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:58:16.681019131Z",
|
||||
"pid": 26248,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:59:16.975092214Z",
|
||||
"pid": 31660,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"failed_operations": 2,
|
||||
"statuses": [
|
||||
503,
|
||||
503
|
||||
],
|
||||
"stop_start_and_ready_seconds": 1.7791777080001339,
|
||||
"original_preserved": true
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 80 KiB |
@@ -0,0 +1 @@
|
||||
{"orphan": "originals/bda0b2eb-b103-46de-9ba4-e08ca5a7580f", "created_at": "2026-10-17T16:56:00.179048+00:00"}
|
||||
@@ -0,0 +1,675 @@
|
||||
{
|
||||
"started_at": "2026-10-17T17:04:23.007911+00:00",
|
||||
"privacy": {
|
||||
"anonymous_get_status": 403,
|
||||
"api_delete_denied": true,
|
||||
"other_bucket_denied": true,
|
||||
"attachment": "d8693ef0-f3bc-4206-988f-d1664afeb827",
|
||||
"console_unreachable_from_api": true
|
||||
},
|
||||
"cross_replica": {
|
||||
"uploader": {
|
||||
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
|
||||
"started_at": "2026-10-17T16:58:16.838496214Z",
|
||||
"pid": 26426,
|
||||
"running": true,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"instance": "e96c5c7e0457467597c53e1af497f8ac"
|
||||
},
|
||||
"after_stop": {
|
||||
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
|
||||
"started_at": "2026-10-17T16:58:16.838496214Z",
|
||||
"pid": 0,
|
||||
"running": false,
|
||||
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
|
||||
"memory": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {}
|
||||
},
|
||||
"downloader_instance": "af0c8373068c4e0782fdb1af3602cac6",
|
||||
"original_preserved": true
|
||||
},
|
||||
"interruption": {
|
||||
"before_rows": 5,
|
||||
"after_rows": 5,
|
||||
"aborted_body_no_intent": true
|
||||
},
|
||||
"uploads": [
|
||||
{
|
||||
"clients": 10,
|
||||
"bytes_each": 1081015,
|
||||
"pixels": 360000,
|
||||
"duration_seconds": 0.17220016600003873,
|
||||
"successful_rps": 58.07195331041522,
|
||||
"resource_window_seconds": 0.34307429199907347,
|
||||
"mean_api_cpu_cores_including_probes": 0.5834712908204168,
|
||||
"errors": 0,
|
||||
"p50_ms": 121.59408299885399,
|
||||
"p95_ms": 159.44758300065587,
|
||||
"p99_ms": 159.44758300065587,
|
||||
"before": {
|
||||
"eca892cc879ee0560cb843d51bf7db300892d6f38979b58e76371900e7c2090d": {
|
||||
"current_bytes": 95768576,
|
||||
"peak_bytes": 167612416,
|
||||
"cpu": {
|
||||
"usage_usec": 2004031,
|
||||
"user_usec": 1828766,
|
||||
"system_usec": 175265,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 27,
|
||||
"nr_throttled": 14,
|
||||
"throttled_usec": 1004492,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
},
|
||||
"e81b5efc6326bfe31b6674579ee91131656809855038b9792c4c31a029ffbc43": {
|
||||
"current_bytes": 95109120,
|
||||
"peak_bytes": 170680320,
|
||||
"cpu": {
|
||||
"usage_usec": 1977149,
|
||||
"user_usec": 1830583,
|
||||
"system_usec": 146566,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 27,
|
||||
"nr_throttled": 12,
|
||||
"throttled_usec": 1187980,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"eca892cc879ee0560cb843d51bf7db300892d6f38979b58e76371900e7c2090d": {
|
||||
"current_bytes": 102014976,
|
||||
"peak_bytes": 167612416,
|
||||
"cpu": {
|
||||
"usage_usec": 2102645,
|
||||
"user_usec": 1895893,
|
||||
"system_usec": 206751,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 30,
|
||||
"nr_throttled": 14,
|
||||
"throttled_usec": 1004492,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
},
|
||||
"e81b5efc6326bfe31b6674579ee91131656809855038b9792c4c31a029ffbc43": {
|
||||
"current_bytes": 101982208,
|
||||
"peak_bytes": 170680320,
|
||||
"cpu": {
|
||||
"usage_usec": 2078709,
|
||||
"user_usec": 1898994,
|
||||
"system_usec": 179715,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 29,
|
||||
"nr_throttled": 12,
|
||||
"throttled_usec": 1187980,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": [
|
||||
{
|
||||
"duration_ms": 122.1009580003738,
|
||||
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
|
||||
"id": "d2b7a203-e6c1-42e7-a798-c567fc956307"
|
||||
},
|
||||
{
|
||||
"duration_ms": 121.59408299885399,
|
||||
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
|
||||
"id": "32fdc39b-f71f-4247-9e46-e0ac5bf042af"
|
||||
},
|
||||
{
|
||||
"duration_ms": 159.44758300065587,
|
||||
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
|
||||
"id": "70ce907e-474b-4e34-8e31-ce3734f5937c"
|
||||
},
|
||||
{
|
||||
"duration_ms": 140.0553330004186,
|
||||
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
|
||||
"id": "596a2e41-a075-4b2b-8c64-0ec5b7a26401"
|
||||
},
|
||||
{
|
||||
"duration_ms": 159.16204099994502,
|
||||
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
|
||||
"id": "ae920131-d7f1-4b00-9083-58987593923d"
|
||||
},
|
||||
{
|
||||
"duration_ms": 100.46366599999601,
|
||||
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
|
||||
"id": "8e788d3e-233e-4aaf-8306-aacea6b5719e"
|
||||
},
|
||||
{
|
||||
"duration_ms": 83.1419169990113,
|
||||
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
|
||||
"id": "e45403aa-fe8e-4485-9bfb-d3ab799064e8"
|
||||
},
|
||||
{
|
||||
"duration_ms": 102.0030420004332,
|
||||
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
|
||||
"id": "542233b0-d431-4b36-a13b-e96250bcca0d"
|
||||
},
|
||||
{
|
||||
"duration_ms": 141.32162500027334,
|
||||
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
|
||||
"id": "15e59f0b-1cd2-4464-88ef-c4a54b22345c"
|
||||
},
|
||||
{
|
||||
"duration_ms": 82.26699999977427,
|
||||
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
|
||||
"id": "2677cb60-6b1a-4d8f-9f66-c9a48e54cb06"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"clients": 10,
|
||||
"bytes_each": 8674572,
|
||||
"pixels": 2890000,
|
||||
"duration_seconds": 0.7398240410002472,
|
||||
"successful_rps": 13.516727553865282,
|
||||
"resource_window_seconds": 0.9066179999990709,
|
||||
"mean_api_cpu_cores_including_probes": 0.9118062954859126,
|
||||
"errors": 0,
|
||||
"p50_ms": 500.3316669990454,
|
||||
"p95_ms": 726.9201670005714,
|
||||
"p99_ms": 726.9201670005714,
|
||||
"before": {
|
||||
"0be0a33b995189ff06406219e52ce8253d76669ea7a27ec00e91e82bf0b3ddd5": {
|
||||
"current_bytes": 97603584,
|
||||
"peak_bytes": 170450944,
|
||||
"cpu": {
|
||||
"usage_usec": 2428375,
|
||||
"user_usec": 2214869,
|
||||
"system_usec": 213505,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 32,
|
||||
"nr_throttled": 19,
|
||||
"throttled_usec": 1560996,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
},
|
||||
"be0ea333307f7c2fe6ec09b49a476beea1b0af0e9ded9d6520ce342172bf296d": {
|
||||
"current_bytes": 95272960,
|
||||
"peak_bytes": 171569152,
|
||||
"cpu": {
|
||||
"usage_usec": 2446614,
|
||||
"user_usec": 2215367,
|
||||
"system_usec": 231247,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 33,
|
||||
"nr_throttled": 16,
|
||||
"throttled_usec": 1407896,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"0be0a33b995189ff06406219e52ce8253d76669ea7a27ec00e91e82bf0b3ddd5": {
|
||||
"current_bytes": 110301184,
|
||||
"peak_bytes": 170450944,
|
||||
"cpu": {
|
||||
"usage_usec": 2855656,
|
||||
"user_usec": 2474167,
|
||||
"system_usec": 381488,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 40,
|
||||
"nr_throttled": 19,
|
||||
"throttled_usec": 1560996,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
},
|
||||
"be0ea333307f7c2fe6ec09b49a476beea1b0af0e9ded9d6520ce342172bf296d": {
|
||||
"current_bytes": 111939584,
|
||||
"peak_bytes": 171569152,
|
||||
"cpu": {
|
||||
"usage_usec": 2845993,
|
||||
"user_usec": 2465191,
|
||||
"system_usec": 380801,
|
||||
"nice_usec": 0,
|
||||
"nr_periods": 41,
|
||||
"nr_throttled": 16,
|
||||
"throttled_usec": 1407896,
|
||||
"nr_bursts": 0,
|
||||
"burst_usec": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": [
|
||||
{
|
||||
"duration_ms": 437.1919159984827,
|
||||
"instance": "40f46dd47d7f4b539d5d50346140bb29",
|
||||
"id": "53f9fa9d-05e3-4f31-9b88-0d721cd86844"
|
||||
},
|
||||
{
|
||||
"duration_ms": 500.3316669990454,
|
||||
"instance": "40f46dd47d7f4b539d5d50346140bb29",
|
||||
"id": "00d1c78e-0639-4430-9bec-624c7a458f35"
|
||||
},
|
||||
{
|
||||
"duration_ms": 639.1502089991263,
|
||||
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
|
||||
"id": "459b42a4-d28d-40ce-a34c-71d49d2d0dda"
|
||||
},
|
||||
{
|
||||
"duration_ms": 479.9554580004042,
|
||||
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
|
||||
"id": "2fd30bb2-9436-4ac0-a962-d3dcbf699aee"
|
||||
},
|
||||
{
|
||||
"duration_ms": 561.1379999991186,
|
||||
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
|
||||
"id": "49680954-5763-44f0-8073-0ece17bbf125"
|
||||
},
|
||||
{
|
||||
"duration_ms": 593.6962920004589,
|
||||
"instance": "40f46dd47d7f4b539d5d50346140bb29",
|
||||
"id": "d79bb81d-be37-4146-a347-7d3e47c1c429"
|
||||
},
|
||||
{
|
||||
"duration_ms": 672.7730829989014,
|
||||
"instance": "40f46dd47d7f4b539d5d50346140bb29",
|
||||
"id": "719e4162-0ac2-463b-b9bf-fec4b3ed2015"
|
||||
},
|
||||
{
|
||||
"duration_ms": 726.9201670005714,
|
||||
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
|
||||
"id": "bb116ad8-87c1-4fcc-863b-399cf0f8f2fd"
|
||||
},
|
||||
{
|
||||
"duration_ms": 317.0152499988035,
|
||||
"instance": "40f46dd47d7f4b539d5d50346140bb29",
|
||||
"id": "db210a76-b08b-492e-a552-d1d33107569b"
|
||||
},
|
||||
{
|
||||
"duration_ms": 401.8378750006377,
|
||||
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
|
||||
"id": "3323d70f-f307-4d16-9635-fab78ae143f5"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"outage": {
|
||||
"before": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:58:16.681019131Z",
|
||||
"pid": 26248,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
|
||||
"started_at": "2026-10-17T16:59:16.975092214Z",
|
||||
"pid": 31660,
|
||||
"running": true,
|
||||
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
|
||||
"memory": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {
|
||||
"9000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "59004"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"failed_operations": 2,
|
||||
"statuses": [
|
||||
503,
|
||||
503
|
||||
],
|
||||
"stop_start_and_ready_seconds": 1.7791777080001339,
|
||||
"original_preserved": true
|
||||
},
|
||||
"reconciliation": {
|
||||
"old_orphan_age_seconds": 503.0134530067444,
|
||||
"dry_run": {
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.18648320799911744,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 31,
|
||||
"fresh_objects_skipped": 4,
|
||||
"s3_head": 30,
|
||||
"delete_orphan": 1,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 29,
|
||||
"ready_preserved": 28,
|
||||
"delete_abandoned_intent": 1
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_orphan",
|
||||
"id": "bda0b2eb-b103-46de-9ba4-e08ca5a7580f"
|
||||
},
|
||||
{
|
||||
"action": "delete_abandoned_intent",
|
||||
"id": "f878fa3a-617d-476c-90d5-1b94404367b9"
|
||||
}
|
||||
],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"apply": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.13833720799993898,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 31,
|
||||
"fresh_objects_skipped": 4,
|
||||
"s3_head": 30,
|
||||
"delete_orphan": 1,
|
||||
"s3_delete": 1,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 29,
|
||||
"ready_preserved": 28,
|
||||
"delete_abandoned_intent": 1
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"action": "delete_orphan",
|
||||
"id": "bda0b2eb-b103-46de-9ba4-e08ca5a7580f"
|
||||
},
|
||||
{
|
||||
"action": "delete_abandoned_intent",
|
||||
"id": "f878fa3a-617d-476c-90d5-1b94404367b9"
|
||||
}
|
||||
],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"repeat": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.12051066699859803,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 30,
|
||||
"fresh_objects_skipped": 4,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 28,
|
||||
"s3_head": 28,
|
||||
"ready_preserved": 28
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"missing_dry_run": {
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.11716975000126695,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 29,
|
||||
"fresh_objects_skipped": 3,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 28,
|
||||
"s3_head": 28,
|
||||
"ready_preserved": 27,
|
||||
"mark_missing": 1
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"action": "mark_missing",
|
||||
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
|
||||
}
|
||||
],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"missing_apply": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.12134083299861231,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 29,
|
||||
"fresh_objects_skipped": 3,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 28,
|
||||
"s3_head": 28,
|
||||
"ready_preserved": 27,
|
||||
"mark_missing": 1
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"action": "mark_missing",
|
||||
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
|
||||
}
|
||||
],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"healing": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.11887154199939687,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 30,
|
||||
"fresh_objects_skipped": 4,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 28,
|
||||
"s3_head": 28,
|
||||
"ready_preserved": 27,
|
||||
"restore_ready": 1
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"action": "restore_ready",
|
||||
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
|
||||
}
|
||||
],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
"fresh_preserved": true,
|
||||
"used_metadata_and_link_preserved": true
|
||||
},
|
||||
"scan_cost": [
|
||||
{
|
||||
"extra_fresh_objects": 0,
|
||||
"repeats": [
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.18440870900121809,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 50,
|
||||
"fresh_objects_skipped": 24,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.14119874999960302,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 50,
|
||||
"fresh_objects_skipped": 24,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.13778470800025389,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 50,
|
||||
"fresh_objects_skipped": 24,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"extra_fresh_objects": 100,
|
||||
"repeats": [
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.16300708299968392,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 150,
|
||||
"fresh_objects_skipped": 124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.15059724999991886,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 150,
|
||||
"fresh_objects_skipped": 124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.16615074999936041,
|
||||
"counts": {
|
||||
"s3_list_pages": 1,
|
||||
"s3_objects": 150,
|
||||
"fresh_objects_skipped": 124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"extra_fresh_objects": 1100,
|
||||
"repeats": [
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.21618029099954583,
|
||||
"counts": {
|
||||
"s3_list_pages": 2,
|
||||
"s3_objects": 1150,
|
||||
"fresh_objects_skipped": 1124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.23353216699979384,
|
||||
"counts": {
|
||||
"s3_list_pages": 2,
|
||||
"s3_objects": 1150,
|
||||
"fresh_objects_skipped": 1124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
},
|
||||
{
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 300,
|
||||
"elapsed_seconds": 0.18981450000137556,
|
||||
"counts": {
|
||||
"s3_list_pages": 2,
|
||||
"s3_objects": 1150,
|
||||
"fresh_objects_skipped": 1124,
|
||||
"pg_metadata_pages": 2,
|
||||
"pg_attachments": 48,
|
||||
"fresh_metadata_skipped": 21,
|
||||
"s3_head": 27,
|
||||
"ready_preserved": 27
|
||||
},
|
||||
"actions": [],
|
||||
"actions_preview_limit": 100
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"finished_at": "2026-10-17T17:04:32.396891+00:00",
|
||||
"measurement_runtime_sha256": {
|
||||
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
|
||||
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
|
||||
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
|
||||
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
|
||||
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
|
||||
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
|
||||
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
|
||||
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
|
||||
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
|
||||
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
|
||||
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
|
||||
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
|
||||
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
|
||||
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
|
||||
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298"
|
||||
},
|
||||
"measurements_after_checks_completed": true,
|
||||
"bucket_bootstrap_repeated": 2
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: private S3, upload memory bound, interruption, outage and reconciler
|
||||
@@ -0,0 +1,13 @@
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
|
||||
rootdir: <repository>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: anyio-4.15.1
|
||||
collected 65 items
|
||||
|
||||
student-tests/test_api.py .................................... [ 55%]
|
||||
student-tests/test_attachments.py ................. [ 81%]
|
||||
student-tests/test_cluster.py ............ [100%]
|
||||
|
||||
======================= 65 passed, 2 warnings in 15.34s ========================
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"project": "mtusi-lab4",
|
||||
"services": {
|
||||
"api": {
|
||||
"image": "mtusi-messenger:lab4",
|
||||
"cpus": 1,
|
||||
"mem_limit": "536870912",
|
||||
"memswap_limit": "536870912",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"secrets": [
|
||||
"database_url",
|
||||
"cache_url",
|
||||
"s3_api_access_key_id",
|
||||
"s3_api_secret_access_key"
|
||||
]
|
||||
},
|
||||
"bucket-init": {
|
||||
"image": "mtusi-lab4-mc:2025-08-13",
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "134217728",
|
||||
"read_only": true,
|
||||
"secrets": [
|
||||
"s3_root_user",
|
||||
"s3_root_password",
|
||||
"s3_api_access_key_id",
|
||||
"s3_api_secret_access_key",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
"s3_test_access_key_id",
|
||||
"s3_test_secret_access_key"
|
||||
]
|
||||
},
|
||||
"lb": {
|
||||
"image": "haproxy:3.2.1-alpine",
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "134217728",
|
||||
"memswap_limit": "134217728",
|
||||
"ports": [
|
||||
{
|
||||
"mode": "ingress",
|
||||
"host_ip": "127.0.0.1",
|
||||
"target": 8080,
|
||||
"published": "8084",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
],
|
||||
"read_only": true,
|
||||
"secrets": []
|
||||
},
|
||||
"migrate": {
|
||||
"image": "mtusi-messenger:lab4",
|
||||
"cpus": 1,
|
||||
"mem_limit": "536870912",
|
||||
"memswap_limit": "536870912",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"secrets": [
|
||||
"database_url"
|
||||
]
|
||||
},
|
||||
"postgres": {
|
||||
"image": "postgres:18.3",
|
||||
"cpus": 1,
|
||||
"mem_limit": "805306368",
|
||||
"memswap_limit": "805306368",
|
||||
"secrets": [
|
||||
"postgres_password"
|
||||
]
|
||||
},
|
||||
"s3": {
|
||||
"image": "mtusi-lab4-s3:2025-10-15",
|
||||
"cpus": 1,
|
||||
"mem_limit": "805306368",
|
||||
"memswap_limit": "805306368",
|
||||
"read_only": true,
|
||||
"secrets": [
|
||||
"s3_root_user",
|
||||
"s3_root_password"
|
||||
]
|
||||
},
|
||||
"valkey": {
|
||||
"image": "valkey/valkey:8.1.3-alpine",
|
||||
"cpus": 0.5,
|
||||
"mem_limit": "134217728",
|
||||
"memswap_limit": "134217728",
|
||||
"secrets": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
Python 3.14.7
|
||||
uv 0.12.13 (Homebrew 2026-09-10 aarch64-apple-darwin)
|
||||
Docker client 29.8.1; server 29.8.1
|
||||
5.5.1
|
||||
minio version DEVELOPMENT.GOGET (commit-id=DEVELOPMENT.GOGET)
|
||||
Runtime: go1.26.7 linux/arm64
|
||||
License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html
|
||||
Copyright: 2015-0000 MinIO, Inc.
|
||||
@@ -0,0 +1 @@
|
||||
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::BUCKET"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::BUCKET/originals/*"]}]}
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Never print mc arguments/errors: they can include access key identifiers.
|
||||
exec 3>&1
|
||||
exec > /tmp/bootstrap.log 2>&1
|
||||
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
|
||||
bucket="$S3_BUCKET"
|
||||
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
|
||||
mc --config-dir /tmp/mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
|
||||
for role in api reconciler test; do
|
||||
target_bucket="$bucket"
|
||||
policy_source=/policies/api-policy.json
|
||||
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
|
||||
if [ "$role" = test ]; then
|
||||
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
|
||||
target_bucket="$bucket-tests"
|
||||
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
|
||||
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
|
||||
fi
|
||||
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
|
||||
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
|
||||
key="$(cat /run/secrets/s3_${role}_access_key_id)"
|
||||
mc --config-dir /tmp/mc admin user add local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
|
||||
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
|
||||
done
|
||||
trap - EXIT
|
||||
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
|
||||
@@ -0,0 +1,44 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout client 20s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 0s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
|
||||
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
|
||||
|
||||
listen metrics
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 1s
|
||||
@@ -0,0 +1 @@
|
||||
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::BUCKET"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::BUCKET/originals/*"]}]}
|
||||
@@ -0,0 +1,18 @@
|
||||
# Official security-fixed source release; no dependency on retired image tags.
|
||||
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
|
||||
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
|
||||
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
|
||||
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
|
||||
COPY --from=builder /go/bin/minio /usr/local/bin/minio
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
|
||||
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
|
||||
USER 10001:10001
|
||||
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
|
||||
|
||||
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
|
||||
COPY --from=builder /go/bin/mc /usr/local/bin/mc
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bin/sh"]
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id,user_id),
|
||||
UNIQUE (chat_id,position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at);
|
||||
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE message_requests (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
idempotency_key varchar(128) COLLATE "C" NOT NULL
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
|
||||
response_message_id uuid NOT NULL REFERENCES messages(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (id,chat_id),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages ADD UNIQUE (id,chat_id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position),
|
||||
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
|
||||
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,45 @@
|
||||
[project]
|
||||
name = "mtusi-messenger"
|
||||
version = "0.4.0"
|
||||
description = "MTUSI lab 4: private object storage and recoverable attachments"
|
||||
requires-python = ">=3.14,<3.15"
|
||||
dependencies = [
|
||||
"argon2-cffi==25.1.0",
|
||||
"boto3==1.43.109",
|
||||
"fastapi==0.143.3",
|
||||
"pillow==13.0.0",
|
||||
"psycopg[binary,pool]==3.3.6",
|
||||
"pydantic==2.14.1",
|
||||
"redis==8.1.0",
|
||||
"uvicorn==0.54.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling==1.27.0"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/messenger"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"httpx==0.28.1",
|
||||
"openapi-spec-validator==0.7.2",
|
||||
"pytest==9.1.1",
|
||||
"ruff==0.17.1",
|
||||
]
|
||||
research = ["matplotlib==3.11.2"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["student-tests"]
|
||||
addopts = "-ra --tb=short"
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py314"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.ruff.per-file-target-version]
|
||||
"scripts/*.py" = "py39"
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Render measured kernel memory peaks and dry-run traversal costs."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("input", nargs="?", type=Path, default=ROOT / "evidence/storage.json")
|
||||
args = parser.parse_args()
|
||||
d = json.loads(args.input.read_text())
|
||||
plt.rcParams.update(
|
||||
{
|
||||
"font.size": 10,
|
||||
"figure.dpi": 150,
|
||||
"axes.grid": True,
|
||||
"grid.alpha": 0.2,
|
||||
"axes.spines.top": False,
|
||||
"axes.spines.right": False,
|
||||
}
|
||||
)
|
||||
fig, ax = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
|
||||
labels = [f"{r['bytes_each'] / 1048576:.2f} MiB" for r in d["uploads"]]
|
||||
base = [max(v["current_bytes"] for v in r["before"].values()) / 1048576 for r in d["uploads"]]
|
||||
peak = [max(v["peak_bytes"] for v in r["after"].values()) / 1048576 for r in d["uploads"]]
|
||||
x = list(range(len(labels)))
|
||||
ax[0].bar([v - 0.18 for v in x], base, width=0.36, label="Before: max API memory.current")
|
||||
ax[0].bar([v + 0.18 for v in x], peak, width=0.36, label="Kernel memory.peak incl. startup")
|
||||
ax[0].axhline(512, color="gray", linestyle="--", label="Per API limit: 512 MiB")
|
||||
ax[0].set(
|
||||
xticks=x,
|
||||
xticklabels=labels,
|
||||
ylabel="MiB per API container",
|
||||
title="10 simultaneous clients, 1 active upload/API",
|
||||
)
|
||||
ax[0].legend(fontsize=7)
|
||||
cost = d["scan_cost"]
|
||||
xs = [r["extra_fresh_objects"] for r in cost]
|
||||
values = [[v["elapsed_seconds"] * 1000 for v in r["repeats"]] for r in cost]
|
||||
ax[1].plot(xs, [median(v) for v in values], marker="o", label="Median of 3 dry runs")
|
||||
ax[1].fill_between(
|
||||
xs, [min(v) for v in values], [max(v) for v in values], alpha=0.15, label="Min/max"
|
||||
)
|
||||
ax[1].set(
|
||||
xlabel="Extra fresh objects (metadata unchanged)",
|
||||
ylabel="Traversal wall time, ms",
|
||||
title="LIST pagination + metadata HEAD checks",
|
||||
)
|
||||
ax[1].legend(fontsize=8)
|
||||
fig.savefig(args.input.with_name(args.input.stem + "-research.png"))
|
||||
plt.close(fig)
|
||||
print("PASS: research figure generated from storage.json")
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once, without printing their values."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = directory / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32) + "\n")
|
||||
password = password_file.read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
|
||||
"cache_url": "redis://valkey:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
# Compose bind-mount secrets must be readable by the non-root container process.
|
||||
# The parent directory remains 0700 on the host.
|
||||
for role in ("root", "api", "reconciler", "test"):
|
||||
names = (
|
||||
("s3_root_user", "s3_root_password")
|
||||
if role == "root"
|
||||
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
|
||||
)
|
||||
for index, name in enumerate(names):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(
|
||||
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
|
||||
)
|
||||
for name in (
|
||||
"postgres_password",
|
||||
"database_url",
|
||||
"cache_url",
|
||||
*[p.name for p in directory.glob("s3_*")],
|
||||
):
|
||||
os.chmod(directory / name, 0o644)
|
||||
env = root / ".env"
|
||||
if not env.exists():
|
||||
env.write_text((root / ".env.example").read_text())
|
||||
print("Local secret files and environment are ready; existing values preserved.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab4 stack."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="http://127.0.0.1:8084", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# All running application/state services; no migration, down -v, flush or seed.
|
||||
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Local-only operator helpers; secret values never leave this process."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def settings(role="reconciler"):
|
||||
password = (ROOT / "secrets/postgres_password").read_text().strip()
|
||||
dsn = make_conninfo(
|
||||
host="127.0.0.1",
|
||||
port=55435,
|
||||
dbname="messenger",
|
||||
user="messenger",
|
||||
password=password,
|
||||
connect_timeout=2,
|
||||
options="-c timezone=UTC",
|
||||
)
|
||||
key, secret = (
|
||||
("s3_root_user", "s3_root_password")
|
||||
if role == "root"
|
||||
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
|
||||
)
|
||||
return Settings(
|
||||
dsn,
|
||||
"redis://127.0.0.1:56382/0",
|
||||
s3_access_key=(ROOT / "secrets" / key).read_text().strip(),
|
||||
s3_secret_key=(ROOT / "secrets" / secret).read_text().strip(),
|
||||
)
|
||||
|
||||
|
||||
def connection(config=None):
|
||||
config = config or settings()
|
||||
return psycopg.connect(config.database_url, autocommit=True, row_factory=dict_row)
|
||||
|
||||
|
||||
def store(role="reconciler"):
|
||||
return ObjectStore(settings(role))
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real LB requests during each replica failure and sequential replacement."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
values = sorted(values)
|
||||
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base, actor, room, workers=6):
|
||||
self.base, self.actor, self.room = base, actor, room
|
||||
self.prefix = uuid4().hex
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.workers = workers
|
||||
|
||||
def worker(self, worker):
|
||||
client, iteration = Client(self.base, timeout=3), 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
write = iteration % 10 == 9
|
||||
try:
|
||||
if write:
|
||||
key = f"{self.prefix}:{worker}:{iteration}"
|
||||
response = client.message(self.actor, self.room, key, key=key)
|
||||
else:
|
||||
response = client.call(
|
||||
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
||||
)
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": True,
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
"message": response["json"] if write else None,
|
||||
}
|
||||
iteration += 1
|
||||
except Failure:
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": False,
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
}
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.executor.shutdown(wait=True)
|
||||
for future in self.futures:
|
||||
future.result()
|
||||
self.finished = time.monotonic()
|
||||
|
||||
def report(self, action_start, action_end):
|
||||
events = sorted(self.events, key=lambda row: row["at"])
|
||||
ok = [row for row in events if row["ok"]]
|
||||
after = [row for row in ok if row["at"] >= action_end]
|
||||
check(bool(after), "No successful request after action")
|
||||
elapsed = self.finished - self.started
|
||||
latency = [row["latency_ms"] for row in ok]
|
||||
return {
|
||||
"duration_seconds": elapsed,
|
||||
"workers": self.workers,
|
||||
"requests": len(events),
|
||||
"successes": len(ok),
|
||||
"errors": len(events) - len(ok),
|
||||
"successful_rps": len(ok) / elapsed,
|
||||
"error_fraction": (len(events) - len(ok)) / len(events),
|
||||
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
||||
"action_seconds": action_end - action_start,
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"instances": sorted({row["instance"] for row in ok}),
|
||||
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
||||
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
||||
}
|
||||
|
||||
def verify_messages(self, client, actor, room):
|
||||
actual = history(client, actor, room)
|
||||
by_id = {value["id"]: value for value in actual}
|
||||
check(len(by_id) == len(actual), "History has duplicate IDs")
|
||||
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
||||
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
||||
for event in self.events:
|
||||
if event["ok"] and event["message"]:
|
||||
check(
|
||||
by_id.get(event["message"]["id"]) == event["message"],
|
||||
"A confirmed message was lost or changed",
|
||||
)
|
||||
|
||||
|
||||
def public_test(target, base):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
target,
|
||||
"ACTION_SCRIPT=scripts/"
|
||||
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
||||
"BASE_URL=" + base,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
)
|
||||
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise SystemExit("Use a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
||||
baseline = instances()
|
||||
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
||||
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
||||
wait_lb(args.base_url, 2)
|
||||
result = {"initial_instances": baseline, "failures": []}
|
||||
client = Client(args.base_url, timeout=4)
|
||||
alice, bob, eve, room = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
|
||||
def same_request(_):
|
||||
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
||||
return response["json"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
responses = list(pool.map(same_request, range(30)))
|
||||
check(
|
||||
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
||||
)
|
||||
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
||||
result["concurrent_replays"] = 30
|
||||
result["concurrent_effects"] = 1
|
||||
try:
|
||||
for selected in baseline:
|
||||
selected = instance(selected["id"])
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
||||
time.sleep(3)
|
||||
report = traffic.report(action_start, action_end)
|
||||
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
||||
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
||||
report["stopped"] = selected
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["failures"].append(report)
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
print("Verified failure and restoration of one API", flush=True)
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
replaced = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/rolling-update.py"),
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["rolling"] = traffic.report(action_start, action_end)
|
||||
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
||||
result["rolling"]["confirmed_messages_preserved"] = True
|
||||
print("Verified sequential replacement under traffic", flush=True)
|
||||
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
before = {
|
||||
service: inspect(compose("ps", "-q", service).strip())
|
||||
for service in ("postgres", "valkey", "s3", "lb")
|
||||
}
|
||||
(ROOT / "evidence/persistence.txt").write_text(
|
||||
public_test("test-persistence", args.base_url)
|
||||
)
|
||||
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
||||
check(
|
||||
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
||||
"Persistence hook did not restart all state/LB services",
|
||||
)
|
||||
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
||||
result["final_instances"] = instances()
|
||||
logs = compose("logs", "--no-color", "api", "lb")
|
||||
check(
|
||||
not any(
|
||||
a["token"] in logs or a["registration"]["password"] in logs
|
||||
for a in (alice, bob, eve)
|
||||
),
|
||||
"Credential appeared in logs",
|
||||
)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
||||
finally:
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,424 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Private S3, 10 concurrent proxy uploads, failure recovery and reconciliation cost."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from cluster import compose, inspect, instances, wait_lb
|
||||
from PIL import Image
|
||||
from storage_lab import ROOT, connection, store
|
||||
|
||||
from messenger.reconcile import Reconciler
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def put_orphan(admin):
|
||||
key = "originals/" + str(uuid4())
|
||||
admin.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
|
||||
return key
|
||||
|
||||
|
||||
def seed(path):
|
||||
admin = store("root")
|
||||
try:
|
||||
key = put_orphan(admin)
|
||||
path.write_text(
|
||||
json.dumps({"orphan": key, "created_at": datetime.now(UTC).isoformat()}) + "\n"
|
||||
)
|
||||
print("Seeded one orphan; its real S3 LastModified must age 300 seconds.")
|
||||
finally:
|
||||
admin.close()
|
||||
|
||||
|
||||
def memory(cids):
|
||||
result = {}
|
||||
for cid in cids:
|
||||
raw = subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"sh",
|
||||
"-c",
|
||||
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak "
|
||||
"/sys/fs/cgroup/cpu.stat",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
lines = raw.splitlines()
|
||||
current, peak = map(int, lines[:2])
|
||||
counters = {k: int(v) for k, v in (line.split() for line in lines[2:])}
|
||||
result[cid] = {"current_bytes": current, "peak_bytes": peak, "cpu": counters}
|
||||
return result
|
||||
|
||||
|
||||
def pressure(client, actor, chat):
|
||||
results = []
|
||||
for side in (600, 1700):
|
||||
image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3))
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="PNG", compress_level=0)
|
||||
data = output.getvalue()
|
||||
check(len(data) <= 10485760, "Fixture accidentally oversized")
|
||||
# Recreate API only, so kernel memory.peak includes this trial and startup.
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "--scale", "api=2", "api")
|
||||
wait_lb(count=2, base=client.base)
|
||||
cids = [row["id"] for row in instances()]
|
||||
resource_started = time.monotonic()
|
||||
baseline = memory(cids)
|
||||
import threading
|
||||
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def one(i, data=data, barrier=barrier):
|
||||
c = Client(client.base, timeout=30)
|
||||
barrier.wait(timeout=10)
|
||||
started = time.monotonic()
|
||||
value = c.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{i}.png",
|
||||
expected=201,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
check(value["json"]["sha256"] == hashlib.sha256(data).hexdigest(), "Checksum mismatch")
|
||||
return {
|
||||
"duration_ms": (time.monotonic() - started) * 1000,
|
||||
"instance": value["headers"]["X-Instance-Id"],
|
||||
"id": value["json"]["id"],
|
||||
}
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
values = list(pool.map(one, range(10)))
|
||||
elapsed = time.monotonic() - started
|
||||
peak = memory(cids)
|
||||
resource_seconds = time.monotonic() - resource_started
|
||||
cpu_cores = (
|
||||
sum(peak[k]["cpu"]["usage_usec"] - baseline[k]["cpu"]["usage_usec"] for k in cids)
|
||||
/ 1e6
|
||||
/ resource_seconds
|
||||
)
|
||||
values_ms = sorted(row["duration_ms"] for row in values)
|
||||
results.append(
|
||||
{
|
||||
"clients": 10,
|
||||
"bytes_each": len(data),
|
||||
"pixels": side * side,
|
||||
"duration_seconds": elapsed,
|
||||
"successful_rps": 10 / elapsed,
|
||||
"resource_window_seconds": resource_seconds,
|
||||
"mean_api_cpu_cores_including_probes": cpu_cores,
|
||||
"errors": 0,
|
||||
"p50_ms": values_ms[4],
|
||||
"p95_ms": values_ms[9],
|
||||
"p99_ms": values_ms[9],
|
||||
"before": baseline,
|
||||
"after": peak,
|
||||
"responses": values,
|
||||
}
|
||||
)
|
||||
print(f"PASS: 10 simultaneous uploads, {len(data)} bytes each", flush=True)
|
||||
return results
|
||||
|
||||
|
||||
def private(admin, api, chat, actor, client):
|
||||
value, _ = client.attachment(actor, chat)
|
||||
with connection() as conn:
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (value["id"],)).fetchone()
|
||||
# HTTP from a different container, no Authorization, no signature, no API.
|
||||
code = (
|
||||
"import urllib.request,urllib.error; "
|
||||
f'u="http://s3:9000/{api.bucket}/{row["object_key"]}"; '
|
||||
"\ntry:\n r=urllib.request.urlopen(u); print(r.status)"
|
||||
"\nexcept urllib.error.HTTPError as e: print(e.code)"
|
||||
)
|
||||
cid = instances()[0]["id"]
|
||||
status = int(subprocess.check_output(["docker", "exec", cid, "python", "-c", code], text=True))
|
||||
check(status == 403, "Anonymous S3 GET succeeded")
|
||||
for method, kwargs in [
|
||||
("delete_object", {"Bucket": api.bucket, "Key": row["object_key"]}),
|
||||
("list_objects_v2", {"Bucket": "unrelated-bucket"}),
|
||||
]:
|
||||
try:
|
||||
getattr(api.client, method)(**kwargs)
|
||||
except ClientError as exc:
|
||||
check(exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Expected IAM 403")
|
||||
else:
|
||||
raise RuntimeError("API identity exceeded its policy")
|
||||
return {
|
||||
"anonymous_get_status": status,
|
||||
"api_delete_denied": True,
|
||||
"other_bucket_denied": True,
|
||||
"attachment": value["id"],
|
||||
}
|
||||
|
||||
|
||||
def cross_replica(client, actor, reader, chat):
|
||||
value = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=switch.png",
|
||||
expected=201,
|
||||
actor=actor,
|
||||
raw=PNG,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
alias = value["headers"]["X-Instance-Id"]
|
||||
selected = next(row for row in instances() if row["instance"] == alias)
|
||||
try:
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env={**os.environ, "TARGET_INSTANCE_ID": alias},
|
||||
timeout=10,
|
||||
)
|
||||
stopped = inspect(selected["id"])
|
||||
check(not stopped["running"], "Uploading process was not stopped")
|
||||
time.sleep(1)
|
||||
result = client.call(
|
||||
"GET", "/api/v1/attachments/" + value["json"]["id"] + "/content", actor=reader
|
||||
)
|
||||
check(
|
||||
result["bytes"] == PNG and result["headers"]["X-Instance-Id"] != alias,
|
||||
"Another API failed to serve original",
|
||||
)
|
||||
return {
|
||||
"uploader": selected,
|
||||
"after_stop": stopped,
|
||||
"downloader_instance": result["headers"]["X-Instance-Id"],
|
||||
"original_preserved": True,
|
||||
}
|
||||
finally:
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(base=client.base, count=2)
|
||||
|
||||
|
||||
def interruption(client, actor, chat):
|
||||
with connection() as conn:
|
||||
before = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
|
||||
# Close in the middle of a correctly declared body; no durable intent must exist yet.
|
||||
s = socket.create_connection(("127.0.0.1", 8084), timeout=3)
|
||||
header = (
|
||||
f"POST /api/v1/chats/{chat['id']}/attachments?filename=interrupted.png HTTP/1.1\r\n"
|
||||
"Host: localhost\r\nContent-Type: image/png\r\nContent-Length: 1048576\r\n"
|
||||
f"Authorization: Bearer {actor['token']}\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
s.sendall(header.encode() + PNG[:32])
|
||||
s.close()
|
||||
time.sleep(0.5)
|
||||
with connection() as conn:
|
||||
after = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
|
||||
check(before == after, "Interrupted inbound body created an intent")
|
||||
return {"before_rows": before, "after_rows": after, "aborted_body_no_intent": True}
|
||||
|
||||
|
||||
def outage(client, actor, chat):
|
||||
att, _ = client.attachment(actor, chat)
|
||||
before = inspect(compose("ps", "-q", "s3").strip())
|
||||
compose("stop", "s3")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
statuses = []
|
||||
for method, path, raw in [
|
||||
("GET", "/api/v1/attachments/" + att["id"] + "/content", None),
|
||||
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=unavailable.png", PNG),
|
||||
]:
|
||||
request = urllib.request.Request(
|
||||
client.base + path,
|
||||
data=raw,
|
||||
method=method,
|
||||
headers={"Authorization": "Bearer " + actor["token"], "Content-Type": "image/png"},
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=8)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
statuses.append(response.code)
|
||||
check(response.code == 503, "S3 outage must produce 503")
|
||||
failed = len(statuses)
|
||||
finally:
|
||||
compose("start", "s3")
|
||||
wait_lb(base=client.base, count=2)
|
||||
after = inspect(compose("ps", "-q", "s3").strip())
|
||||
actual = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)[
|
||||
"bytes"
|
||||
]
|
||||
check(actual == PNG, "S3 restart lost original")
|
||||
return {
|
||||
"before": before,
|
||||
"after": after,
|
||||
"failed_operations": failed,
|
||||
"statuses": statuses,
|
||||
"stop_start_and_ready_seconds": time.monotonic() - started,
|
||||
"original_preserved": True,
|
||||
}
|
||||
|
||||
|
||||
def reconciliation(seed_path, admin, reconciler, client, actor, chat):
|
||||
saved = json.loads(seed_path.read_text())
|
||||
orphan = saved["orphan"]
|
||||
head = admin.head(orphan)
|
||||
check(head is not None, "Seed orphan is missing")
|
||||
age = time.time() - head["LastModified"].timestamp()
|
||||
check(age >= 300, "Seed must be at least 300s old; keep it while other checks run")
|
||||
fresh = put_orphan(admin)
|
||||
used, _ = client.attachment(actor, chat)
|
||||
client.message(actor, chat, "used attachment", attachments=[used["id"]])
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
|
||||
(used["id"],),
|
||||
)
|
||||
dry = Reconciler(conn, reconciler).run()
|
||||
check(
|
||||
any(
|
||||
a["action"] == "delete_orphan" and a["id"] == orphan.split("/")[1]
|
||||
for a in dry["actions"]
|
||||
),
|
||||
"Dry run did not detect old orphan",
|
||||
)
|
||||
check(admin.head(orphan) is not None, "Dry-run mutated S3")
|
||||
applied = Reconciler(conn, reconciler, apply=True).run()
|
||||
repeated = Reconciler(conn, reconciler, apply=True).run()
|
||||
check(admin.head(orphan) is None and admin.head(fresh) is not None, "Grace safety failed")
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (used["id"],)).fetchone()
|
||||
check(
|
||||
row["storage_state"] == "ready" and admin.head(row["object_key"]) is not None,
|
||||
"Used attachment was removed",
|
||||
)
|
||||
check(repeated["actions"] == [], "Repeated apply failed to converge")
|
||||
# Remove a USED object's bytes, preserve its metadata/link, and repair after re-upload.
|
||||
admin.client.delete_object(Bucket=admin.bucket, Key=row["object_key"])
|
||||
missing_dry = Reconciler(conn, reconciler).run()
|
||||
check(
|
||||
conn.execute(
|
||||
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
|
||||
).fetchone()["storage_state"]
|
||||
== "ready",
|
||||
"Dry run mutated metadata",
|
||||
)
|
||||
missing_apply = Reconciler(conn, reconciler, apply=True).run()
|
||||
check(
|
||||
conn.execute(
|
||||
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
|
||||
).fetchone()["storage_state"]
|
||||
== "missing",
|
||||
"Missing original was not marked",
|
||||
)
|
||||
check(
|
||||
conn.execute(
|
||||
"SELECT count(*) AS n FROM message_attachments WHERE attachment_id=%s",
|
||||
(used["id"],),
|
||||
).fetchone()["n"]
|
||||
== 1,
|
||||
"A used relation was removed",
|
||||
)
|
||||
admin.put(
|
||||
row["object_key"],
|
||||
io.BytesIO(PNG),
|
||||
len(PNG),
|
||||
"image/png",
|
||||
hashlib.sha256(PNG).hexdigest(),
|
||||
)
|
||||
healing = Reconciler(conn, reconciler, apply=True).run()
|
||||
check(
|
||||
client.call("GET", "/api/v1/attachments/" + used["id"] + "/content", actor=actor)[
|
||||
"bytes"
|
||||
]
|
||||
== PNG,
|
||||
"Restored original is wrong",
|
||||
)
|
||||
return {
|
||||
"old_orphan_age_seconds": age,
|
||||
"dry_run": dry,
|
||||
"apply": applied,
|
||||
"repeat": repeated,
|
||||
"missing_dry_run": missing_dry,
|
||||
"missing_apply": missing_apply,
|
||||
"healing": healing,
|
||||
"fresh_preserved": True,
|
||||
"used_metadata_and_link_preserved": True,
|
||||
}
|
||||
|
||||
|
||||
def scan_cost(admin, reconciler):
|
||||
results = []
|
||||
created = []
|
||||
try:
|
||||
for target in (0, 100, 1100):
|
||||
while len(created) < target:
|
||||
created.append(put_orphan(admin))
|
||||
repeats = []
|
||||
for _ in range(3):
|
||||
with connection() as conn:
|
||||
repeats.append(Reconciler(conn, reconciler).run())
|
||||
results.append({"extra_fresh_objects": target, "repeats": repeats})
|
||||
return results
|
||||
finally:
|
||||
for key in created:
|
||||
admin.client.delete_object(Bucket=admin.bucket, Key=key)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seed", type=Path)
|
||||
parser.add_argument("--resume-basics", type=Path)
|
||||
parser.add_argument("--seed-file", type=Path, default=ROOT / "evidence/storage-seed.json")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "evidence/storage.json")
|
||||
args = parser.parse_args()
|
||||
if args.seed:
|
||||
seed(args.seed)
|
||||
return
|
||||
if args.output.exists():
|
||||
raise SystemExit("Choose a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost,127.0.0.1"
|
||||
admin, api, rec = store("root"), store("api"), store("reconciler")
|
||||
try:
|
||||
client = Client("http://127.0.0.1:8084", timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
result = {"started_at": datetime.now(UTC).isoformat()}
|
||||
if args.resume_basics:
|
||||
result.update(json.loads(args.resume_basics.read_text()))
|
||||
else:
|
||||
result["privacy"] = private(admin, api, chat, alice, client)
|
||||
result["cross_replica"] = cross_replica(client, alice, bob, chat)
|
||||
result["interruption"] = interruption(client, alice, chat)
|
||||
result["uploads"] = pressure(client, alice, chat)
|
||||
result["outage"] = outage(client, alice, chat)
|
||||
result["reconciliation"] = reconciliation(args.seed_file, admin, rec, client, alice, chat)
|
||||
result["scan_cost"] = scan_cost(admin, rec)
|
||||
result["finished_at"] = datetime.now(UTC).isoformat()
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: private S3, upload memory bound, interruption, outage and reconciler", flush=True
|
||||
)
|
||||
finally:
|
||||
for value in (admin, api, rec):
|
||||
value.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""One-process implementation of the lab 1 messenger contract."""
|
||||
@@ -0,0 +1,24 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
def main():
|
||||
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
|
||||
logging.basicConfig(level=level, format="%(message)s")
|
||||
uvicorn.run(
|
||||
"messenger.app:create_app",
|
||||
factory=True,
|
||||
host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.getenv("APP_PORT", "8080")),
|
||||
workers=1,
|
||||
access_log=False,
|
||||
log_level=level.lower(),
|
||||
timeout_keep_alive=2,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,337 @@
|
||||
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from time import perf_counter
|
||||
from typing import Annotated
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from messenger.attachments import Attachments
|
||||
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User, invalid
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.schemas import (
|
||||
AttachmentView,
|
||||
ChatPage,
|
||||
ChatView,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
MessagePage,
|
||||
MessageView,
|
||||
SessionView,
|
||||
UserView,
|
||||
canonical_uuid,
|
||||
)
|
||||
from messenger.service import Messenger
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
request_id = str(uuid4())
|
||||
scope.setdefault("state", {})["request_id"] = request_id
|
||||
start, status, response_started = perf_counter(), 500, False
|
||||
|
||||
async def with_id(message: Message):
|
||||
nonlocal status, response_started
|
||||
if message["type"] == "http.response.start":
|
||||
response_started = True
|
||||
status = message["status"]
|
||||
message["headers"] = [
|
||||
pair
|
||||
for pair in message.get("headers", [])
|
||||
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
|
||||
] + [
|
||||
(b"x-request-id", request_id.encode()),
|
||||
(b"x-instance-id", self.instance_id.encode()),
|
||||
]
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, with_id)
|
||||
except Exception as exc:
|
||||
# Do not let Uvicorn print exception values that could contain user data.
|
||||
if response_started:
|
||||
raise
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
response = error_response(
|
||||
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
await response(scope, receive, with_id)
|
||||
finally:
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
logger.info(
|
||||
json.dumps(
|
||||
{
|
||||
"event": "request",
|
||||
"request_id": request_id,
|
||||
"method": scope["method"],
|
||||
"route": route,
|
||||
"status": status,
|
||||
"duration_ms": round((perf_counter() - start) * 1000, 3),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request, status: int, code: str, message: str, headers: dict | None = None
|
||||
) -> JSONResponse:
|
||||
identity = getattr(request.state, "request_id", str(uuid4()))
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": identity}},
|
||||
headers={"X-Request-Id": identity, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
settings = settings or Settings.from_env()
|
||||
repository = PostgresRepository(
|
||||
settings.database_url,
|
||||
settings.pool_size,
|
||||
settings.idempotency_ttl,
|
||||
settings.read_autocommit,
|
||||
)
|
||||
cache = SessionCache(settings.cache_url)
|
||||
store = ObjectStore(settings)
|
||||
authentication = AuthService(repository, cache, settings.session_ttl)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
repository.open()
|
||||
app.state.ready = True
|
||||
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.ready = False
|
||||
repository.close()
|
||||
cache.close()
|
||||
store.close()
|
||||
logger.info(json.dumps({"event": "stopped"}))
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
|
||||
app.state.messenger = Messenger(repository)
|
||||
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
|
||||
app.state.repository = repository
|
||||
app.state.authentication = authentication
|
||||
app.state.cache = cache
|
||||
app.state.ready = False
|
||||
|
||||
@app.exception_handler(DomainError)
|
||||
async def domain_error(request: Request, exc: DomainError):
|
||||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException):
|
||||
code = "not_found" if exc.status_code == 404 else "invalid_request"
|
||||
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
|
||||
def service(request: Request) -> Messenger:
|
||||
return request.app.state.messenger
|
||||
|
||||
def token_digest(request: Request) -> bytes:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized()
|
||||
return bearer_digest(request.headers.get("authorization"))
|
||||
|
||||
def identity(
|
||||
digest: Annotated[bytes, Depends(token_digest)],
|
||||
) -> Identity:
|
||||
return authentication.identify(digest)
|
||||
|
||||
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
|
||||
return current.user
|
||||
|
||||
async def credentials(request: Request) -> tuple[str, str]:
|
||||
if len(request.headers.getlist("authorization")) != 1:
|
||||
raise unauthorized("Basic")
|
||||
result = basic_credentials(request.headers.get("authorization"))
|
||||
if await request.body():
|
||||
raise invalid("Вход не принимает тело запроса")
|
||||
return result
|
||||
|
||||
def resource_id(chat_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(chat_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID чата") from None
|
||||
|
||||
def pagination(request: Request) -> tuple[int, str | None]:
|
||||
for key in ("limit", "cursor"):
|
||||
if len(request.query_params.getlist(key)) > 1:
|
||||
raise invalid("Параметр пагинации указан несколько раз")
|
||||
raw = request.query_params.get("limit", "50")
|
||||
try:
|
||||
if not re.fullmatch(r"[0-9]+", raw):
|
||||
raise ValueError
|
||||
limit = int(raw)
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid("limit должен быть целым от 1 до 100") from None
|
||||
return limit, request.query_params.get("cursor")
|
||||
|
||||
def json_input(request: Request):
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media != "application/json":
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
|
||||
|
||||
def message_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
|
||||
raise invalid("Требуется корректный Idempotency-Key")
|
||||
return values[0]
|
||||
|
||||
CurrentUser = Annotated[User, Depends(actor)]
|
||||
Service = Annotated[Messenger, Depends(service)]
|
||||
ChatId = Annotated[str, Depends(resource_id)]
|
||||
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
|
||||
|
||||
@app.get("/health/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/health/ready")
|
||||
def ready(request: Request):
|
||||
if not request.app.state.ready:
|
||||
raise DomainError(503, "unavailable", "Приложение не готово")
|
||||
repository.ready()
|
||||
store.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=UserView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def register(body: CreateUser):
|
||||
return authentication.register(body.username, body.display_name, body.password)
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
|
||||
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
|
||||
return authentication.login(*value)
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(current: Annotated[Identity, Depends(identity)]):
|
||||
authentication.logout(current)
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=UserView)
|
||||
def me(current: CurrentUser):
|
||||
return current
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=ChatView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
|
||||
return messenger.create_chat(current, body.title, body.member_ids)
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def chats(current: CurrentUser, messenger: Service, page: Pagination):
|
||||
return messenger.chats(current, *page)
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
|
||||
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
|
||||
return messenger.chat(current, chat)
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=MessageView,
|
||||
dependencies=[Depends(json_input)],
|
||||
)
|
||||
def send_message(
|
||||
body: CreateMessage,
|
||||
current: CurrentUser,
|
||||
chat: ChatId,
|
||||
messenger: Service,
|
||||
key: Annotated[str, Depends(message_key)],
|
||||
):
|
||||
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
|
||||
return messenger.messages(current, chat, *page)
|
||||
|
||||
def attachment_id(attachment_id: str) -> str:
|
||||
try:
|
||||
return canonical_uuid(attachment_id)
|
||||
except ValueError:
|
||||
raise invalid("Некорректный UUID вложения") from None
|
||||
|
||||
AttachmentId = Annotated[str, Depends(attachment_id)]
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201, response_model=AttachmentView)
|
||||
async def upload(request: Request, current: CurrentUser, chat: ChatId):
|
||||
return await app.state.attachments.upload(request, current, chat)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
|
||||
def attachment(current: CurrentUser, id: AttachmentId):
|
||||
return app.state.attachments.metadata(current, id)
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def content(request: Request, current: CurrentUser, id: AttachmentId):
|
||||
row = app.state.attachments.record(current, id)
|
||||
values = request.query_params.getlist("variant")
|
||||
if len(values) > 1 or values and values[0] != "original":
|
||||
raise invalid("Некорректный вариант вложения")
|
||||
body = store.download(row)
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
while data := body.read(65536):
|
||||
yield data
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=row["content_type"],
|
||||
headers={
|
||||
"Content-Length": str(row["size_bytes"]),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Bounded binary upload. Durable intent precedes S3; ready follows both stores."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
from anyio import to_thread
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from messenger.domain import DomainError, invalid, new_id, not_found
|
||||
from messenger.repository import attachment_value
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def filename(value):
|
||||
if (
|
||||
not value
|
||||
or not value.strip()
|
||||
or len(value) > 255
|
||||
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
|
||||
):
|
||||
raise invalid("Некорректное имя файла")
|
||||
return value
|
||||
|
||||
|
||||
def image_type(file, maximum):
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
if image.width * image.height > maximum:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise ValueError
|
||||
media = "image/png" if image.format == "PNG" else "image/jpeg"
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load() # Reject truncated pixel streams, not just an intact header.
|
||||
file.seek(0)
|
||||
return media
|
||||
except Image.DecompressionBombError, Image.DecompressionBombWarning:
|
||||
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
|
||||
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
|
||||
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
|
||||
|
||||
|
||||
class Attachments:
|
||||
def __init__(self, repository, messenger, store, settings):
|
||||
self.repository, self.messenger, self.store = repository, messenger, store
|
||||
self.settings = settings
|
||||
self.slots = asyncio.Semaphore(settings.upload_slots)
|
||||
|
||||
async def upload(self, request, actor, chat_id):
|
||||
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
|
||||
values = request.query_params.getlist("filename")
|
||||
if len(values) != 1:
|
||||
raise invalid("Требуется одно имя файла")
|
||||
name = filename(values[0])
|
||||
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if media not in ("image/png", "image/jpeg"):
|
||||
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal():
|
||||
raise invalid("Некорректный Content-Length")
|
||||
if int(length) > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
try:
|
||||
await asyncio.wait_for(self.slots.acquire(), timeout=5)
|
||||
except TimeoutError:
|
||||
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
|
||||
try:
|
||||
with tempfile.TemporaryFile() as file:
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(15):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > self.settings.max_upload_bytes:
|
||||
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
|
||||
digest.update(chunk)
|
||||
await to_thread.run_sync(file.write, chunk)
|
||||
except TimeoutError:
|
||||
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
|
||||
except ClientDisconnect:
|
||||
raise invalid("Загрузка прервана") from None
|
||||
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
|
||||
if actual != media:
|
||||
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
|
||||
id = new_id()
|
||||
row = await to_thread.run_sync(
|
||||
self.repository.begin_attachment,
|
||||
id,
|
||||
chat_id,
|
||||
actor.id,
|
||||
name,
|
||||
actual,
|
||||
size,
|
||||
digest.hexdigest(),
|
||||
)
|
||||
# Leave the committed intent on every ambiguous failure. Reconciler owns recovery.
|
||||
await to_thread.run_sync(
|
||||
self.store.put, row["object_key"], file, size, actual, digest.hexdigest()
|
||||
)
|
||||
return await to_thread.run_sync(self.repository.finish_attachment, id)
|
||||
finally:
|
||||
self.slots.release()
|
||||
|
||||
def record(self, actor, id):
|
||||
row = self.repository.attachment(id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
self.messenger.chat(actor, str(row["chat_id"]))
|
||||
if row["storage_state"] != "ready":
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
return row
|
||||
|
||||
def metadata(self, actor, id):
|
||||
return attachment_value(self.record(actor, id))
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from threading import BoundedSemaphore
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from argon2.profiles import RFC_9106_LOW_MEMORY
|
||||
|
||||
from messenger.cache import SessionCache
|
||||
from messenger.domain import DomainError, User
|
||||
from messenger.repository import PostgresRepository
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> DomainError:
|
||||
challenge = scheme + ' realm="messenger"'
|
||||
if scheme == "Basic":
|
||||
challenge += ', charset="UTF-8"'
|
||||
return DomainError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется действительная идентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def basic_credentials(header: str | None) -> tuple[str, str]:
|
||||
try:
|
||||
if header is None or len(header) > 2048:
|
||||
raise ValueError
|
||||
scheme, encoded = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
raise ValueError
|
||||
if not 12 <= len(password) <= 128 or not password.strip():
|
||||
raise ValueError
|
||||
return username, password
|
||||
except ValueError, UnicodeError, binascii.Error:
|
||||
raise unauthorized("Basic") from None
|
||||
|
||||
|
||||
def bearer_digest(header: str | None) -> bytes:
|
||||
if header:
|
||||
parts = header.split(" ")
|
||||
if len(parts) == 2 and parts[0].lower() == "bearer":
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
|
||||
return hashlib.sha256(parts[1].encode()).digest()
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Identity:
|
||||
user: User
|
||||
token_hash: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
|
||||
self.repository, self.cache, self.ttl = repository, cache, ttl
|
||||
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
|
||||
self.kdf_slots = BoundedSemaphore(2)
|
||||
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
@contextmanager
|
||||
def kdf(self):
|
||||
if not self.kdf_slots.acquire(timeout=0.5):
|
||||
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.kdf_slots.release()
|
||||
|
||||
def register(self, username: str, display_name: str, password: str) -> User:
|
||||
with self.kdf():
|
||||
encoded = self.hasher.hash(password)
|
||||
return self.repository.add_user(username, display_name, encoded)
|
||||
|
||||
def login(self, username: str, password: str) -> Session:
|
||||
credentials = self.repository.credentials(username)
|
||||
encoded = credentials[1] if credentials else self.dummy_hash
|
||||
try:
|
||||
with self.kdf():
|
||||
self.hasher.verify(encoded, password)
|
||||
except VerificationError, InvalidHashError:
|
||||
raise unauthorized("Basic") from None
|
||||
if credentials is None:
|
||||
raise unauthorized("Basic")
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode()).digest()
|
||||
user = credentials[0]
|
||||
expires = self.repository.create_session(digest, user.id, self.ttl)
|
||||
return Session(token, "Bearer", expires, user)
|
||||
|
||||
def identify(self, digest: bytes) -> Identity:
|
||||
session = self.repository.active_session(digest)
|
||||
if session is None:
|
||||
raise unauthorized()
|
||||
user_id = str(session["user_id"])
|
||||
user = self.cache.get(digest, user_id)
|
||||
if user is None:
|
||||
user = self.repository.get_user(user_id)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
|
||||
self.cache.put(digest, user, ttl)
|
||||
return Identity(user, digest)
|
||||
|
||||
def logout(self, identity: Identity):
|
||||
self.repository.revoke_session(identity.token_hash)
|
||||
self.cache.delete(identity.token_hash)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Disposable session-user cache; PostgreSQL always decides token validity."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from messenger.domain import User
|
||||
|
||||
logger = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str):
|
||||
self.client = Redis.from_url(
|
||||
url,
|
||||
socket_connect_timeout=0.2,
|
||||
socket_timeout=0.2,
|
||||
max_connections=8,
|
||||
decode_responses=True,
|
||||
retry_on_timeout=False,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session-user:" + digest.hex()
|
||||
|
||||
def get(self, digest: bytes, user_id: str) -> User | None:
|
||||
try:
|
||||
raw = self.client.get(self.key(digest))
|
||||
if raw:
|
||||
value = User(**json.loads(raw))
|
||||
if value.id == user_id:
|
||||
return value
|
||||
except RedisError, ValueError, TypeError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
|
||||
return None
|
||||
|
||||
def put(self, digest: bytes, user: User, ttl: int):
|
||||
try:
|
||||
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
|
||||
|
||||
def delete(self, digest: bytes):
|
||||
try:
|
||||
self.client.delete(self.key(digest))
|
||||
except RedisError:
|
||||
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Versioned keyset position, bound to a list and its resource, never an offset."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.domain import HistoryKey, invalid
|
||||
|
||||
|
||||
def encode_cursor(scope: str, position: HistoryKey) -> str:
|
||||
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
|
||||
return (
|
||||
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
|
||||
|
||||
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not value or len(value) > 1024:
|
||||
raise ValueError
|
||||
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
|
||||
data = json.loads(raw)
|
||||
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
|
||||
raise ValueError
|
||||
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
|
||||
raise ValueError
|
||||
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
|
||||
raise ValueError
|
||||
stamp = datetime.fromisoformat(data["created_at"])
|
||||
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
|
||||
raise ValueError
|
||||
identity = str(UUID(data["id"]))
|
||||
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
|
||||
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
|
||||
raise invalid("Некорректный cursor или другая область пагинации") from None
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Immutable domain values; no dependency on HTTP or a storage technology."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def new_id() -> str:
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message: str = "Некорректный запрос") -> DomainError:
|
||||
return DomainError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def not_found() -> DomainError:
|
||||
return DomainError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
status: str = "ready"
|
||||
variants: tuple = ()
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
type HistoryKey = tuple[str, str]
|
||||
|
||||
|
||||
def history_key(item: Chat | Message) -> HistoryKey:
|
||||
return item.created_at, item.id
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Transactional, checksummed migrations, serialized across concurrent starts."""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from messenger.settings import secret
|
||||
|
||||
|
||||
def migrate(conninfo: str, directory: Path):
|
||||
with psycopg.connect(conninfo, connect_timeout=5) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version text PRIMARY KEY, checksum text NOT NULL, "
|
||||
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
|
||||
)
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
content = path.read_text()
|
||||
checksum = hashlib.sha256(content.encode()).hexdigest()
|
||||
current = conn.execute(
|
||||
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
|
||||
).fetchone()
|
||||
if current:
|
||||
if current[0] != checksum:
|
||||
raise RuntimeError("Applied migration checksum changed: " + path.stem)
|
||||
continue
|
||||
conn.execute(content)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
|
||||
(path.stem, checksum),
|
||||
)
|
||||
print("Applied migration " + path.stem)
|
||||
|
||||
|
||||
def main():
|
||||
migrate(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Conservative, repeatable PG/S3 reconciliation. Dry-run is the default."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from datetime import timedelta
|
||||
from time import perf_counter
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from messenger.settings import Settings
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
OWNED = re.compile(r"originals/[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\Z")
|
||||
|
||||
|
||||
class Reconciler:
|
||||
def __init__(self, conn, store, *, apply=False, grace=300):
|
||||
if grace < 300:
|
||||
raise ValueError("Minimum grace period is 300 seconds")
|
||||
self.conn, self.store, self.apply, self.grace = conn, store, apply, grace
|
||||
self.counts, self.actions = Counter(), []
|
||||
|
||||
def action(self, kind, id):
|
||||
self.counts[kind] += 1
|
||||
if len(self.actions) < 100:
|
||||
self.actions.append({"action": kind, "id": str(id)})
|
||||
|
||||
def head(self, key):
|
||||
self.counts["s3_head"] += 1
|
||||
return self.store.head(key)
|
||||
|
||||
def objects(self, cutoff):
|
||||
paginator = self.store.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(
|
||||
Bucket=self.store.bucket, Prefix="originals/", PaginationConfig={"PageSize": 1000}
|
||||
):
|
||||
self.counts["s3_list_pages"] += 1
|
||||
for item in page.get("Contents", []):
|
||||
self.counts["s3_objects"] += 1
|
||||
key = item["Key"]
|
||||
if not OWNED.fullmatch(key):
|
||||
self.counts["foreign_keys_ignored"] += 1
|
||||
continue
|
||||
if item["LastModified"] >= cutoff:
|
||||
self.counts["fresh_objects_skipped"] += 1
|
||||
continue
|
||||
with self.conn.transaction():
|
||||
existing = self.conn.execute(
|
||||
"SELECT id FROM attachments WHERE object_key=%s FOR UPDATE",
|
||||
(key,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
continue
|
||||
# Re-check immutable generation/age immediately before deleting an orphan.
|
||||
current = self.head(key)
|
||||
if (
|
||||
current is None
|
||||
or current["LastModified"] >= cutoff
|
||||
or current["ETag"] != item["ETag"]
|
||||
or current["ContentLength"] != item["Size"]
|
||||
):
|
||||
continue
|
||||
self.action("delete_orphan", key.removeprefix("originals/"))
|
||||
if self.apply:
|
||||
self.store.client.delete_object(Bucket=self.store.bucket, Key=key)
|
||||
self.counts["s3_delete"] += 1
|
||||
|
||||
@staticmethod
|
||||
def matches(row, head):
|
||||
return (
|
||||
head is not None
|
||||
and head["ContentLength"] == row["size_bytes"]
|
||||
and head.get("ContentType") == row["content_type"]
|
||||
and head.get("Metadata", {}).get("sha256") == row["sha256"]
|
||||
)
|
||||
|
||||
def metadata(self, cutoff):
|
||||
last = "00000000-0000-0000-0000-000000000000"
|
||||
while True:
|
||||
batch = self.conn.execute(
|
||||
"SELECT id FROM attachments WHERE id>%s::uuid ORDER BY id LIMIT 100",
|
||||
(last,),
|
||||
).fetchall()
|
||||
self.counts["pg_metadata_pages"] += 1
|
||||
if not batch:
|
||||
break
|
||||
for value in batch:
|
||||
id = value["id"]
|
||||
last = str(id)
|
||||
self.counts["pg_attachments"] += 1
|
||||
with self.conn.transaction():
|
||||
row = self.conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
continue
|
||||
state = row["storage_state"]
|
||||
if row["updated_at"] >= cutoff and state != "missing":
|
||||
self.counts["fresh_metadata_skipped"] += 1
|
||||
continue
|
||||
head = self.head(row["object_key"])
|
||||
valid = self.matches(row, head)
|
||||
if state == "pending" and head is None:
|
||||
used = self.conn.execute(
|
||||
"SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1",
|
||||
(id,),
|
||||
).fetchone()
|
||||
if used:
|
||||
self.action("review_used_pending", id)
|
||||
continue
|
||||
self.action("delete_abandoned_intent", id)
|
||||
if self.apply:
|
||||
self.conn.execute("DELETE FROM attachments WHERE id=%s", (id,))
|
||||
elif state in ("pending", "missing") and valid:
|
||||
self.action("restore_ready", id)
|
||||
if self.apply:
|
||||
self.conn.execute(
|
||||
"UPDATE attachments SET storage_state='ready',"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(id,),
|
||||
)
|
||||
elif state == "ready" and not valid:
|
||||
self.action("mark_missing", id)
|
||||
if self.apply:
|
||||
self.conn.execute(
|
||||
"UPDATE attachments SET storage_state='missing',"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(id,),
|
||||
)
|
||||
elif state == "pending" and not valid:
|
||||
self.action("review_invalid_pending", id)
|
||||
elif state == "missing":
|
||||
self.counts["still_missing"] += 1
|
||||
else:
|
||||
self.counts["ready_preserved"] += 1
|
||||
|
||||
def run(self):
|
||||
started = perf_counter()
|
||||
if not self.conn.execute("SELECT pg_try_advisory_lock(2301004) AS locked").fetchone()[
|
||||
"locked"
|
||||
]:
|
||||
raise RuntimeError("Another reconciler is active")
|
||||
try:
|
||||
cutoff = self.conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]
|
||||
cutoff -= timedelta(seconds=self.grace)
|
||||
self.store.ready() # Permission/network errors are never interpreted as missing bytes.
|
||||
self.objects(cutoff)
|
||||
self.metadata(cutoff)
|
||||
return {
|
||||
"mode": "apply" if self.apply else "dry-run",
|
||||
"grace_seconds": self.grace,
|
||||
"elapsed_seconds": perf_counter() - started,
|
||||
"counts": dict(self.counts),
|
||||
"actions": self.actions,
|
||||
"actions_preview_limit": 100,
|
||||
}
|
||||
finally:
|
||||
self.conn.execute("SELECT pg_advisory_unlock(2301004)")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--grace-seconds", type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 300:
|
||||
parser.error("Grace period must be at least 300 seconds")
|
||||
store = None
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
store = ObjectStore(settings)
|
||||
with psycopg.connect(
|
||||
settings.database_url,
|
||||
autocommit=True,
|
||||
row_factory=dict_row,
|
||||
connect_timeout=2,
|
||||
options="-c lock_timeout=1000 "
|
||||
"-c statement_timeout=2500 -c idle_in_transaction_session_timeout=5000 "
|
||||
"-c timezone=UTC",
|
||||
) as conn:
|
||||
print(
|
||||
json.dumps(
|
||||
Reconciler(conn, store, apply=args.apply, grace=args.grace_seconds).run(),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"error": "reconciliation_incomplete",
|
||||
"type": type(exc).__name__,
|
||||
"changes_may_have_committed": args.apply,
|
||||
}
|
||||
)
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
finally:
|
||||
if store:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,353 @@
|
||||
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
|
||||
|
||||
from messenger.domain import (
|
||||
Attachment,
|
||||
Chat,
|
||||
DomainError,
|
||||
HistoryKey,
|
||||
Message,
|
||||
User,
|
||||
invalid,
|
||||
new_id,
|
||||
)
|
||||
|
||||
|
||||
def stamp(value: datetime) -> str:
|
||||
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_value(row) -> User:
|
||||
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
|
||||
|
||||
|
||||
def chat_value(row) -> Chat:
|
||||
return Chat(
|
||||
str(row["id"]),
|
||||
row["title"],
|
||||
tuple(str(x) for x in row["member_ids"]),
|
||||
stamp(row["created_at"]),
|
||||
)
|
||||
|
||||
|
||||
def message_value(row) -> Message:
|
||||
return Message(
|
||||
str(row["id"]),
|
||||
str(row["chat_id"]),
|
||||
str(row["sender_id"]),
|
||||
row["text"],
|
||||
stamp(row["created_at"]),
|
||||
tuple(str(x) for x in row.get("attachment_ids", ())),
|
||||
)
|
||||
|
||||
|
||||
def attachment_value(row) -> Attachment:
|
||||
return Attachment(
|
||||
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
|
||||
row["filename"],
|
||||
row["content_type"],
|
||||
row["size_bytes"],
|
||||
row["sha256"],
|
||||
stamp(row["created_at"]),
|
||||
stamp(row["updated_at"]),
|
||||
)
|
||||
|
||||
|
||||
class Repository(Protocol):
|
||||
def get_user(self, user_id: str) -> User | None: ...
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
|
||||
def get_chat(self, chat_id: str) -> Chat | None: ...
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message: ...
|
||||
def page_messages(
|
||||
self, chat_id: str, after: HistoryKey | None, limit: int
|
||||
) -> list[Message]: ...
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
|
||||
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
|
||||
|
||||
|
||||
class PostgresRepository:
|
||||
def __init__(
|
||||
self,
|
||||
conninfo: str,
|
||||
pool_size: int = 8,
|
||||
idempotency_ttl: int = 86400,
|
||||
read_autocommit: bool = True,
|
||||
):
|
||||
self.read_autocommit = read_autocommit
|
||||
self.idempotency_ttl = idempotency_ttl
|
||||
self.pool = ConnectionPool(
|
||||
conninfo,
|
||||
min_size=1,
|
||||
max_size=pool_size,
|
||||
timeout=1,
|
||||
max_waiting=32,
|
||||
open=False,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": True,
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
|
||||
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
|
||||
},
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=15)
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=5)
|
||||
|
||||
@contextmanager
|
||||
def connection(self, *, write: bool = False):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
transaction = (
|
||||
conn.transaction() if write or not self.read_autocommit else nullcontext()
|
||||
)
|
||||
with transaction:
|
||||
yield conn
|
||||
except (
|
||||
psycopg.OperationalError,
|
||||
psycopg.InterfaceError,
|
||||
PoolTimeout,
|
||||
TooManyRequests,
|
||||
errors.QueryCanceled,
|
||||
errors.LockNotAvailable,
|
||||
):
|
||||
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
|
||||
|
||||
def ready(self):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT count(*) AS n FROM schema_migrations "
|
||||
"WHERE version IN ('001_initial','002_idempotency','003_attachments')"
|
||||
).fetchone()
|
||||
if row["n"] != 3:
|
||||
raise DomainError(503, "unavailable", "Схема ещё не готова")
|
||||
|
||||
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (id,username,display_name,password_hash) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return user_value(row)
|
||||
except errors.UniqueViolation:
|
||||
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
|
||||
|
||||
def credentials(self, username: str):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
|
||||
return (user_value(row), row["password_hash"]) if row else None
|
||||
|
||||
def get_user(self, user_id: str) -> User | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
|
||||
return user_value(row) if row else None
|
||||
|
||||
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
|
||||
try:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
|
||||
[(row["id"], member, i) for i, member in enumerate(member_ids)],
|
||||
)
|
||||
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
|
||||
except errors.ForeignKeyViolation:
|
||||
raise invalid("Указан неизвестный участник") from None
|
||||
|
||||
def get_chat(self, chat_id: str) -> Chat | None:
|
||||
with self.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
|
||||
).fetchone()
|
||||
return chat_value(row) if row else None
|
||||
|
||||
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
|
||||
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (user_id, *after, limit) if after else (user_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
|
||||
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
|
||||
+ condition
|
||||
+ " ORDER BY c.created_at,c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [chat_value(row) for row in rows]
|
||||
|
||||
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
return message_value(row)
|
||||
|
||||
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
|
||||
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
|
||||
params = (chat_id, *after, limit) if after else (chat_id, limit)
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT "
|
||||
+ MESSAGE_COLUMNS
|
||||
+ " FROM messages m WHERE m.chat_id=%s"
|
||||
+ condition
|
||||
+ " ORDER BY m.created_at,m.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
return [message_value(row) for row in rows]
|
||||
|
||||
def idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
key: str,
|
||||
attachment_ids: tuple[str, ...] = (),
|
||||
) -> Message:
|
||||
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
|
||||
payload = json.dumps(
|
||||
{"text": text, "attachment_ids": list(attachment_ids)},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
digest = hashlib.sha256(payload.encode()).digest()
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
previous = conn.execute(
|
||||
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
|
||||
"JOIN messages m ON m.id=r.response_message_id "
|
||||
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
|
||||
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
|
||||
(sender_id, chat_id, key),
|
||||
).fetchone()
|
||||
if previous:
|
||||
if previous["payload_hash"] != digest:
|
||||
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
|
||||
return message_value(previous)
|
||||
conn.execute(
|
||||
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
|
||||
"AND chat_id=%s AND idempotency_key=%s",
|
||||
(sender_id, chat_id, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
|
||||
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
|
||||
(chat_id, list(attachment_ids)),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise DomainError(404, "not_found", "Объект не найден")
|
||||
if any(row["storage_state"] != "ready" for row in rows):
|
||||
raise DomainError(409, "conflict", "Вложение пока недоступно")
|
||||
message = message_value(
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id,chat_id,sender_id,text) "
|
||||
"VALUES (%s,%s,%s,%s) RETURNING *",
|
||||
(new_id(), chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
)
|
||||
for position, attachment in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
|
||||
(message.id, chat_id, attachment, position),
|
||||
)
|
||||
message = Message(
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at,
|
||||
attachment_ids,
|
||||
)
|
||||
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
|
||||
return message
|
||||
|
||||
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
|
||||
conn.execute(
|
||||
"INSERT INTO message_requests "
|
||||
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
|
||||
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
|
||||
)
|
||||
|
||||
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO sessions (token_hash,user_id,expires_at) "
|
||||
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
|
||||
(token_hash, user_id, ttl),
|
||||
).fetchone()
|
||||
return stamp(row["expires_at"])
|
||||
|
||||
def active_session(self, token_hash: bytes):
|
||||
with self.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
|
||||
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
|
||||
def revoke_session(self, token_hash: bytes):
|
||||
with self.connection(write=True) as conn:
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
|
||||
|
||||
def begin_attachment(self, id, chat_id, owner_id, filename, content_type, size, digest):
|
||||
with self.connection(write=True) as conn:
|
||||
return conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
|
||||
"object_key,storage_state) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending') RETURNING *",
|
||||
(id, chat_id, owner_id, filename, content_type, size, digest, "originals/" + id),
|
||||
).fetchone()
|
||||
|
||||
def finish_attachment(self, id):
|
||||
with self.connection(write=True) as conn:
|
||||
row = conn.execute(
|
||||
"UPDATE attachments SET storage_state='ready',updated_at=clock_timestamp() "
|
||||
"WHERE id=%s AND storage_state='pending' RETURNING *",
|
||||
(id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise DomainError(503, "unavailable", "Загрузка не завершена")
|
||||
return attachment_value(row)
|
||||
|
||||
def attachment(self, id):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Strict lab 4 wire types; password never belongs to a response User."""
|
||||
|
||||
import re
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def canonical_uuid(value: str) -> str:
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
|
||||
raise ValueError("Ожидается UUID")
|
||||
return str(UUID(value))
|
||||
|
||||
|
||||
def nonblank(value: str) -> str:
|
||||
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
|
||||
raise ValueError("Ожидается непустая строка Unicode")
|
||||
return value
|
||||
|
||||
|
||||
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
|
||||
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
|
||||
|
||||
|
||||
class Input(BaseModel):
|
||||
model_config = ConfigDict(strict=True, extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(Input):
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
display_name: ShortText
|
||||
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def valid_username(cls, value: str) -> str:
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
|
||||
raise ValueError("Недопустимое имя пользователя")
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(Input):
|
||||
title: ShortText
|
||||
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
|
||||
|
||||
|
||||
class CreateMessage(Input):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def valid_content(self):
|
||||
if (
|
||||
not self.text.strip()
|
||||
and not self.attachment_ids
|
||||
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
|
||||
or len(set(self.attachment_ids)) != len(self.attachment_ids)
|
||||
):
|
||||
raise ValueError("Некорректное содержимое сообщения")
|
||||
return self
|
||||
|
||||
|
||||
class View(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, extra="forbid")
|
||||
|
||||
|
||||
class UserView(View):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionView(View):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: str
|
||||
user: UserView
|
||||
|
||||
|
||||
class ChatView(View):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessageView(View):
|
||||
attachment_ids: list[str]
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(View):
|
||||
items: list[ChatView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class MessagePage(View):
|
||||
items: list[MessageView]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class AttachmentView(View):
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from messenger.cursor import decode_cursor, encode_cursor
|
||||
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
|
||||
from messenger.repository import Repository
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page[T]:
|
||||
items: list[T]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class Messenger:
|
||||
def __init__(self, repository: Repository):
|
||||
self.repository = repository
|
||||
|
||||
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
|
||||
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
|
||||
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
|
||||
return self.repository.add_chat(title, tuple(member_ids))
|
||||
|
||||
def chat(self, actor: User, chat_id: str) -> Chat:
|
||||
chat = self.repository.get_chat(chat_id)
|
||||
if chat is None or actor.id not in chat.member_ids:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
|
||||
scope = "chats:" + actor.id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
|
||||
|
||||
def send(
|
||||
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
|
||||
) -> Message:
|
||||
self.chat(actor, chat_id)
|
||||
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
|
||||
|
||||
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
|
||||
self.chat(actor, chat_id)
|
||||
scope = "messages:" + chat_id
|
||||
after = decode_cursor(cursor, scope)
|
||||
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
|
||||
|
||||
@staticmethod
|
||||
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
|
||||
visible = items[:limit]
|
||||
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
|
||||
return Page(visible, cursor)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Explicit configuration; credentials are read from untracked secret files."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name: str, default: str) -> str:
|
||||
return Path(os.getenv(name, default)).read_text().strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
database_url: str = field(repr=False)
|
||||
cache_url: str = field(repr=False)
|
||||
session_ttl: int = 3600
|
||||
pool_size: int = 8
|
||||
idempotency_ttl: int = 86400
|
||||
read_autocommit: bool = True
|
||||
|
||||
s3_endpoint: str = "http://127.0.0.1:59004"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key: str = field(default="", repr=False)
|
||||
s3_secret_key: str = field(default="", repr=False)
|
||||
max_upload_bytes: int = 10485760
|
||||
max_image_pixels: int = 40000000
|
||||
upload_slots: int = 1
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
|
||||
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
if idempotency_ttl < 86400:
|
||||
raise ValueError("Idempotency retention must be at least 24 hours")
|
||||
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
|
||||
raise ValueError("Invalid session lifetime or connection pool size")
|
||||
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
|
||||
raise ValueError("Invalid upload limits")
|
||||
return cls(
|
||||
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
|
||||
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
|
||||
ttl,
|
||||
size,
|
||||
idempotency_ttl,
|
||||
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
|
||||
os.getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
os.getenv("S3_REGION", "us-east-1"),
|
||||
os.getenv("S3_BUCKET", "messenger-attachments"),
|
||||
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id"),
|
||||
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key"),
|
||||
maximum,
|
||||
pixels,
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from messenger.domain import DomainError
|
||||
|
||||
|
||||
def unavailable():
|
||||
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
config=Config(
|
||||
connect_timeout=0.5,
|
||||
read_timeout=2,
|
||||
max_pool_connections=8,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def put(self, key, file, size, media, digest):
|
||||
try:
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=file,
|
||||
ContentLength=size,
|
||||
ContentType=media,
|
||||
Metadata={"sha256": digest},
|
||||
)
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if str(exc.response.get("Error", {}).get("Code")) in ("404", "NoSuchKey", "NotFound"):
|
||||
return None
|
||||
raise unavailable() from None
|
||||
except BotoCoreError:
|
||||
raise unavailable() from None
|
||||
|
||||
def download(self, row):
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
|
||||
if response["ContentLength"] != row["size_bytes"]:
|
||||
response["Body"].close()
|
||||
raise unavailable()
|
||||
return response["Body"]
|
||||
except BotoCoreError, ClientError:
|
||||
raise unavailable() from None
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,85 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from psycopg import sql
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.migrate import migrate
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PASSWORD = "Учебный:пароль😀-12345"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def settings():
|
||||
password = (ROOT / "secrets/postgres_password").read_text().strip()
|
||||
admin = make_conninfo(
|
||||
host="127.0.0.1", port=55435, user="messenger", password=password, dbname="messenger"
|
||||
)
|
||||
database = "student_tests_" + uuid4().hex
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
|
||||
config = Settings(
|
||||
make_conninfo(admin, dbname=database),
|
||||
"redis://127.0.0.1:56382/1",
|
||||
s3_bucket="messenger-attachments-tests",
|
||||
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
|
||||
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
|
||||
)
|
||||
migrate(config.database_url, ROOT / "migrations")
|
||||
with pytest.MonkeyPatch.context() as environment:
|
||||
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
|
||||
environment.setenv("S3_BUCKET", config.s3_bucket)
|
||||
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
|
||||
environment.setenv(
|
||||
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
|
||||
)
|
||||
yield config
|
||||
with psycopg.connect(admin, autocommit=True) as conn:
|
||||
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(settings):
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
|
||||
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
|
||||
client.app.state.cache.client.flushdb()
|
||||
yield client
|
||||
|
||||
|
||||
def basic(username, password=PASSWORD):
|
||||
return {
|
||||
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actors(client):
|
||||
result = []
|
||||
for name in ("alice", "bob", "eve"):
|
||||
r = client.post(
|
||||
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
|
||||
)
|
||||
assert r.status_code == 201
|
||||
session = client.post("/api/v1/auth/sessions", headers=basic(name))
|
||||
assert session.status_code == 201
|
||||
result.append({**r.json(), "token": session.json()["token"]})
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def room(client, actors):
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers={"Authorization": "Bearer " + actors[0]["token"]},
|
||||
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
return response.json()
|
||||
@@ -0,0 +1,299 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from conftest import PASSWORD, ROOT, basic
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from messenger.app import create_app
|
||||
from messenger.auth import basic_credentials
|
||||
from messenger.migrate import migrate
|
||||
|
||||
|
||||
def headers(actor):
|
||||
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
|
||||
|
||||
|
||||
def error(response, status, code):
|
||||
assert response.status_code == status, response.text
|
||||
body = response.json()["error"]
|
||||
assert set(body) == {"code", "message", "request_id"}
|
||||
assert body["code"] == code
|
||||
assert body["request_id"] == response.headers["X-Request-Id"]
|
||||
assert len(response.headers.get_list("X-Request-Id")) == 1
|
||||
if status == 401:
|
||||
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
|
||||
def test_malformed_json(client, body):
|
||||
error(
|
||||
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"username": "ab"},
|
||||
{"username": "a" * 33},
|
||||
{"username": "Upper"},
|
||||
{"username": "кириллица"},
|
||||
{"username": "valid\n"},
|
||||
{"username": 123},
|
||||
{"username": True},
|
||||
{"display_name": 123},
|
||||
{"display_name": None},
|
||||
{"display_name": " \n\t"},
|
||||
{"display_name": "😀" * 101},
|
||||
{"display_name": "\ud800"},
|
||||
{"password": "short"},
|
||||
{"password": "😀" * 129},
|
||||
{"password": 123},
|
||||
{"password": " " * 12},
|
||||
{"password": "\ud800" * 12},
|
||||
{"extra": True},
|
||||
],
|
||||
)
|
||||
def test_strict_registration(client, change):
|
||||
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
|
||||
response = client.post(
|
||||
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
|
||||
)
|
||||
error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_unicode_password_hash_and_authentication(client, settings, actors):
|
||||
for u in actors:
|
||||
assert "password" not in u and "password_hash" not in u
|
||||
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
|
||||
0
|
||||
]
|
||||
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
|
||||
assert (
|
||||
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
|
||||
== 32
|
||||
)
|
||||
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
|
||||
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
|
||||
error(bad, 401, "unauthorized")
|
||||
error(absent, 401, "unauthorized")
|
||||
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
|
||||
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
|
||||
for value in (
|
||||
{},
|
||||
basic("alice"),
|
||||
{"X-User-Id": actors[0]["id"]},
|
||||
{"Authorization": "Bearer unknown"},
|
||||
):
|
||||
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
|
||||
actual = client.get(
|
||||
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
|
||||
).json()
|
||||
assert actual["id"] == actors[0]["id"]
|
||||
error(
|
||||
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
|
||||
def test_invalid_basic(value):
|
||||
with pytest.raises(Exception) as caught:
|
||||
basic_credentials(value)
|
||||
assert caught.value.status == 401
|
||||
|
||||
|
||||
def test_registration_race(client, settings):
|
||||
def register(_):
|
||||
return client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
|
||||
).status_code
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
|
||||
path = "/api/v1/chats/" + room["id"]
|
||||
for suffix in ("", "/messages"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
|
||||
error(
|
||||
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
|
||||
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
|
||||
error(
|
||||
client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "rollback", "member_ids": members},
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
|
||||
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
|
||||
for body in (
|
||||
{"text": ""},
|
||||
{"text": " \t"},
|
||||
{"text": "😀" * 4001},
|
||||
{"text": 123},
|
||||
{"text": None},
|
||||
{"text": "ok", "sender_id": actors[1]["id"]},
|
||||
):
|
||||
error(
|
||||
client.post(path + "/messages", headers=headers(actors[0]), json=body),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
for query in (
|
||||
"limit=0",
|
||||
"limit=101",
|
||||
"limit=true",
|
||||
"limit=1.0",
|
||||
"limit=1&limit=2",
|
||||
"cursor=",
|
||||
"cursor=bad!",
|
||||
"cursor=e30",
|
||||
"cursor=x&cursor=y",
|
||||
):
|
||||
error(
|
||||
client.get(path + "/messages?" + query, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
|
||||
def send(i):
|
||||
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
|
||||
assert r.status_code == 201
|
||||
return r.json()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
sent = list(pool.map(send, range(100)))
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
|
||||
)
|
||||
for m in sent:
|
||||
m["created_at"] = "2026-10-17T12:00:00.000000Z"
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
query = {"limit": 17 if cursor else 1}
|
||||
if cursor:
|
||||
query["cursor"] = cursor
|
||||
response = client.get(path, params=query, headers=headers(actors[1]))
|
||||
assert response.status_code == 200
|
||||
page = response.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
assert len(found) == len({x["id"] for x in found}) == 100
|
||||
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
|
||||
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
|
||||
error(
|
||||
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
|
||||
user = actors[0]
|
||||
url = "/api/v1/users/me"
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
repository = client.app.state.repository
|
||||
original = repository.get_user
|
||||
|
||||
def must_be_cached(*args):
|
||||
raise AssertionError("cache hit should avoid fetching immutable user again")
|
||||
|
||||
monkeypatch.setattr(repository, "get_user", must_be_cached)
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
monkeypatch.setattr(repository, "get_user", original)
|
||||
cache = client.app.state.cache.client
|
||||
digest = hashlib.sha256(user["token"].encode()).digest()
|
||||
key = client.app.state.cache.key(digest)
|
||||
assert 0 < cache.ttl(key) <= 3600
|
||||
stale = cache.get(key)
|
||||
cache.flushdb()
|
||||
assert client.get(url, headers=headers(user)).status_code == 200
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
|
||||
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
|
||||
error(client.get(url, headers=headers(user)), 401, "unauthorized")
|
||||
error(
|
||||
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
|
||||
)
|
||||
|
||||
|
||||
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
|
||||
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
|
||||
database_file.write_text(settings.database_url)
|
||||
cache_file.write_text(settings.cache_url)
|
||||
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
|
||||
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
|
||||
with TestClient(create_app()) as short:
|
||||
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
|
||||
assert response.status_code == 201
|
||||
token = {"Authorization": "Bearer " + response.json()["token"]}
|
||||
assert short.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
with TestClient(create_app(settings)) as restarted:
|
||||
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
|
||||
time.sleep(2.1)
|
||||
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
|
||||
migrate(settings.database_url, ROOT / "migrations")
|
||||
modified = tmp_path / "001_initial.sql"
|
||||
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
|
||||
with pytest.raises(RuntimeError, match="checksum changed"):
|
||||
migrate(settings.database_url, tmp_path)
|
||||
|
||||
|
||||
def test_logs_and_safe_errors(client, monkeypatch, caplog):
|
||||
canary = "PRIVATE_BODY_CANARY_123"
|
||||
with caplog.at_level("INFO", logger="messenger"):
|
||||
r = client.post(
|
||||
"/api/v1/users",
|
||||
json={"username": "logging", "display_name": canary, "password": PASSWORD},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert canary not in caplog.text and PASSWORD not in caplog.text
|
||||
|
||||
def broken(*args):
|
||||
raise RuntimeError(canary)
|
||||
|
||||
monkeypatch.setattr(client.app.state.authentication, "register", broken)
|
||||
response = client.post(
|
||||
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
|
||||
)
|
||||
error(response, 500, "internal_error")
|
||||
assert canary not in response.text and canary not in caplog.text
|
||||
error(
|
||||
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
@@ -0,0 +1,244 @@
|
||||
import hashlib
|
||||
import io
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import replace
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from conftest import ROOT
|
||||
from PIL import Image
|
||||
from psycopg.rows import dict_row
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.domain import DomainError
|
||||
from messenger.reconcile import Reconciler
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def upload(client, actor, room, content=PNG, media="image/png", name="sample.png"):
|
||||
return client.post(
|
||||
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
|
||||
headers={**headers(actor), "Content-Type": media},
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin(settings):
|
||||
store = ObjectStore(
|
||||
replace(
|
||||
settings,
|
||||
s3_access_key=(ROOT / "secrets/s3_root_user").read_text().strip(),
|
||||
s3_secret_key=(ROOT / "secrets/s3_root_password").read_text().strip(),
|
||||
)
|
||||
)
|
||||
yield store
|
||||
store.close()
|
||||
|
||||
|
||||
def age(conn, id):
|
||||
conn.execute(
|
||||
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
|
||||
(id,),
|
||||
)
|
||||
|
||||
|
||||
def test_original_png_jpeg_and_acl(client, actors, room):
|
||||
jpg = io.BytesIO()
|
||||
Image.new("RGB", (17, 11), "red").save(jpg, format="JPEG")
|
||||
for data, media in ((PNG, "image/png"), (jpg.getvalue(), "image/jpeg")):
|
||||
response = upload(client, actors[0], room, data, media, "Фото 1.png")
|
||||
assert response.status_code == 201, response.text
|
||||
value = response.json()
|
||||
assert value["status"] == "ready" and value["variants"] == []
|
||||
assert value["error_code"] is None and value["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert value["size_bytes"] == len(data) and value["content_type"] == media
|
||||
assert "object_key" not in value
|
||||
path = "/api/v1/attachments/" + value["id"]
|
||||
assert client.get(path, headers=headers(actors[1])).json() == value
|
||||
download = client.get(path + "/content", headers=headers(actors[1]))
|
||||
assert download.status_code == 200 and download.content == data
|
||||
assert download.headers["Content-Type"] == media
|
||||
for suffix in ("", "/content"):
|
||||
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
||||
error(client.get(path + suffix), 401, "unauthorized")
|
||||
error(
|
||||
client.get(path + "/content?variant=thumbnail", headers=headers(actors[0])),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
|
||||
def test_unsafe_names(client, actors, room, name):
|
||||
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data,media",
|
||||
[
|
||||
(b"", "image/png"),
|
||||
(b"not a file", "image/png"),
|
||||
(PNG[:30], "image/png"),
|
||||
(PNG, "image/jpeg"),
|
||||
(PNG, "text/plain"),
|
||||
],
|
||||
)
|
||||
def test_invalid_images(client, actors, room, data, media):
|
||||
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
|
||||
|
||||
|
||||
def test_byte_limit_without_content_length_and_pixel_bomb(client, actors, room):
|
||||
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
|
||||
stream = (b"x" * 1048576 for _ in range(11))
|
||||
error(upload(client, actors[0], room, stream), 413, "payload_too_large")
|
||||
bomb = bytearray(PNG)
|
||||
bomb[16:24] = struct.pack(">II", 10000, 4001)
|
||||
bomb[29:33] = struct.pack(">I", zlib.crc32(bomb[12:29]) & 0xFFFFFFFF)
|
||||
error(upload(client, actors[0], room, bytes(bomb)), 413, "payload_too_large")
|
||||
|
||||
|
||||
def test_attachment_message_relations_and_replay(client, actors, room):
|
||||
att = upload(client, actors[0], room).json()
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = headers(actors[1])
|
||||
body = {"text": "", "attachment_ids": [att["id"]]}
|
||||
result = client.post(path, headers=h, json=body)
|
||||
assert result.status_code == 201 and result.json()["attachment_ids"] == [att["id"]]
|
||||
assert client.post(path, headers=h, json=body).json() == result.json()
|
||||
assert client.get(path, headers=headers(actors[0])).json()["items"] == [result.json()]
|
||||
error(
|
||||
client.post(
|
||||
path,
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "", "attachment_ids": [att["id"], att["id"]]},
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
error(
|
||||
client.post(
|
||||
path, headers=headers(actors[0]), json={"text": "", "attachment_ids": [str(uuid4())]}
|
||||
),
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
other = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "other", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
error(
|
||||
client.post(f"/api/v1/chats/{other['id']}/messages", headers=headers(actors[0]), json=body),
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
legacy = headers(actors[0])
|
||||
one = client.post(path, headers=legacy, json={"text": "legacy"})
|
||||
two = client.post(path, headers=legacy, json={"attachment_ids": [], "text": "legacy"})
|
||||
assert one.status_code == 201 and two.json() == one.json()
|
||||
|
||||
|
||||
def test_scoped_credentials_cannot_delete_or_access_other_bucket(client, settings):
|
||||
store = ObjectStore(settings)
|
||||
try:
|
||||
with pytest.raises(ClientError) as denied:
|
||||
store.client.delete_object(Bucket=store.bucket, Key="originals/" + str(uuid4()))
|
||||
assert denied.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
with pytest.raises(ClientError) as denied:
|
||||
store.client.list_objects_v2(Bucket="unrelated-bucket")
|
||||
assert denied.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def test_db_failure_after_put_is_recoverable_with_dry_run(
|
||||
client, actors, room, settings, monkeypatch, admin
|
||||
):
|
||||
repo = client.app.state.repository
|
||||
|
||||
def fail(*args):
|
||||
raise RuntimeError("injected after durable PUT")
|
||||
|
||||
monkeypatch.setattr(repo, "finish_attachment", fail)
|
||||
error(upload(client, actors[0], room), 500, "internal_error")
|
||||
store = client.app.state.attachments.store
|
||||
with psycopg.connect(settings.database_url, autocommit=True, row_factory=dict_row) as conn:
|
||||
row = conn.execute("SELECT * FROM attachments").fetchone()
|
||||
assert row["storage_state"] == "pending" and store.head(row["object_key"]) is not None
|
||||
assert Reconciler(conn, admin, apply=True).run()["counts"].get("restore_ready", 0) == 0
|
||||
age(conn, row["id"])
|
||||
dry = Reconciler(conn, admin).run()
|
||||
assert dry["counts"]["restore_ready"] == 1
|
||||
assert (
|
||||
conn.execute("SELECT storage_state FROM attachments").fetchone()["storage_state"]
|
||||
== "pending"
|
||||
)
|
||||
assert Reconciler(conn, admin, apply=True).run()["counts"]["restore_ready"] == 1
|
||||
assert Reconciler(conn, admin, apply=True).run()["actions"] == []
|
||||
row = conn.execute("SELECT * FROM attachments").fetchone()
|
||||
assert row["storage_state"] == "ready"
|
||||
assert (
|
||||
client.get(
|
||||
"/api/v1/attachments/" + str(row["id"]) + "/content", headers=headers(actors[1])
|
||||
).content
|
||||
== PNG
|
||||
)
|
||||
|
||||
|
||||
def test_failed_put_and_missing_used_object(client, actors, room, settings, admin, monkeypatch):
|
||||
store = client.app.state.attachments.store
|
||||
original = store.put
|
||||
|
||||
def fail(*args):
|
||||
raise DomainError(503, "unavailable", "injected transport failure")
|
||||
|
||||
monkeypatch.setattr(store, "put", fail)
|
||||
error(upload(client, actors[0], room), 503, "unavailable")
|
||||
with psycopg.connect(settings.database_url, autocommit=True, row_factory=dict_row) as conn:
|
||||
pending = conn.execute("SELECT id FROM attachments").fetchone()["id"]
|
||||
age(conn, pending)
|
||||
assert Reconciler(conn, admin, apply=True).run()["counts"]["delete_abandoned_intent"] == 1
|
||||
monkeypatch.setattr(store, "put", original)
|
||||
att = upload(client, actors[0], room).json()
|
||||
response = client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages",
|
||||
headers=headers(actors[0]),
|
||||
json={"text": "used", "attachment_ids": [att["id"]]},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (att["id"],)).fetchone()
|
||||
age(conn, row["id"])
|
||||
admin.client.delete_object(Bucket=store.bucket, Key=row["object_key"])
|
||||
assert Reconciler(conn, admin).run()["counts"]["mark_missing"] == 1
|
||||
assert (
|
||||
conn.execute("SELECT storage_state FROM attachments").fetchone()["storage_state"]
|
||||
== "ready"
|
||||
)
|
||||
assert Reconciler(conn, admin, apply=True).run()["counts"]["mark_missing"] == 1
|
||||
error(
|
||||
client.get("/api/v1/attachments/" + att["id"], headers=headers(actors[0])),
|
||||
409,
|
||||
"conflict",
|
||||
)
|
||||
assert conn.execute("SELECT count(*) AS n FROM message_attachments").fetchone()["n"] == 1
|
||||
admin.put(
|
||||
row["object_key"],
|
||||
io.BytesIO(PNG),
|
||||
len(PNG),
|
||||
"image/png",
|
||||
hashlib.sha256(PNG).hexdigest(),
|
||||
)
|
||||
assert Reconciler(conn, admin, apply=True).run()["counts"]["restore_ready"] == 1
|
||||
assert (
|
||||
client.get(
|
||||
"/api/v1/attachments/" + att["id"] + "/content", headers=headers(actors[0])
|
||||
).content
|
||||
== PNG
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from test_api import error, headers
|
||||
|
||||
from messenger.app import create_app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
|
||||
def test_message_key_required(client, actors, room, key):
|
||||
h = {"Authorization": "Bearer " + actors[0]["token"]}
|
||||
if key is not None:
|
||||
h["Idempotency-Key"] = key
|
||||
error(
|
||||
client.post(
|
||||
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
key = "Scope.Key:123"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": key}
|
||||
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
|
||||
assert first.status_code == 201
|
||||
repeat = client.post(
|
||||
path,
|
||||
headers={**h, "Content-Type": "application/json"},
|
||||
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
|
||||
)
|
||||
assert repeat.status_code == 201 and repeat.json() == first.json()
|
||||
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
|
||||
outsider = {**headers(actors[2]), "Idempotency-Key": key}
|
||||
error(
|
||||
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
|
||||
)
|
||||
other_user = {**headers(actors[1]), "Idempotency-Key": key}
|
||||
own = client.post(path, headers=other_user, json={"text": "another author"})
|
||||
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
|
||||
chat = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers(actors[0]),
|
||||
json={"title": "another", "member_ids": [actors[0]["id"]]},
|
||||
).json()
|
||||
other_chat = client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
|
||||
)
|
||||
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
|
||||
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
|
||||
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
with TestClient(create_app(settings)) as second:
|
||||
assert second.app.state.instance_id != client.app.state.instance_id
|
||||
|
||||
def send(i):
|
||||
c = client if i % 2 else second
|
||||
r = c.post(path, headers=h, json={"text": "same concurrent body"})
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(send, range(30)))
|
||||
assert len({item[0]["id"] for item in results}) == 1
|
||||
assert len({item[1] for item in results}) == 2
|
||||
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
|
||||
|
||||
|
||||
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
|
||||
repository = client.app.state.repository
|
||||
original = repository._save_request
|
||||
|
||||
def fail_after_message_insert(*args):
|
||||
raise RuntimeError("injected between message and result")
|
||||
|
||||
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
error(
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
|
||||
)
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
|
||||
monkeypatch.setattr(repository, "_save_request", original)
|
||||
result = client.post(path, headers=h, json={"text": "retry after rollback"})
|
||||
assert result.status_code == 201
|
||||
assert (
|
||||
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
|
||||
)
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
|
||||
path = f"/api/v1/chats/{room['id']}/messages"
|
||||
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
|
||||
first = client.post(path, headers=h, json={"text": "old"}).json()
|
||||
with psycopg.connect(settings.database_url) as conn:
|
||||
conn.execute(
|
||||
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
|
||||
"expires_at=clock_timestamp()-interval '1 day'"
|
||||
)
|
||||
second = client.post(path, headers=h, json={"text": "new"})
|
||||
assert second.status_code == 201 and second.json()["id"] != first["id"]
|
||||
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("optimized", [True, False])
|
||||
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
|
||||
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
|
||||
c.get("/api/v1/users/me", headers=headers(actors[0]))
|
||||
pool = c.app.state.repository.pool
|
||||
before = pool.get_stats().get("requests_num", 0)
|
||||
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
|
||||
assert response.status_code == 200 and response.json()["items"] == []
|
||||
assert pool.get_stats()["requests_num"] - before == 3
|
||||
|
||||
|
||||
def test_instance_header_covers_health_errors_and_success(client):
|
||||
values = [
|
||||
client.get("/health/live"),
|
||||
client.get("/health/ready"),
|
||||
client.get("/api/v1/users/me"),
|
||||
client.get("/not-a-route"),
|
||||
]
|
||||
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
|
||||
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
|
||||
Reference in New Issue
Block a user