ЛР 4 — БПИ2304 — Кондрашов — на 5 #156
+5
-15
@@ -1,20 +1,10 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
DB_POOL_MAX_SIZE=8
|
||||
API_CPUS=1.00
|
||||
API_MEMORY=512M
|
||||
UPLOAD_CONCURRENCY=1
|
||||
LB_PORT=8080
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
FROM python:3.12.8-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
WORKDIR /srv/messenger
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
COPY lab.json .
|
||||
COPY contracts ./contracts
|
||||
COPY app ./app
|
||||
COPY scripts ./scripts
|
||||
COPY tests ./tests
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-keep-alive", "5", "--timeout-graceful-shutdown", "10"]
|
||||
@@ -4,10 +4,15 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help init-secrets check validate test test-grade4 test-s3-failure test-reconcile test-upload-load load test-persistence test-failover test-failover-target
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "init-secrets: create local secrets; check: offline structure; test: public smoke; test-grade4: negatives/concurrency"
|
||||
@echo "test-s3-failure: stops/restarts local S3; test-reconcile: reconciliation fault injection"
|
||||
@echo "test-upload-load: >=10 concurrent uploads; load: synthetic writes"
|
||||
|
||||
init-secrets:
|
||||
sh scripts/init-secrets.sh
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,6 +23,18 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-grade4:
|
||||
$(PYTHON) tests/grade4.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-s3-failure:
|
||||
$(PYTHON) tests/s3_failure.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile:
|
||||
@docker compose run --rm -T --no-deps api python tests/reconcile_integration.py
|
||||
|
||||
test-upload-load:
|
||||
$(PYTHON) tests/upload_load.py --base-url "$(BASE_URL)" --concurrency 10 $(if $(OUTPUT),--output "$(OUTPUT)",)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -28,3 +45,8 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover-target:
|
||||
@test -n "$(TARGET_INSTANCE_ID)" || (echo "Set TARGET_INSTANCE_ID to an observed X-Instance-Id"; exit 2)
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/replica_failover.py --target-id "$(TARGET_INSTANCE_ID)" --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) $(if $(OUTPUT),--output "$(OUTPUT)",)
|
||||
|
||||
@@ -46,25 +46,39 @@
|
||||
|
||||
## Как начать и проверить
|
||||
|
||||
```sh
|
||||
# Работает прямо в skeleton, Python 3.9+; приложение не запускается:
|
||||
make check
|
||||
|
||||
# После реализации запустите свой сервер/стенд по REPORT.md.
|
||||
# По умолчанию тест использует http://localhost:8080.
|
||||
make test
|
||||
```
|
||||
|
||||
`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость.
|
||||
Для этого решения используется Compose, PostgreSQL, Redis, два API, HAProxy и RustFS. В начале требуются Docker Compose v2 и Python 3.9+.
|
||||
|
||||
```sh
|
||||
# Compose-вариант после реализации:
|
||||
cp .env.example .env
|
||||
make init-secrets
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Дождитесь /health/ready; затем make test.
|
||||
# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md.
|
||||
docker compose --env-file .env build api migrate
|
||||
docker compose --env-file .env up -d postgres redis s3
|
||||
docker compose --env-file .env run --rm --no-deps s3-bootstrap
|
||||
docker compose --env-file .env run --rm migrate
|
||||
docker compose --env-file .env up -d --scale api=2 api load_balancer
|
||||
|
||||
until curl -fsS http://localhost:8080/health/ready; do sleep 1; done
|
||||
make check
|
||||
make test
|
||||
make test-grade4
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
```
|
||||
|
||||
`.env.example` содержит только параметры без секретов; `make init-secrets` создаёт локальные файлы с правами `0600`. RustFS console и S3-порт не публикуются наружу. `s3-bootstrap` идемпотентно создаёт приватный bucket, scoped policy и отдельного пользователя API. Образ RustFS закреплён на версии 1.0.6.
|
||||
|
||||
Тесты на восстановление и отказ:
|
||||
|
||||
```sh
|
||||
make test-s3-failure
|
||||
make test-reconcile
|
||||
make test-upload-load OUTPUT=evidence/upload-load.json
|
||||
```
|
||||
|
||||
`test-reconcile` — fault-injection в отдельном учебном стенде: он создаёт синтетические записи и специально запускает `--apply --grace-hours 0`, чтобы проверить исправления; не запускайте его на полезных данных. Сам reconciler по умолчанию только dry-run и использует grace period 24 часа.
|
||||
|
||||
`docker compose down` останавливает стенд, сохраняя volumes. Только намеренная команда `docker compose down -v` удаляет локальные volumes и учебные данные. Для полной OpenAPI-валидации установите `requirements-dev.txt` в venv и запустите `make validate`.
|
||||
|
||||
|
||||
Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически):
|
||||
|
||||
@@ -76,6 +90,8 @@ make validate PYTHON=.venv/bin/python
|
||||
|
||||
Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте.
|
||||
|
||||
Для оценки 5 выбрана ветка исследования согласованности: `python scripts/reconcile.py` сначала выводит план; для применения нужен явный `--apply`. Объекты без метаданных и записи без объекта сверяются по уникальному server-generated key. Свежие orphans не удаляются, использованные вложения никогда не чистятся автоматически; missing object у используемого вложения остаётся отмечен в отчёте для ручного восстановления из резервной копии. Fault-injection в `tests/reconcile_integration.py` проверяет обнаружение, исправление, dry-run, grace period, повторный запуск и сохранность используемого вложения.
|
||||
|
||||
## Сценарий защиты
|
||||
|
||||
1. Загрузить fixture, отправить message с attachment_ids, скачать как второй участник и сравнить SHA-256.
|
||||
|
||||
@@ -1,37 +1,100 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: не указаны в исходных материалах; заполнить перед сдачей.
|
||||
- Целевая оценка: 5.
|
||||
- Выбранный трек: 2 — согласованность PostgreSQL/S3.
|
||||
- Commit/PR: не создавались по просьбе автора.
|
||||
- Основа: предоставленный skeleton лабораторной 4 и перенесённая реализация лабораторной 3.
|
||||
- Библиотеки: FastAPI, psycopg, Redis, boto3, Pillow, argon2-cffi; версии заданы в `requirements.txt`. Помощь при реализации и проверках: Copilot.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Compose v2 и Python 3.9+ на хосте. Секреты генерируются локально и не добавляются в репозиторий.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
make init-secrets
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env build api migrate
|
||||
docker compose --env-file .env up -d postgres redis s3
|
||||
docker compose --env-file .env run --rm --no-deps s3-bootstrap
|
||||
docker compose --env-file .env run --rm migrate
|
||||
docker compose --env-file .env up -d --scale api=2 api load_balancer
|
||||
|
||||
until curl -fsS http://localhost:8080/health/ready; do sleep 1; done
|
||||
make check
|
||||
make test
|
||||
make test-grade4
|
||||
make test-s3-failure
|
||||
make test-upload-load OUTPUT=evidence/upload-load.json
|
||||
make test-reconcile
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
```
|
||||
|
||||
Для точечного failover сначала укажите один из наблюдаемых `X-Instance-Id`:
|
||||
|
||||
```sh
|
||||
make test-failover-target TARGET_INSTANCE_ID=<observed-id> \
|
||||
ACTION_SCRIPT=scripts/stop-one.sh OUTPUT=evidence/failover-target.json
|
||||
```
|
||||
|
||||
Обычная остановка сохраняет учебные volumes: `docker compose down`. Только намеренное удаление данных: `docker compose down -v`. Скрипт `scripts/restart.sh` перезапускает PostgreSQL, Redis, S3 и по очереди здоровые API-реплики; `scripts/stop-one.sh` останавливает ровно выбранную API-реплику и оставляет её остановленной.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → HAProxy → API × 2+
|
||||
├─ PostgreSQL: пользователи, чаты, сообщения, сессии, метаданные вложений
|
||||
├─ Redis: общие сессии
|
||||
└─ приватный RustFS bucket: байты вложений
|
||||
```
|
||||
|
||||
Проверенная среда: macOS 27 arm64; Docker client/server 29.1.3, Docker сообщает 12 CPU и около 7.65 GiB памяти. Образы закреплены в `compose.yaml`: Python 3.12.8 для API, PostgreSQL 16.6, Redis 7.4.2, HAProxy 3.1 и RustFS 1.0.6. Лимиты Compose: API 1 CPU/512 MiB на реплику, PostgreSQL 0.5 CPU/384 MiB, Redis 0.25 CPU/128 MiB, S3 0.75 CPU/512 MiB. Это локальный учебный стенд, не production-конфигурация.
|
||||
|
||||
PostgreSQL является источником истины для прав и метаданных; объектный ключ формируется сервером как `attachments/<UUID>`, а исходное имя хранится только для отображения. Составные внешние ключи в `message_attachments` не позволяют связать файл и сообщение из разных чатов. Bucket приватный; root credentials используются только bootstrap-сервисом, API получает отдельного пользователя с правами на bucket и `attachments/*`. Консоль и S3-порт не опубликованы. Bootstrap запускался повторно без ошибки.
|
||||
|
||||
Загрузка проверяет членство пользователя в чате, имя и заявленный MIME, читает тело потоково с пределом 10 MiB и проверяет фактический PNG/JPEG через Pillow; лимит изображения — 40 млн пикселей. `SpooledTemporaryFile` хранит первые 1 MiB в памяти и переносит превышение во временный файл. Семафор ограничивает одновременные загрузки каждой API-реплики. Сначала выполняется S3 PUT, затем транзакция PostgreSQL; `201 ready` возвращается только после обоих шагов. Ошибка S3 даёт `503`; ошибка вставки метаданных запускает компенсирующий S3 delete. Если компенсация тоже не удалась, возможный orphan удаляется reconciler после grace period.
|
||||
|
||||
Скачивание сначала проверяет авторизацию по метаданным, затем читает объект из S3 и сравнивает размер и SHA-256 с PostgreSQL. Приватный bucket не заменяет проверку доступа приложения: участники другого чата получают `404`.
|
||||
|
||||
Reconciler по умолчанию только строит план. Он сканирует ключи с префиксом `attachments/`, сравнивает их с метаданными, а для удаления требует возраст не меньше 24 часов. В apply-режиме удаляются только старые orphan-объекты и старые отсутствующие metadata без ссылок в сообщениях. Отсутствующий объект у используемого вложения не удаляется автоматически: несоответствие сохраняется для ручного восстановления. Повторный запуск безопасен и идемпотентен для уже исправленного состояния.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Контракт и структура | `make check` | PASS; проверены структура, контракт и обязательные файлы | stdout при локальном запуске |
|
||||
| Сквозной API, сохранение lab3 | `make test` | PASS; пользователи, сессии, чат, сообщения, права, пагинация и вложение | stdout при локальном запуске |
|
||||
| Негативные проверки уровня 4 | `make test-grade4` | PASS; размер, повреждённый файл, MIME, пиксельный лимит, имя, cross-chat, чужое вложение и конкурентная идемпотентность | stdout при локальном запуске |
|
||||
| S3 outage и восстановление | `make test-s3-failure` | PASS; upload вернул 503, после запуска S3 bucket и API восстановились | `evidence/s3-failure.txt` |
|
||||
| Bootstrap/private bucket | повторный запуск `s3-bootstrap`; anonymous S3 GET в `make test-reconcile` | Два последовательных bootstrap завершились успешно; анонимное чтение запрещено | `evidence/bootstrap.txt`, `evidence/reconcile-integration.json` |
|
||||
| Потоковая загрузка и нагрузка уровня 4 | `make test-upload-load OUTPUT=evidence/upload-load.json` | 10/10 загрузок по 4,322,578 байт, concurrency 10, время 3.05 s. Пиковая память API: 81,128,325 и 82,732,646 байт (семплирование раз в 0.5 s) | `evidence/upload-load.json` |
|
||||
| Persistence/restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; подтверждённое сообщение, старый токен и оригинал вложения сохранились после перезапуска PostgreSQL, Redis, S3 и здоровых API | `evidence/persistence.txt` |
|
||||
| Failover уровня 4 | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS; две реплики наблюдались до отказа, 113 успешных записей, 1 переходная ошибка, восстановление записи за 2.558 s, данные и сессия сохранены | `evidence/failover.txt` |
|
||||
| Целевой failover | `make test-failover-target ... OUTPUT=evidence/failover-target.json` | До отказа наблюдались 2 реплики; после остановки target оставалась другая; 109 успешных записей, 1 ошибка записи, 0 ошибок чтения, восстановление за 2.454 s, дубликатов нет | `evidence/failover-target.json` |
|
||||
| Трек 2: обнаружение и dry-run | `make test-reconcile` | Сканировано 38 строк metadata и 33 объекта за 0.081 s; dry-run ничего не удалил; свежий orphan сохранён; anonymous read запрещён | `evidence/reconcile-integration.json` |
|
||||
| Трек 2: исправление и повтор | `make test-reconcile` | На синтетических расхождениях apply удалил 1 старый orphan и 1 старую неиспользуемую запись metadata; используемое вложение и referenced missing metadata сохранены; повтор удалил 0 объектов и 0 metadata | `evidence/reconcile-integration.json` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотеза:** ограничение размера, spooling и локальный семафор позволят обслужить не менее десяти параллельных загрузок без пропорционального удержания всех тел в RAM; reconciler с grace period не удалит свежие или связанные данные и повторный apply не внесёт повторных изменений.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Методика загрузки:** синтетический PNG 1200×1200 размером 4,322,578 байт загружался десятью потоками через HAProxy. Во время сценария `docker stats` снимал память двух здоровых API-реплик каждые 0.5 s. Все десять запросов завершились успешно за 3.05 s; наблюдаемые пики составили 81,128,325 и 82,732,646 байт. Это абсолютные sampled peaks, не прирост относительно baseline. Один запуск и короткий профиль не дают статистической оценки; частота семплирования может пропустить кратковременный пик. Измерение исключает память PostgreSQL, S3, HAProxy и клиента.
|
||||
|
||||
**Методика сверки:** интеграционный тест создал синтетические записи и удалил объекты из S3, чтобы получить два направления несогласованности; затем создал orphan. На измеренном запуске при 38 строках metadata и 33 объектах dry-run занял 0.081 s, не удалив данные. Apply с обычным grace period также ничего не удалил, потому что объект был свежим. В изолированном тесте с grace period 0 удалены один старый orphan и одна старая неиспользуемая строка metadata; missing object у связанного сообщения и существующее используемое вложение сохранены. Повторный apply удалил ноль объектов и строк. На таком малом наборе 0.081 s — только локальная оценка, а не прогноз для production. Реализация загружает список объектов и metadata в память, поэтому для больших bucket нужны измерения и обработка порциями.
|
||||
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление/данные |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановить S3 во время upload | Явный 503, без ложного `ready` | PASS; после возврата S3 сервис снова стал ready | Автоматическое восстановление, persistent volumes сохранены |
|
||||
| Перезапуск PostgreSQL, Redis, S3 и API | Сохранить сообщение, сессию и оригинал файла | PASS со старым токеном; hook сохранил volumes | Здоровые API перезапущены по очереди |
|
||||
| Остановить одну API-реплику | LB продолжает обслуживать запросы другой; retry не дублирует запись | 113 успешных writes и 1 переходная ошибка в общем сценарии; targeted test сохранил 109 writes при 1 ошибке | В targeted test восстановление за 2.454 s; остановленная реплика оставлена остановленной, затем восстановлена вручную |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Измерения выполнены один раз на локальном Docker Desktop; для сравнения производительности нужны прогрев, несколько повторов, baseline/delta RAM и более длительная нагрузка.
|
||||
- Reconciler выполняет полный проход по bucket и собирает списки в память. На больших объёмах потребуются порционная сверка, метрики, контроль времени/нагрузки и отдельное планирование запуска.
|
||||
- Не автоматизируется восстановление потерянного байтового объекта, если к нему привязано сообщение; требуется backup/ручное восстановление.
|
||||
- S3, PostgreSQL, Redis и HAProxy в Compose представлены одиночными экземплярами. Тестируется отказ API-реплики, но не отказ host, балансировщика, БД или самого S3-сервера.
|
||||
- Upload синхронный и временные файлы используют локальный диск контейнера после 1 MiB; полноценные quotas, отдельный scratch volume и distributed concurrency limits не реализованы.
|
||||
- Автор и группа не содержались в переданных материалах; их нужно вписать перед сдачей.
|
||||
|
||||
+1030
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,35 @@
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.settings import database_connection
|
||||
|
||||
|
||||
def main() -> None:
|
||||
migration_dir = Path(__file__).with_name("migrations")
|
||||
with psycopg.connect(**database_connection(), autocommit=False) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version varchar(255) PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
"""
|
||||
)
|
||||
for migration in sorted(migration_dir.glob("*.sql")):
|
||||
already_applied = connection.execute(
|
||||
"SELECT 1 FROM schema_migrations WHERE version = %s",
|
||||
(migration.name,),
|
||||
).fetchone()
|
||||
if already_applied:
|
||||
continue
|
||||
connection.execute(migration.read_text(encoding="utf-8"))
|
||||
connection.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES (%s)",
|
||||
(migration.name,),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_nonblank CHECK (length(btrim(display_name)) > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT chats_title_nonblank CHECK (length(btrim(title)) > 0)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_by_user
|
||||
ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_nonblank CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_history_keyset
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sessions_active_by_expiry
|
||||
ON sessions (expires_at)
|
||||
WHERE revoked_at IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version varchar(255) PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid REFERENCES messages(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idempotency_keys_expiry
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key varchar(160) NOT NULL UNIQUE,
|
||||
status varchar(16) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT attachments_filename_nonblank CHECK (length(filename) > 0),
|
||||
UNIQUE (id, chat_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_by_chat
|
||||
ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS messages_id_chat_unique
|
||||
ON messages (id, chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, ordinal),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages(id, chat_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments(id, chat_id) ON DELETE RESTRICT
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT IF EXISTS messages_text_nonblank;
|
||||
@@ -0,0 +1,54 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
def read_setting(
|
||||
name: str,
|
||||
file_name: str,
|
||||
default: Optional[str] = None,
|
||||
default_file: Optional[str] = None,
|
||||
) -> str:
|
||||
configured_secret_path = os.environ.get(file_name)
|
||||
secret_path = configured_secret_path or default_file
|
||||
if secret_path and (configured_secret_path or Path(secret_path).is_file()):
|
||||
try:
|
||||
return Path(secret_path).read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise RuntimeError("Unable to read configured secret file") from exc
|
||||
value = os.environ.get(name, default)
|
||||
if value is None:
|
||||
if default_file:
|
||||
raise RuntimeError(
|
||||
"Required setting is missing: "
|
||||
+ name
|
||||
+ "; set "
|
||||
+ file_name
|
||||
+ " or mount the secret at "
|
||||
+ default_file
|
||||
)
|
||||
raise RuntimeError("Required application setting is missing: " + name)
|
||||
return value
|
||||
|
||||
|
||||
def database_connection() -> Dict[str, object]:
|
||||
return {
|
||||
"host": os.environ.get("DB_HOST", "localhost"),
|
||||
"port": int(os.environ.get("DB_PORT", "5432")),
|
||||
"dbname": os.environ.get("DB_NAME", "messenger"),
|
||||
"user": os.environ.get("DB_USER", "messenger"),
|
||||
"password": read_setting(
|
||||
"DB_PASSWORD",
|
||||
"DB_PASSWORD_FILE",
|
||||
default_file="/run/secrets/database_password",
|
||||
),
|
||||
"connect_timeout": int(os.environ.get("DB_CONNECT_TIMEOUT_SECONDS", "3")),
|
||||
"options": "-c statement_timeout=5000 -c lock_timeout=3000",
|
||||
}
|
||||
|
||||
|
||||
def session_ttl_seconds() -> int:
|
||||
value = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
if value < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be at least 1")
|
||||
return value
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/database_password
|
||||
secrets:
|
||||
- database_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.50
|
||||
mem_limit: 384m
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- backend
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- backend
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: messenger
|
||||
DB_USER: messenger
|
||||
DB_PASSWORD_FILE: /run/secrets/database_password
|
||||
secrets:
|
||||
- database_password
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
cpus: 0.50
|
||||
mem_limit: 256m
|
||||
networks:
|
||||
- backend
|
||||
profiles:
|
||||
- tools
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: messenger
|
||||
DB_USER: messenger
|
||||
DB_PASSWORD_FILE: /run/secrets/database_password
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-1}
|
||||
secrets:
|
||||
- database_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 8s
|
||||
stop_grace_period: 12s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
endpoint_mode: dnsrr
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-1.00}"
|
||||
memory: ${API_MEMORY:-512M}
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- backend
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.1-alpine
|
||||
ports:
|
||||
- "${LB_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- frontend
|
||||
- backend
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.6
|
||||
environment:
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ADDRESS: 0.0.0.0:9000
|
||||
RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_REGION: us-east-1
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
export RUSTFS_ACCESS_KEY="$$(cat /run/secrets/s3_root_access_key)"
|
||||
export RUSTFS_SECRET_KEY="$$(cat /run/secrets/s3_root_secret_key)"
|
||||
exec /entrypoint.sh
|
||||
secrets:
|
||||
- s3_root_access_key
|
||||
- s3_root_secret_key
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- backend
|
||||
cpus: 0.75
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 8s
|
||||
depends_on:
|
||||
s3-volume-permissions:
|
||||
condition: service_completed_successfully
|
||||
|
||||
s3-volume-permissions:
|
||||
image: alpine:3.21.3
|
||||
user: "0:0"
|
||||
command: ["sh", "-c", "chown -R 10001:10001 /data"]
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
restart: "no"
|
||||
networks:
|
||||
- backend
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python", "scripts/bootstrap-s3.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key
|
||||
S3_ROOT_SECRET_KEY_FILE: /run/secrets/s3_root_secret_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- s3_root_access_key
|
||||
- s3_root_secret_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks:
|
||||
- backend
|
||||
|
||||
secrets:
|
||||
database_password:
|
||||
file: ./secrets/database_password
|
||||
s3_root_access_key:
|
||||
file: ./secrets/s3_root_access_key
|
||||
s3_root_secret_key:
|
||||
file: ./secrets/s3_root_secret_key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
@@ -0,0 +1,2 @@
|
||||
S3 bootstrap complete: private bucket and scoped API credentials are ready.
|
||||
S3 bootstrap complete: private bucket and scoped API credentials are ready.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"target_instance_id": "e162139af177",
|
||||
"observed_instances_before": 2,
|
||||
"instances_serving_after": [
|
||||
"ad823d83c069"
|
||||
],
|
||||
"duration_seconds": 15.121,
|
||||
"successful_writes": 109,
|
||||
"failed_write_attempts": 1,
|
||||
"failed_read_attempts": 0,
|
||||
"successful_rps": 7.209,
|
||||
"write_error_ratio": 0.009091,
|
||||
"recovery_seconds": 2.454,
|
||||
"confirmed_data_preserved": true,
|
||||
"duplicate_messages": 0
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 1, "first_recovery_seconds": 2.558}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1 @@
|
||||
{"apply_deleted_objects": 1, "apply_deleted_unreferenced_metadata": 1, "dry_run_elapsed_seconds": 0.081, "dry_run_orphan_candidates": 0, "fresh_orphan_preserved": 1, "metadata_scanned": 38, "objects_scanned": 33, "private_bucket_anonymous_read": "denied", "referenced_missing_metadata_preserved": true, "repeat_deleted_metadata": 0, "repeat_deleted_objects": 0, "used_attachment_preserved": true}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: upload returned 503 during S3 outage; bucket and API recovered with volumes intact.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"api_container_ids": [
|
||||
"21c94e568619d0d4b8048e2ed7adc7a327f399dca08df3e1c8cd561d5a28e78f",
|
||||
"d9c73db60b17c0e4fb7d24145ad52955f077c3a021028413d85c3591b95ee97c"
|
||||
],
|
||||
"api_container_peak_memory_bytes": {
|
||||
"21c94e568619d0d4b8048e2ed7adc7a327f399dca08df3e1c8cd561d5a28e78f": 83078676,
|
||||
"d9c73db60b17c0e4fb7d24145ad52955f077c3a021028413d85c3591b95ee97c": 83760250
|
||||
},
|
||||
"concurrency": 10,
|
||||
"elapsed_seconds": 3.045,
|
||||
"notes": "Peak sampled every 0.5s using docker stats; excludes database, LB, and client memory.",
|
||||
"upload_bytes_each": 4322578,
|
||||
"uploads_succeeded": 10
|
||||
}
|
||||
+11
-4
@@ -1,7 +1,14 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Compose описан в корневом `compose.yaml`. Роли:
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
| Роль | Service | Версия | Сеть и сохранность |
|
||||
| --- | --- | --- | --- |
|
||||
| Входной балансировщик | `load_balancer` | HAProxy 3.1-alpine | Публикует только порт 8080 |
|
||||
| API | `api` × 2 | Python 3.12.8 image | Только внутренняя сеть; stateless |
|
||||
| Метаданные/сессии | `postgres`, `redis` | PostgreSQL 16.6-alpine, Redis 7.4.2-alpine | Внутренняя сеть; persistent volumes |
|
||||
| Объекты | `s3` | RustFS 1.0.6 | Внутренняя сеть; persistent volume |
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
`scripts/init-secrets.sh` идемпотентно создаёт секреты с правами `0600`; `scripts/bootstrap-s3.py` создаёт bucket, API-учётную запись и bucket-scoped policy через подписанный RustFS admin API. Консоль RustFS выключена, внутренние порты не публикуются. `s3-bootstrap` можно повторить после рестарта без удаления данных.
|
||||
|
||||
Миграции выполняются единожды `docker compose run --rm migrate`, до старта API. Restart hook сохраняет volumes; failover hook останавливает только выбранную API-реплику.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
timeout queue 5s
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
hold valid 2s
|
||||
hold obsolete 2s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger_api
|
||||
|
||||
backend messenger_api
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1
|
||||
server-template api 1-8 api:8080 check resolvers docker init-addr libc,none
|
||||
@@ -0,0 +1,7 @@
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.36.26
|
||||
fastapi==0.115.6
|
||||
Pillow==11.1.0
|
||||
psycopg[binary,pool]==3.2.4
|
||||
redis==5.2.1
|
||||
uvicorn[standard]==0.34.0
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotently create the private bucket and restricted RustFS API user."""
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import boto3
|
||||
import botocore
|
||||
from botocore.auth import SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.credentials import Credentials
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def signed_admin_request(root_access, root_secret, method, path, body=None):
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
content = json.dumps(body, separators=(",", ":")).encode("utf-8") if body else b""
|
||||
url = endpoint.rstrip("/") + path
|
||||
headers = {
|
||||
"Host": url.split("/", 3)[2],
|
||||
"Content-Type": "application/json",
|
||||
"X-Amz-Content-SHA256": hashlib.sha256(content).hexdigest(),
|
||||
}
|
||||
request = AWSRequest(method=method, url=url, data=content, headers=headers)
|
||||
SigV4Auth(Credentials(root_access, root_secret), "s3", region).add_auth(request)
|
||||
http_request = Request(
|
||||
url,
|
||||
data=content if content else None,
|
||||
headers=dict(request.headers),
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urlopen(http_request, timeout=5) as response:
|
||||
response.read()
|
||||
except HTTPError as error:
|
||||
detail = error.read(2048).decode("utf-8", "replace")
|
||||
if error.code == 409 and "add-user" in path:
|
||||
return
|
||||
raise RuntimeError(
|
||||
"RustFS IAM bootstrap request failed: "
|
||||
+ method
|
||||
+ " "
|
||||
+ path.split("?", 1)[0]
|
||||
+ " returned "
|
||||
+ str(error.code)
|
||||
+ ": "
|
||||
+ detail[:300]
|
||||
) from None
|
||||
except URLError as error:
|
||||
raise RuntimeError("RustFS IAM bootstrap service is unavailable") from error
|
||||
|
||||
|
||||
def main():
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
root_access = secret("S3_ROOT_ACCESS_KEY")
|
||||
root_secret = secret("S3_ROOT_SECRET_KEY")
|
||||
app_access = secret("S3_ACCESS_KEY_ID")
|
||||
app_secret = secret("S3_SECRET_ACCESS_KEY")
|
||||
config = Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
)
|
||||
root = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=root_access,
|
||||
aws_secret_access_key=root_secret,
|
||||
config=config,
|
||||
)
|
||||
for attempt in range(30):
|
||||
try:
|
||||
try:
|
||||
root.create_bucket(Bucket=bucket)
|
||||
except ClientError as error:
|
||||
if error.response.get("Error", {}).get("Code") != "BucketAlreadyOwnedByYou":
|
||||
raise
|
||||
root.head_bucket(Bucket=bucket)
|
||||
break
|
||||
except (botocore.exceptions.BotoCoreError, ClientError):
|
||||
if attempt == 29:
|
||||
raise
|
||||
time.sleep(2)
|
||||
|
||||
policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::" + bucket],
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::" + bucket + "/attachments/*"],
|
||||
},
|
||||
],
|
||||
}
|
||||
signed_admin_request(
|
||||
root_access,
|
||||
root_secret,
|
||||
"PUT",
|
||||
"/rustfs/admin/v3/add-user?" + urlencode({"accessKey": app_access}),
|
||||
{"secretKey": app_secret, "status": "enabled"},
|
||||
)
|
||||
signed_admin_request(
|
||||
root_access,
|
||||
root_secret,
|
||||
"PUT",
|
||||
"/rustfs/admin/v3/add-canned-policy?" + urlencode({"name": "messenger-api"}),
|
||||
policy,
|
||||
)
|
||||
signed_admin_request(
|
||||
root_access,
|
||||
root_secret,
|
||||
"POST",
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
{"policies": ["messenger-api"], "user": app_access},
|
||||
)
|
||||
|
||||
api_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=app_access,
|
||||
aws_secret_access_key=app_secret,
|
||||
config=config,
|
||||
)
|
||||
api_client.head_bucket(Bucket=bucket)
|
||||
root.close()
|
||||
api_client.close()
|
||||
print("S3 bootstrap complete: private bucket and scoped API credentials are ready.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
|
||||
create_secret() {
|
||||
name=$1
|
||||
length=$2
|
||||
if [ ! -s "secrets/$name" ]; then
|
||||
openssl rand -hex "$length" > "secrets/$name"
|
||||
chmod 600 "secrets/$name"
|
||||
fi
|
||||
}
|
||||
|
||||
create_secret database_password 32
|
||||
create_secret s3_root_access_key 16
|
||||
create_secret s3_root_secret_key 32
|
||||
create_secret s3_access_key_id 16
|
||||
create_secret s3_secret_access_key 32
|
||||
printf '%s\n' 'Local Compose secrets are ready; values were not printed.'
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconcile attachment object keys and metadata; safe by default."""
|
||||
import argparse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.settings import database_connection, read_setting
|
||||
|
||||
LOGGER = logging.getLogger("attachment-reconciler")
|
||||
|
||||
|
||||
def plan_reconciliation(metadata, objects, now, grace):
|
||||
cutoff = now - grace
|
||||
object_keys = {item["Key"]: item for item in objects}
|
||||
metadata_by_key = {row["object_key"]: row for row in metadata}
|
||||
return {
|
||||
"orphan_old": sorted(
|
||||
key
|
||||
for key, item in object_keys.items()
|
||||
if key.startswith("attachments/")
|
||||
and key not in metadata_by_key
|
||||
and item.get("LastModified") is not None
|
||||
and item["LastModified"] <= cutoff
|
||||
),
|
||||
"orphan_fresh": sorted(
|
||||
key
|
||||
for key, item in object_keys.items()
|
||||
if key.startswith("attachments/")
|
||||
and key not in metadata_by_key
|
||||
and (
|
||||
item.get("LastModified") is None
|
||||
or item["LastModified"] > cutoff
|
||||
)
|
||||
),
|
||||
"missing_unreferenced_old": sorted(
|
||||
row["id"]
|
||||
for row in metadata
|
||||
if row["object_key"] not in object_keys
|
||||
and not row["referenced"]
|
||||
and row["created_at"] <= cutoff
|
||||
),
|
||||
"missing_protected": sorted(
|
||||
row["id"]
|
||||
for row in metadata
|
||||
if row["object_key"] not in object_keys
|
||||
and (row["referenced"] or row["created_at"] > cutoff)
|
||||
),
|
||||
"object_count": len(object_keys),
|
||||
"metadata_count": len(metadata_by_key),
|
||||
}
|
||||
|
||||
|
||||
def load_state(connection, s3, bucket):
|
||||
metadata = connection.execute(
|
||||
"""
|
||||
SELECT a.id::text AS id, a.object_key, a.created_at,
|
||||
EXISTS (
|
||||
SELECT 1 FROM message_attachments ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
) AS referenced
|
||||
FROM attachments a
|
||||
"""
|
||||
).fetchall()
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
objects = []
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"):
|
||||
objects.extend(page.get("Contents", []))
|
||||
return metadata, objects
|
||||
|
||||
|
||||
def apply_plan(connection, s3, bucket, plan):
|
||||
deleted_objects = 0
|
||||
deleted_metadata = 0
|
||||
for key in plan["orphan_old"]:
|
||||
s3.delete_object(Bucket=bucket, Key=key)
|
||||
deleted_objects += 1
|
||||
for attachment_id in plan["missing_unreferenced_old"]:
|
||||
with connection.transaction():
|
||||
row = connection.execute(
|
||||
"""
|
||||
DELETE FROM attachments AS a
|
||||
WHERE a.id = %s
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM message_attachments ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
)
|
||||
RETURNING a.id
|
||||
""",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if row:
|
||||
deleted_metadata += 1
|
||||
return deleted_objects, deleted_metadata
|
||||
|
||||
|
||||
def main():
|
||||
import boto3
|
||||
import botocore
|
||||
import psycopg
|
||||
from botocore.config import Config
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="delete stale orphan objects and stale unreferenced missing metadata",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--grace-hours",
|
||||
type=float,
|
||||
default=24,
|
||||
help="minimum age before repair; defaults to 24 hours",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.grace_hours < 0:
|
||||
parser.error("--grace-hours cannot be negative")
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
||||
started = time.monotonic()
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"),
|
||||
aws_secret_access_key=read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"),
|
||||
config=Config(
|
||||
connect_timeout=2, read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
try:
|
||||
with psycopg.connect(**database_connection(), row_factory=dict_row) as connection:
|
||||
metadata, objects = load_state(connection, s3, bucket)
|
||||
plan = plan_reconciliation(
|
||||
metadata,
|
||||
objects,
|
||||
datetime.now(timezone.utc),
|
||||
timedelta(hours=args.grace_hours),
|
||||
)
|
||||
changes = {"objects_deleted": 0, "metadata_deleted": 0}
|
||||
if args.apply:
|
||||
(
|
||||
changes["objects_deleted"],
|
||||
changes["metadata_deleted"],
|
||||
) = apply_plan(connection, s3, bucket, plan)
|
||||
except (psycopg.Error, botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError):
|
||||
LOGGER.exception("reconciliation failed; no success result was produced")
|
||||
raise SystemExit(1) from None
|
||||
summary = {
|
||||
"mode": "apply" if args.apply else "dry-run",
|
||||
"grace_hours": args.grace_hours,
|
||||
"objects_scanned": plan["object_count"],
|
||||
"metadata_scanned": plan["metadata_count"],
|
||||
"orphan_objects": len(plan["orphan_old"]) + len(plan["orphan_fresh"]),
|
||||
"orphan_old_candidates": len(plan["orphan_old"]),
|
||||
"orphan_fresh_preserved": len(plan["orphan_fresh"]),
|
||||
"missing_metadata_rows": len(plan["missing_unreferenced_old"])
|
||||
+ len(plan["missing_protected"]),
|
||||
"missing_unreferenced_old_candidates": len(plan["missing_unreferenced_old"]),
|
||||
"missing_referenced_or_fresh_preserved": len(plan["missing_protected"]),
|
||||
**changes,
|
||||
"elapsed_seconds": round(time.monotonic() - started, 3),
|
||||
}
|
||||
print(json.dumps(summary, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
docker compose restart postgres redis s3 >/dev/null
|
||||
deadline=$(( $(date +%s) + 60 ))
|
||||
for service in postgres redis; do
|
||||
container=$(docker compose ps -q "$service")
|
||||
while :; do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||
if [ "$status" = healthy ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
printf '%s\n' "$service did not become healthy after restart" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
|
||||
docker compose run --rm --no-deps s3-bootstrap >/dev/null
|
||||
"$ROOT/scripts/rolling-restart.sh"
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
containers=$(docker compose ps -q api)
|
||||
healthy_count=0
|
||||
for container in $containers; do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||
if [ "$status" = healthy ]; then
|
||||
healthy_count=$((healthy_count + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$healthy_count" -lt 2 ]; then
|
||||
printf '%s\n' 'Rolling restart requires at least two healthy API replicas' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for container in $containers; do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||
if [ "$status" != healthy ]; then
|
||||
printf '%s\n' "Skipping unhealthy API replica: $container" >&2
|
||||
continue
|
||||
fi
|
||||
docker restart --time 10 "$container" >/dev/null
|
||||
deadline=$(( $(date +%s) + 60 ))
|
||||
while :; do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
||||
if [ "$status" = healthy ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
printf '%s\n' "API replica did not become healthy: $container" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
|
||||
for container in $(docker compose ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 2 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2
|
||||
exit 1
|
||||
@@ -55,3 +55,16 @@ make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.
|
||||
## Что ещё остаётся доказать
|
||||
|
||||
Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
|
||||
|
||||
## Дополнительные проверки lab4
|
||||
|
||||
```sh
|
||||
make test-grade4
|
||||
make test-s3-failure
|
||||
make test-upload-load OUTPUT=evidence/upload-load.json
|
||||
make test-reconcile
|
||||
```
|
||||
|
||||
`test-grade4` проверяет слишком большой файл, повреждение, MIME-подмену, пиксельный лимит, опасное имя, cross-chat привязку, чужое вложение и конкурентную идемпотентность. `test-s3-failure` временно останавливает локальный Compose S3 и обязательно поднимает его обратно. `test-upload-load` отправляет десять параллельных синтетических JPEG и семплирует память API через `docker stats`; частота замеров ограничивает точность пика.
|
||||
|
||||
Reconciler `python scripts/reconcile.py` по умолчанию работает только в режиме dry-run и использует grace period 24 часа. Он рассматривает для удаления только старые объекты `attachments/`, для которых нет metadata, и старые missing metadata без ссылок в сообщениях. `--apply` включает исправление; уменьшать grace period до нуля можно лишь на disposable synthetic стенде. Ссылочные вложения и свежие объекты сохраняются. Missing object у используемого вложения выводится как защищённое несоответствие для ручного восстановления, автоматическое удаление metadata не выполняется. `make test-reconcile` создаёт синтетические сбои и демонстрирует dry-run, grace period, исправление и повторный запуск; команда не подходит для production или ценных данных.
|
||||
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent idempotency and replica-distribution checks."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from image_fixture import solid_gray_png
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Concurrent shared-idempotency and API-replica check")
|
||||
client = Client(args.base_url, args.ca_file, timeout=5)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": "parallel-idempotency-" + key}
|
||||
|
||||
def send(_):
|
||||
return Client(args.base_url, args.ca_file, timeout=5).call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
201,
|
||||
body=body,
|
||||
actor=alice,
|
||||
headers={"Idempotency-Key": key},
|
||||
schema="Message",
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=12) as executor:
|
||||
responses = list(executor.map(send, range(12)))
|
||||
messages = [response["json"] for response in responses]
|
||||
check(all(message == messages[0] for message in messages), "concurrent replay differed")
|
||||
|
||||
conflict = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
409,
|
||||
body={"text": "different body"},
|
||||
actor=alice,
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
check(conflict["status"] == 409, "same key with a different body was accepted")
|
||||
|
||||
instances = {
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][
|
||||
"X-Instance-Id"
|
||||
]
|
||||
for _ in range(80)
|
||||
}
|
||||
check(len(instances) >= 2, "load balancer did not expose two API instances")
|
||||
|
||||
upload_path = "/api/v1/chats/" + chat["id"] + "/attachments"
|
||||
fixture = (Path(__file__).parent / "fixtures" / "sample.png").read_bytes()
|
||||
client.call(
|
||||
"POST",
|
||||
upload_path + "?filename=too-large.png",
|
||||
413,
|
||||
actor=alice,
|
||||
raw=b"x" * (10 * 1024 * 1024 + 1),
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
upload_path + "?filename=broken.png",
|
||||
415,
|
||||
actor=alice,
|
||||
raw=b"\x89PNG\r\n\x1a\nnot-an-image",
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
upload_path + "?filename=mime.jpg",
|
||||
415,
|
||||
actor=alice,
|
||||
raw=fixture,
|
||||
headers={"Content-Type": "image/jpeg"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
upload_path + "?filename=pixels.png",
|
||||
413,
|
||||
actor=alice,
|
||||
raw=solid_gray_png(6401, 6251),
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
upload_path + "?filename=..%2Fprivate.png",
|
||||
400,
|
||||
actor=alice,
|
||||
raw=fixture,
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
other_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + other_chat["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
for path in (
|
||||
"/api/v1/attachments/" + attachment["id"],
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
):
|
||||
client.call("GET", path, 404, actor=eve)
|
||||
print("PASS: concurrent idempotency returned one message; observed %d API replicas" % len(instances))
|
||||
print("PASS: level-4 attachment error, pixel limit, chat binding and outsider checks.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Small deterministic PNG builders for standard-library-only acceptance tests."""
|
||||
import random
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def _chunk(name, data):
|
||||
kind = name.encode("ascii")
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def solid_gray_png(width, height):
|
||||
row_size = (width + 7) // 8
|
||||
raw = (b"\x00" + bytes(row_size)) * height
|
||||
image_header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + _chunk("IHDR", image_header) + _chunk(
|
||||
"IDAT", zlib.compress(raw)
|
||||
) + _chunk("IEND", b"")
|
||||
|
||||
|
||||
def noisy_rgb_png(width, height, seed=42024):
|
||||
image_header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
compressor = zlib.compressobj(level=1)
|
||||
compressed = bytearray()
|
||||
rng = random.Random(seed)
|
||||
for _ in range(height):
|
||||
compressed.extend(compressor.compress(b"\x00" + rng.randbytes(width * 3)))
|
||||
compressed.extend(compressor.flush())
|
||||
return b"\x89PNG\r\n\x1a\n" + _chunk("IHDR", image_header) + _chunk(
|
||||
"IDAT", bytes(compressed)
|
||||
) + _chunk("IEND", b"")
|
||||
Executable
+148
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inject safe synthetic S3/DB inconsistencies and verify reconciliation."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore import UNSIGNED
|
||||
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def reconcile(*options):
|
||||
result = subprocess.run(
|
||||
[sys.executable, "scripts/reconcile.py", *options],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
check(result.returncode == 0, "reconciler command failed: " + result.stderr.strip())
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def upload(client, alice, chat):
|
||||
return client.attachment(alice, chat)[0]
|
||||
|
||||
|
||||
def main():
|
||||
base_url = os.environ.get("RECONCILE_TEST_BASE_URL", "http://load_balancer:8080")
|
||||
client = Client(base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
linked = upload(client, alice, chat)
|
||||
client.message(alice, chat, "", attachments=[linked["id"]])
|
||||
missing_referenced = upload(client, alice, chat)
|
||||
client.message(alice, chat, "", attachments=[missing_referenced["id"]])
|
||||
missing_unreferenced = upload(client, alice, chat)
|
||||
|
||||
with open(os.environ["S3_ACCESS_KEY_ID_FILE"], encoding="utf-8") as file:
|
||||
access_key = file.read().strip()
|
||||
with open(os.environ["S3_SECRET_ACCESS_KEY_FILE"], encoding="utf-8") as file:
|
||||
secret_key = file.read().strip()
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5),
|
||||
)
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
for attachment in (missing_referenced, missing_unreferenced):
|
||||
s3.delete_object(Bucket=bucket, Key="attachments/" + attachment["id"])
|
||||
fresh_orphan = "attachments/" + str(uuid.uuid4())
|
||||
with open("tests/fixtures/sample.png", "rb") as file:
|
||||
fixture = file.read()
|
||||
s3.put_object(
|
||||
Bucket=bucket,
|
||||
Key=fresh_orphan,
|
||||
Body=fixture,
|
||||
ContentType="image/png",
|
||||
)
|
||||
|
||||
denied = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"),
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
config=Config(signature_version=UNSIGNED, connect_timeout=2, read_timeout=5),
|
||||
)
|
||||
try:
|
||||
denied.get_object(Bucket=bucket, Key="attachments/" + linked["id"])
|
||||
raise AssertionError("private S3 bucket allowed an anonymous object read")
|
||||
except ClientError as error:
|
||||
check(
|
||||
error.response["ResponseMetadata"]["HTTPStatusCode"] in {403, 404},
|
||||
"anonymous S3 request did not fail as private access",
|
||||
)
|
||||
|
||||
dry_run = reconcile("--grace-hours", "24")
|
||||
check(dry_run["mode"] == "dry-run", "default reconciliation must be dry-run")
|
||||
check(dry_run["orphan_fresh_preserved"] >= 1, "fresh orphan was not protected")
|
||||
check(
|
||||
dry_run["missing_referenced_or_fresh_preserved"] >= 2,
|
||||
"referenced/fresh missing metadata was not protected",
|
||||
)
|
||||
applied_grace = reconcile("--apply", "--grace-hours", "24")
|
||||
check(applied_grace["objects_deleted"] == 0, "grace run deleted fresh object")
|
||||
check(applied_grace["metadata_deleted"] == 0, "grace run deleted fresh metadata")
|
||||
s3.head_object(Bucket=bucket, Key=fresh_orphan)
|
||||
s3.head_object(Bucket=bucket, Key="attachments/" + linked["id"])
|
||||
|
||||
repaired = reconcile("--apply", "--grace-hours", "0")
|
||||
check(repaired["objects_deleted"] >= 1, "stale orphan object was not removed")
|
||||
check(repaired["metadata_deleted"] >= 1, "stale unreferenced missing row was not removed")
|
||||
check(
|
||||
repaired["missing_referenced_or_fresh_preserved"] >= 1,
|
||||
"missing referenced attachment was not protected",
|
||||
)
|
||||
repeated = reconcile("--apply", "--grace-hours", "0")
|
||||
check(repeated["objects_deleted"] == 0, "repeated run was not idempotent for objects")
|
||||
check(repeated["metadata_deleted"] == 0, "repeated run was not idempotent for metadata")
|
||||
check(
|
||||
client.call("GET", "/api/v1/attachments/" + linked["id"], actor=alice)["status"] == 200,
|
||||
"reconciler removed a used attachment",
|
||||
)
|
||||
downloaded = client.call(
|
||||
"GET", "/api/v1/attachments/" + linked["id"] + "/content", actor=alice
|
||||
)
|
||||
check(downloaded["bytes"] == fixture, "reconciler damaged a used attachment")
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + missing_referenced["id"],
|
||||
200,
|
||||
actor=alice,
|
||||
schema="Attachment",
|
||||
)
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + missing_unreferenced["id"],
|
||||
404,
|
||||
actor=alice,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"private_bucket_anonymous_read": "denied",
|
||||
"dry_run_elapsed_seconds": dry_run["elapsed_seconds"],
|
||||
"objects_scanned": dry_run["objects_scanned"],
|
||||
"metadata_scanned": dry_run["metadata_scanned"],
|
||||
"dry_run_orphan_candidates": dry_run["orphan_old_candidates"],
|
||||
"fresh_orphan_preserved": applied_grace["orphan_fresh_preserved"],
|
||||
"apply_deleted_objects": repaired["objects_deleted"],
|
||||
"apply_deleted_unreferenced_metadata": repaired["metadata_deleted"],
|
||||
"referenced_missing_metadata_preserved": True,
|
||||
"repeat_deleted_objects": repeated["objects_deleted"],
|
||||
"repeat_deleted_metadata": repeated["metadata_deleted"],
|
||||
"used_attachment_preserved": True,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail one explicitly selected replica and write sanitized evidence."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def history(client, actor, chat):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
messages = []
|
||||
seen = set()
|
||||
while True:
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
messages.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return messages
|
||||
check(cursor not in seen, "history cursor cycle")
|
||||
seen.add(cursor)
|
||||
path = (
|
||||
"/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/messages?limit=100&cursor="
|
||||
+ quote(cursor, safe="")
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--target-id", required=True)
|
||||
parser.add_argument("--action-script", required=True)
|
||||
parser.add_argument("--output")
|
||||
|
||||
args = arguments("Targeted API failover evidence", configure)
|
||||
target = args.target_id
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script, os.X_OK), "action script must be executable")
|
||||
client = Client(args.base_url, args.ca_file, timeout=2)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
observed_before = {
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][
|
||||
"X-Instance-Id"
|
||||
]
|
||||
for _ in range(40)
|
||||
}
|
||||
check(len(observed_before) >= 2, "fewer than two API replicas observed")
|
||||
check(target in observed_before, "target ID is not a currently observed API replica")
|
||||
|
||||
original = None
|
||||
for _ in range(20):
|
||||
original = client.message(
|
||||
alice, chat, "pre-failure-" + uuid.uuid4().hex, key=uuid.uuid4().hex
|
||||
)
|
||||
if original["headers"]["X-Instance-Id"] == target:
|
||||
break
|
||||
check(original["headers"]["X-Instance-Id"] == target, "could not select requested replica")
|
||||
|
||||
env = dict(os.environ)
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
process = subprocess.Popen(
|
||||
[str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
started = time.monotonic()
|
||||
recovered_at = None
|
||||
successful_writes = []
|
||||
failed_writes = 0
|
||||
failed_reads = 0
|
||||
served_after = []
|
||||
key = uuid.uuid4().hex
|
||||
text = "during-failover-" + key
|
||||
try:
|
||||
while time.monotonic() - started < 15:
|
||||
status = process.poll()
|
||||
check(status is None or status == 0, "action script failed")
|
||||
check(status is not None or time.monotonic() - started < 10, "action script exceeded 10s")
|
||||
try:
|
||||
read = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
served_after.append(read["headers"]["X-Instance-Id"])
|
||||
except Failure:
|
||||
failed_reads += 1
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
try:
|
||||
response = client.message(alice, chat, text, key=key)
|
||||
successful_writes.append(response["json"])
|
||||
if status == 0 and recovered_at is None:
|
||||
recovered_at = time.monotonic() - started
|
||||
key = uuid.uuid4().hex
|
||||
text = "during-failover-" + key
|
||||
except Failure:
|
||||
failed_writes += 1
|
||||
time.sleep(0.1)
|
||||
|
||||
check(process.poll() == 0, "action script did not finish successfully")
|
||||
check(recovered_at is not None and recovered_at <= 10, "writes did not recover within 10s")
|
||||
check(len(successful_writes) >= 5, "too few successful writes after the failure")
|
||||
check(target not in served_after[-5:], "stopped replica still serves requests")
|
||||
all_messages = history(client, bob, chat)
|
||||
ids = [message["id"] for message in all_messages]
|
||||
texts = [message["text"] for message in all_messages]
|
||||
check(len(ids) == len(set(ids)), "duplicate message IDs in history")
|
||||
check(len(texts) == len(set(texts)), "retry created duplicate message bodies")
|
||||
check(original["json"] in all_messages, "confirmed pre-failure message was lost")
|
||||
for message in successful_writes:
|
||||
check(message in all_messages, "confirmed failover write was lost")
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
total_attempts = len(successful_writes) + failed_writes
|
||||
result = {
|
||||
"target_instance_id": target,
|
||||
"observed_instances_before": len(observed_before),
|
||||
"instances_serving_after": sorted(set(served_after)),
|
||||
"duration_seconds": round(elapsed, 3),
|
||||
"successful_writes": len(successful_writes),
|
||||
"failed_write_attempts": failed_writes,
|
||||
"failed_read_attempts": failed_reads,
|
||||
"successful_rps": round(len(successful_writes) / elapsed, 3),
|
||||
"write_error_ratio": round(failed_writes / total_attempts, 6),
|
||||
"recovery_seconds": round(recovered_at, 3),
|
||||
"confirmed_data_preserved": True,
|
||||
"duplicate_messages": 0,
|
||||
}
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(rendered, encoding="utf-8")
|
||||
print(rendered, end="")
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify uploads return an explicit 503 while the Compose S3 service is stopped."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, ROOT, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def healthy_api_container():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "-q", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for container in result.stdout.split():
|
||||
health = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.State.Health.Status}}", container],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if health.stdout.strip() == "healthy":
|
||||
return container
|
||||
raise AssertionError("no healthy API replica is available")
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("S3 outage behavior; stops and restores the local Compose S3 service")
|
||||
client = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
fixture = (Path(__file__).parent / "fixtures" / "sample.png").read_bytes()
|
||||
api_container = healthy_api_container()
|
||||
stopped = False
|
||||
try:
|
||||
compose("stop", "s3")
|
||||
stopped = True
|
||||
payload = json.dumps(
|
||||
{"alice": alice, "chat": chat, "fixture": fixture.hex()}
|
||||
)
|
||||
worker = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-i",
|
||||
api_container,
|
||||
"python",
|
||||
"tests/s3_failure_worker.py",
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=payload,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
check(worker.returncode == 0, "S3 failure worker failed: " + worker.stderr.strip())
|
||||
check(worker.stdout.strip() == "PASS", "API did not return 503 for unavailable S3")
|
||||
finally:
|
||||
if stopped:
|
||||
compose("up", "-d", "s3")
|
||||
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except AssertionError:
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise AssertionError("API did not become ready after S3 recovery")
|
||||
print("PASS: upload returned 503 during S3 outage; bucket and API recovered with volumes intact.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Internal worker for tests/s3_failure.py; reads auth data only from stdin."""
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main():
|
||||
payload = json.load(sys.stdin)
|
||||
alice = payload["alice"]
|
||||
chat = payload["chat"]
|
||||
content = bytes.fromhex(payload["fixture"])
|
||||
request = urllib.request.Request(
|
||||
"http://127.0.0.1:8080/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/attachments?filename=s3-down.png",
|
||||
data=content,
|
||||
headers={
|
||||
"Authorization": "Bearer " + alice["token"],
|
||||
"Content-Type": "image/png",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
urllib.request.urlopen(request, timeout=15)
|
||||
except urllib.error.HTTPError as response:
|
||||
if response.code != 503:
|
||||
raise AssertionError("expected HTTP 503, received " + str(response.code))
|
||||
body = json.loads(response.read())
|
||||
request_id = response.headers.get("X-Request-Id")
|
||||
if body.get("error", {}).get("code") != "unavailable":
|
||||
raise AssertionError("expected unavailable error response")
|
||||
if not request_id or body["error"].get("request_id") != request_id:
|
||||
raise AssertionError("error request ID does not match response header")
|
||||
if not response.headers.get("X-Instance-Id"):
|
||||
raise AssertionError("missing API instance ID")
|
||||
else:
|
||||
raise AssertionError("expected HTTP 503 during the S3 outage")
|
||||
print("PASS")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from scripts.reconcile import plan_reconciliation
|
||||
|
||||
|
||||
class ReconcilePlanTests(unittest.TestCase):
|
||||
def test_only_stale_orphans_and_old_unreferenced_rows_are_candidates(self):
|
||||
now = datetime.now(timezone.utc)
|
||||
old = now - timedelta(days=2)
|
||||
fresh = now - timedelta(minutes=2)
|
||||
metadata = [
|
||||
{"id": "used", "object_key": "attachments/used", "created_at": old, "referenced": True},
|
||||
{"id": "unreferenced", "object_key": "attachments/missing", "created_at": old, "referenced": False},
|
||||
{"id": "fresh", "object_key": "attachments/fresh-missing", "created_at": fresh, "referenced": False},
|
||||
]
|
||||
objects = [
|
||||
{"Key": "attachments/used", "LastModified": old},
|
||||
{"Key": "attachments/orphan-old", "LastModified": old},
|
||||
{"Key": "attachments/orphan-fresh", "LastModified": fresh},
|
||||
{"Key": "unrelated/keep", "LastModified": old},
|
||||
]
|
||||
|
||||
plan = plan_reconciliation(metadata, objects, now, timedelta(hours=24))
|
||||
|
||||
self.assertEqual(plan["orphan_old"], ["attachments/orphan-old"])
|
||||
self.assertEqual(plan["orphan_fresh"], ["attachments/orphan-fresh"])
|
||||
self.assertEqual(plan["missing_unreferenced_old"], ["unreferenced"])
|
||||
self.assertEqual(plan["missing_protected"], ["fresh"])
|
||||
|
||||
def test_repeat_after_repair_has_no_candidates(self):
|
||||
now = datetime.now(timezone.utc)
|
||||
plan = plan_reconciliation([], [], now, timedelta(hours=24))
|
||||
self.assertEqual(plan["orphan_old"], [])
|
||||
self.assertEqual(plan["missing_unreferenced_old"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload 10+ valid images concurrently and sample API container memory."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from client import Client, check
|
||||
from image_fixture import noisy_rgb_png
|
||||
|
||||
|
||||
def memory_bytes(value):
|
||||
match = re.match(r"\s*([\d.]+)\s*([KMG]i?B)", value)
|
||||
if not match:
|
||||
return 0
|
||||
amount = float(match.group(1))
|
||||
unit = match.group(2).upper()
|
||||
return int(
|
||||
amount
|
||||
* {
|
||||
"KB": 1000,
|
||||
"MB": 1000**2,
|
||||
"GB": 1000**3,
|
||||
"KIB": 1024,
|
||||
"MIB": 1024**2,
|
||||
"GIB": 1024**3,
|
||||
}[unit]
|
||||
)
|
||||
|
||||
|
||||
def compose_api_ids():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "-q", "api"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
healthy_ids = []
|
||||
for container_id in result.stdout.split():
|
||||
health = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.State.Health.Status}}", container_id],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if health.stdout.strip() == "healthy":
|
||||
healthy_ids.append(container_id)
|
||||
check(len(healthy_ids) >= 2, "need at least two healthy API containers for the measurement")
|
||||
return healthy_ids
|
||||
|
||||
|
||||
def sample_memory(ids, stop, peak):
|
||||
while not stop.is_set():
|
||||
for container_id in ids:
|
||||
result = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
used = memory_bytes(result.stdout.split("/", 1)[0])
|
||||
peak[container_id] = max(peak.get(container_id, 0), used)
|
||||
stop.wait(0.5)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--output", default=None)
|
||||
args = parser.parse_args()
|
||||
if args.concurrency < 10:
|
||||
parser.error("--concurrency must be at least 10")
|
||||
|
||||
bootstrap = Client(args.base_url)
|
||||
alice, _, _, chat = bootstrap.bootstrap()
|
||||
content = noisy_rgb_png(1200, 1200)
|
||||
check(0 < len(content) <= 10 * 1024 * 1024, "generated PNG must be within 10 MiB")
|
||||
api_ids = compose_api_ids()
|
||||
stop = threading.Event()
|
||||
peak = {}
|
||||
sampler = threading.Thread(target=sample_memory, args=(api_ids, stop, peak), daemon=True)
|
||||
sampler.start()
|
||||
started = time.monotonic()
|
||||
attachment_ids = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
tasks = [
|
||||
executor.submit(
|
||||
Client(args.base_url).call,
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load.png",
|
||||
201,
|
||||
None,
|
||||
alice,
|
||||
{"Content-Type": "image/png"},
|
||||
content,
|
||||
"Attachment",
|
||||
)
|
||||
for _ in range(args.concurrency)
|
||||
]
|
||||
for task in as_completed(tasks):
|
||||
attachment_ids.append(task.result()["json"]["id"])
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
result = {
|
||||
"concurrency": args.concurrency,
|
||||
"upload_bytes_each": len(content),
|
||||
"uploads_succeeded": len(attachment_ids),
|
||||
"elapsed_seconds": round(time.monotonic() - started, 3),
|
||||
"api_container_peak_memory_bytes": peak,
|
||||
"api_container_ids": api_ids,
|
||||
"notes": "Peak sampled every 0.5s using docker stats; excludes database, LB, and client memory.",
|
||||
}
|
||||
if len(attachment_ids) != args.concurrency:
|
||||
raise SystemExit("not every upload succeeded")
|
||||
output = json.dumps(result, indent=2, sort_keys=True)
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as file:
|
||||
file.write(output + "\n")
|
||||
print(output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user