This commit is contained in:
2026-10-16 22:47:54 +03:00
parent ec540a1a72
commit 16709c91b3
85 changed files with 19098 additions and 22 deletions
+1
View File
@@ -4,6 +4,7 @@
.env.*
!.env.example
secrets
.secrets
certs
volumes
data
+1
View File
@@ -3,6 +3,7 @@
.env.*
!.env.example
secrets/
.secrets/
certs/private/
*.key
*.p12
+30
View File
@@ -0,0 +1,30 @@
FROM python:3.12.10-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir --requirement requirements.txt \
&& groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app
COPY --chown=app:app src ./src
COPY --chown=app:app migrations ./migrations
COPY --chown=app:app scripts ./scripts
COPY --chown=app:app student-tests ./student-tests
COPY --chown=app:app tests ./tests
COPY --chown=app:app contracts ./contracts
COPY --chown=app:app lab.json ./lab.json
RUN mkdir -p evidence && chown app:app evidence
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
CMD ["python", "/app/scripts/healthcheck.py"]
CMD ["sh", "-c", "exec uvicorn src.main:app --host \"${APP_HOST}\" --port \"${APP_PORT}\" --workers 1 --timeout-graceful-shutdown 8"]
+32
View File
@@ -28,3 +28,35 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: setup start ready student-checks test-availability test-expiry test-dependencies test-files test-file-failover test-storage experiment reconcile stop
setup:
bash scripts/setup.sh
start:
bash scripts/start.sh
ready:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/stack.py wait --count 2
student-checks:
$(PYTHON) student-tests/test_api.py --base-url "$(BASE_URL)"
$(PYTHON) student-tests/checks.py --base-url "$(BASE_URL)"
$(PYTHON) student-tests/idempotency.py --base-url "$(BASE_URL)"
docker exec "$$(docker compose ps --status running -q api | head -n 1)" python student-tests/database.py
test-files:
$(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)"
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py --mode privacy
test-file-failover:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/file_failover.py
test-storage:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py --mode faults
test-availability:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/availability.py
test-dependencies:
BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/dependencies.py
test-expiry:
bash scripts/test_expiry.sh
experiment:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py
reconcile:
docker compose run --rm --no-deps reconciler python -m src.reconcile
stop:
docker compose down
+398 -22
View File
@@ -1,37 +1,413 @@
# Отчёт — лабораторная 4
# Лабораторная 4 — S3 и вложения
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: Максим Шептунов, БВТ2301.
- Целевая оценка: 5; трек 2, «Согласованность двух хранилищ».
- Skeleton: `ec540a1a725862a8619fcb758daec4ad197f0dd5`.
- Проверки и эксперимент: 16.10.2026, локальный Docker-стенд в WSL2.
- Помощь ИИ: помощь с реализацией, проверками и разбором результатов.
Ниже описано решение и результаты моего запуска. Числа рассчитаны по
`evidence/local_*` и исходным JSON; результаты эксперимента сохранены в evidence.
README, публичные tests и контракт skeleton сохранены.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine с Compose v2/BuildKit, Python 3.10+ на хосте и Linux/amd64
для закреплённых S3-бинарников. В Windows — WSL Ubuntu с Docker Desktop.
Порт 8080 должен быть свободен. Сборка требует доступа к PyPI, Docker Hub
для базовых образов и GitHub Releases для MinIO/mc.
## Архитектура и решения
```bash
python3 -m venv .venv
source .venv/bin/activate
pip install -r requirements-dev.txt -r requirements-experiment.txt
chmod +x scripts/*.sh
make setup
make start
make ready
make check
make validate
make test
make student-checks
make test-files
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
bash scripts/restore-api.sh
make test-file-failover
make test-availability
make test-dependencies
make test-expiry
make experiment
make test
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
API-зависимости устанавливаются внутри Docker из `requirements.txt`.
`setup.sh` генерирует `.env` и секреты в `.secrets`, не выводя значения.
Папка секретов 700; файлы, читаемые пользователем API через mount, 644 внутри
закрытой папки; root-credentials MinIO и пароль PG — 600. Secrets/.env исключены
из Git и build context. Настройки повторно не генерируют новые credentials.
## Доказательства критериев
Сервис `migrate` применяет SQL и завершается, `s3-bootstrap` создаёт приватный
bucket и ограниченного пользователя API и тоже завершается. Exited(0) для них
нормален. Затем работают две API, LB, PG, Redis, MinIO и reconciler.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
Общий запуск выполняется без Compose `--wait`: у фонового reconciler нет healthcheck.
После `up` скрипт отдельно ждёт две готовые API через LB (`stack.py wait --count 2`).
`make stop` сохраняет volumes. Намеренное удаление всех учебных данных:
`docker compose down -v`; оно не используется в тестах сохранности.
Python-образ 3.12.10 slim-bookworm, PG 16.9 bookworm, Redis 7.4.3 bookworm,
HAProxy 3.0.11 bookworm. MinIO RELEASE.2025-04-22T22-12-26Z и mc
RELEASE.2025-04-16T18-13-26Z собираются в локальные образы через `infra/s3.Dockerfile`:
официальные GitHub Release binaries, проверка закреплённого SHA-256 до запуска.
Это не требует доступа к переставшим быть общедоступными образам minio/minio.
Все библиотеки закреплены по версиям.
## Окружение измерений
| Параметр | Значение |
| --- | --- |
| Дата | 16.10.2026 |
| Система | WSL2, Linux 5.15.146.1-microsoft-standard-WSL2, x86_64 |
| Ресурсы Docker | 16 логических CPU; 8187711488 bytes RAM, около 7.63 GiB |
| Docker Engine сервера | 26.1.1, Docker Desktop |
| Docker CLI | 28.0.1 |
| Docker Compose | 2.27.0-desktop.2 |
| Python на хосте | 3.10.12 |
| API | 2 реплики, каждая 1 CPU / 512 MiB, DB_POOL_MAX=8 |
| Reconciler | 0.5 CPU / 256 MiB |
Источник: `evidence/local_environment.txt`, `experiment_environment.json`,
`containers_before.json`, `containers_final.json`. CPU/RAM-лимиты API также
сохранены отдельно для каждого повтора upload в `upload_memory.json`.
После всех сценариев снова видны две готовые API; финальный smoke прошёл.
## Перенос и архитектура
Основа — фактическое решение lab3 после проверок: PostgreSQL, общие сессии,
Argon2id, Redis-кеш, keyset-пагинация, две API за HAProxy, X-Instance-Id,
идемпотентность сообщений, рестарт и rolling. Сохранены исправления выбора
реально работающего контейнера вместо жёсткого индекса API и Redis fallback.
Исследование масштабирования lab3 повторно не запускается: в lab4 выбран другой трек.
Также перенесено исправление проверки cursor из lab3: JSON должен быть объектом,
created_at и id — строками; неверные типы возвращают 400 invalid_request.
Регрессионные проверки охватывают историю и список чатов на обеих API.
Добавлены MinIO, bootstrap bucket/IAM, вложения, третья миграция,
reconciler и сценарии файлов/RAM/двух хранилищ. Описание переноса включено
в этот отчёт, отдельного Markdown-файла миграции нет.
Клиент → HAProxy → API ×2 → PostgreSQL + приватный S3.
PostgreSQL хранит метаданные, ACL, связи с сообщениями и результаты повторов;
S3 — оригинальные bytes. Redis — кеш публичных полей профиля, не источник сессий.
Для download авторизация и участие проверяются снова; случайный ключ не заменяет ACL.
Наружу опубликован только `127.0.0.1:8080` у LB. У API/S3 нет опубликованных портов.
Административная консоль MinIO выключена и привязана к loopback контейнера.
API и reconciler используют общего ограниченного S3-пользователя, отдельного
от root. Его права: GET/PUT/DELETE в `messenger-attachments/originals/*` и LIST
этого namespace. Нет прав создания
bucket или администрирования IAM. Root credentials доступны S3 и bootstrap,
но не API/reconciler. Bootstrap повторно безопасен: bucket/private policy/user
создаются или приводятся к заданной конфигурации.
### Контракт файла и сообщения
POST attachment принимает raw bytes PNG/JPEG, не multipart. Лимит 10485760 bytes
проверяется по Content-Length до чтения и по счётчику потока, включая chunked body.
Проверяются отображаемое имя 1–255 символов, отсутствие разделителей пути
и управляющих символов, формат по содержимому и соответствие MIME.
Через Pillow проверяются структура и полноценное декодирование; площадь
ограничена 40000000 пикселей до load. PNG text metadata также ограничены.
Повреждение/MIME — 415, byte/pixel limit — 413, опасное имя — 400.
Object key назначает сервер: `originals/<chat UUID>/<attachment UUID>`.
Filename не используется как путь. SHA-256 и размер описывают исходные bytes.
Валидация не перекодирует оригинал. Upload возвращает 201 ready только после
успешного S3 PUT и COMMIT метаданных. Публичный Attachment не содержит object key
или S3 URL. Download — авторизованный StreamingResponse с chunks 64 KiB, без redirect.
Message содержит `attachment_ids`. Можно передать пустой текст при хотя бы одном
вложении; все ID уникальны и принадлежат этому чату. Любой участник может ссылаться
на доступное вложение, независимо от uploader. Чужое/неизвестное — 404,
неготовое — 409. Message и связи вложений сохраняются одной транзакцией.
Отложенный SQL trigger проверяет текст/связи при COMMIT.
Идемпотентность lab3 сохранена. В хеш включается упорядоченный список непустых
attachment_ids; отсутствующее поле и [] равны. Для старых текстовых запросов
хеш не меняется. Миграция дополняет сохранённые старые ответы полем [] и
не удаляет сообщения/сессии. Разные JSON key order/whitespace не меняют смысл.
### Память, таймауты, готовность
Тело пишется потоково во временный bounded spool; это не постоянное хранилище
оригинала в API. TemporaryFile закрывается после запроса, download читает S3.
На процесс допускаются четыре активных upload, декодируется только одно
изображение одновременно. Остальные клиенты ждут, а не декодируют параллельно.
Нет `await request.body()` на весь файл. Дисковая запись/валидация выполняются
в thread pool, чтобы не блокировать event loop.
API — по 1 CPU/512 MiB: RAM увеличена относительно lab3 для разрешённых больших
изображений. PG 1 CPU/512 MiB, MinIO 1 CPU/512 MiB, Redis/LB по 0.5 CPU/128 MiB,
reconciler 0.5 CPU/256 MiB. Фактические значения фиксируются через Docker inspect.
S3 connect/read timeout 2/5 s, SDK не повторяет PUT автоматически; upload timeout
45 s, LB server timeout 50 s. SQL/lock/pool timeouts 3/2/2 s для API.
Uvicorn graceful shutdown 8 s, Docker grace 10 s. Fault hook намеренно останавливает
выбранную API; после неопределённого message POST повторяется прежний ключ.
Liveness проверяет процесс. Readiness проверяет PG и доступ к приватному S3 namespace;
Redis не критичен. HAProxy опрашивает ready с interval 1 s, check timeout 1 s,
fall 2/rise 1; Docker DNS обнаруживает актуальные реплики. Ориентир обнаружения
отказа — несколько секунд; фактическую границу ≤10 s измеряет failover.
Проверка LIST не обещает отсутствие любого будущего отказа диска или PUT.
`.env` управляет APP_PORT, SESSION_TTL_SECONDS, API_CPU, API_MEMORY, DB_POOL_MAX.
S3 endpoint/bucket/region и пути credentials заданы в compose; MAX_* не превышают
контрактные значения. API_REPLICAS из исходного .env.example заменено Compose scale;
SESSION_STORE фиксирован как PG, CACHE_URL задан напрямую, TTL ключа — 24 h.
## Согласованность PostgreSQL и S3
Общей транзакции между ними нет. Протокол upload:
1. Полностью принять и проверить bounded файл.
2. Взять PostgreSQL session advisory lock на UUID вложения.
3. Сохранить устойчивую запись uploading с lease 120 s и COMMIT.
4. Выполнить S3 PUT с transport Content-MD5 и метаданными SHA-256.
5. Обновить ready и COMMIT, затем разрешить 201.
6. Освободить lock. После ошибки остаётся uploading для безопасной компенсации.
Если PUT прошёл, а запись ready не прошла, объект учтён durable intent.
Если PUT отказал, остаётся uploading без объекта. Обрыв HTTP до полной валидации
не создаёт объектов/метаданных. Upload не имеет гарантии идемпотентности:
повтор его POST может создать другое вложение, как допускает контракт.
Reconciler периодически запускается отдельным контейнером: каждые 60 s,
grace 600 s. Разовый запуск по умолчанию dry-run:
```bash
make reconcile
docker compose run --rm --no-deps reconciler python -m src.reconcile --apply
```
Можно ограничить обход одним чатом через `--chat-id UUID`. Используются LIST,
повторный HEAD и свежий запрос PG под общей блокировкой, а не только старый snapshot.
Недоступность зависимости не считается отсутствием объекта. Dry-run не меняет данные.
| Состояние | Безопасное действие |
| --- | --- |
| Объект без метаданных | Удалить только после grace, повторной проверки и отсутствия активного lock |
| Незавершённый uploading | Ждать lease/grace; при отсутствии ссылок очистить старый объект и запись |
| Используемый ready | Сохранить объект и метаданные |
| Слишком свежий объект/запись | Пропустить |
| Ready, но bytes отсутствуют | Сохранить метаданные/ссылки, пометить missing; API сообщает 503 |
| Исходные bytes возвращены | Проверить размер/SHA-256, затем восстановить ready |
Reconciler не восстанавливает потерянные bytes из ничего. В сценарии оператор
возвращает оригинал из fixture; только после проверки он снова становится ready.
При исправлении missing ссылки из сообщений не удаляются. Повтор применённой
сверки не должен повторно менять уже исправленное состояние.
## Проверки и доказательства
| Критерий | Команда | Результат моего стенда | Evidence |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура/контракт | make check; make validate | PASS; структура и схемы, отдельно от HTTP | local_check.txt, local_validate.txt |
| Обязательный API | make test | PASS до/после отказов и в финале | local_smoke.txt, local_smoke_after_failures.txt, local_final_smoke.txt |
| Минимум lab3, SQL и ключи | make student-checks | PASS; 100 сообщений от 10 клиентов, 32 повтора → 1 сообщение | local_student_checks.txt, idempotency.json |
| Типы cursor | make student-checks | PASS; 400 invalid_request на обеих API, история и список чатов | local_student_checks.txt |
| PNG/JPEG, лимиты, ACL | make test-files | PASS; bytes/pixels, MIME, filename, чужие ID/чаты, replay | local_files.txt |
| Приватность bucket и права | make test-files; make experiment | PASS; anonymous GET 403, CreateBucket AccessDenied, console недоступна | private_bucket.json |
| Прерывание body | make test-files; make experiment | PASS; отправлено 16 из заявленных 4096 bytes, 0 объектов и записей | upload_interrupted.json |
| Полный restart | make test-persistence ACTION_SCRIPT=scripts/restart.sh | PASS; старый токен, сообщения и оригинал сохранились | local_persistence.txt, containers_before.json, containers_after_restart.json |
| Отказ выбранной API | make test-failover ACTION_SCRIPT=scripts/stop-one.sh | PASS; первая успешная запись после hook за 1.524 s | local_failover.txt |
| Стоп uploader → другая API | make test-file-failover | PASS; 111817 bytes, SHA-256 совпал, 1.370 s | attachment_failover.json |
| Каждая API и rolling | make test-availability | PASS; подтверждённые сообщения и оригиналы сохранены | local_availability.txt, failures.json, rolling.json, rolling_containers.json |
| Redis/PG/S3 outage | make test-dependencies | PASS; Redis fallback, PG/S3 → 503, затем восстановление | dependencies.json, local_dependencies.txt |
| Истечение сессии | make test-expiry | PASS; TTL=2 s, затем обычная конфигурация возвращена | local_expiry.txt |
| Ошибки двух хранилищ | make experiment, этап faults | PASS; dry-run/apply/restore/repeat и защита вложений | reconcile_faults.json, s3_write_failure.json |
| 10 concurrent upload, память | make experiment, этап memory | PASS; 6 серий, 60/60 оригиналов, 0 ошибок | upload_memory.json, upload_memory.png |
| Цена обхода | make experiment, этап cost | PASS; 100/500/1000, по 3 измерения после прогрева | reconcile_cost.json, reconcile_cost.png, experiment_summary.json |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Сохранность и переключение реплик
## Эксперименты и отказы
Snapshots до и после restart показывают новый `started_at` у PG, Redis, S3,
LB и обеих API. Для PG/Redis/S3 сохранены container ID; API/LB пересозданы.
Публичный persistence повторно прочитал данные и оригинал с прежней сессией.
`attachment_failover.json` фиксирует остановленный контейнер uploader и другой
`X-Instance-Id` скачавшей API. Контейнер uploader действительно остановлен в `after`.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Публичный failover: 122 успешные записи, 3 неуспешные попытки, первая успешная
запись после старта hook через 1.524 s. Отдельный файловый failover: 1.370 s,
0 неуспешных попыток скачивания, оригинал 111817 bytes совпал побайтно и по SHA-256.
Это разные сценарии и разные точки отсчёта, а не одно измерение.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Нагрузка | Успешных запросов | Неуспешных попыток | Доля ошибок | RPS успешных | p95, ms | Подтверждено / найдено сообщений |
| --- | --- | --- | --- | --- | --- | --- |
| Стоп API №1 | 2034 | 38 | 1.834% | 126.663 | 86.066 | 1228 / 1228 |
| Стоп API №2 | 1995 | 56 | 2.730% | 124.562 | 82.536 | 1212 / 1212 |
| Rolling двух API | 7020 | 176 | 2.446% | 200.515 | 71.279 | 4254 / 4254 |
## Ограничения и следующие шаги
Для остановки каждой API верхняя оценка восстановления по секундным bucket —
2.998 s, меньше требуемых 10 s. Во всех трёх сценариях оригинал также проверен.
Ошибки во время переключения не скрываются: клиенты повторяют неопределённые
message POST с прежним Idempotency-Key. Все подтверждения сверены с историей.
Нулевая потеря подтверждений не означает нулевую долю неуспешных попыток.
В rolling сохранены шаги замены контейнеров; это не бесшовное обновление без ошибок.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
При остановке Redis профиль и запись сообщения продолжили работать; серия
контрольных запросов заняла 7.079 s. Это длительность серии с fallback, не latency
одного запроса. При остановке PG readiness и запрос профиля вернули 503;
при остановке S3 readiness и download вернули 503. После возврата зависимостей
сохранились прежний токен, сообщение и оригинал.
## Эксперимент на 5
Гипотеза: durable intent вместе с lease/lock/grace и повторной сверкой позволяет
исправлять разрыв между S3 и PG, сохраняя свежие, активные и используемые вложения.
Ошибки реальные: временное удаление PUT из IAM-прав API; PostgreSQL trigger,
отказывающий UPDATE ready после S3 PUT; объект без метаданных; удаление bytes
используемого вложения. Все данные синтетические. Trigger удаляется в finally,
обычная IAM policy возвращается, фоновый reconciler на время сценария остановлен.
В тесте grace 20 s и контролируемые старые timestamps PG; возраст S3 orphan
измеряется настоящим ожиданием. Это не настройка production grace.
Проверяются dry-run, apply, сохранность свежих/используемых объектов и активного
lease/lock, возвращение оригинала, повторный apply и сохранность истории.
Автоматическое восстановление S3/PG-кластеров не заявляется.
RAM-серия: два размера noise PNG, 256×256 и 1024×1024; по десять одновременно
отправляющих клиентов, три повтора. API пересоздаются между случаями для нового
high-water mark. Измеряется peak RSS процесса API и cgroup peak при наличии
счётчика ядра, а не только редкие snapshots. Фиксируются реальные лимиты, aliases,
RPS конечного burst, ошибки, p50/p95/p99 и проверка оригиналов второго участника.
Burst RPS не является предельной sustained throughput сервера.
Цена обхода: отдельный синтетический чат, 100/500/1000 объектов и метаданных,
прогрев полного прохода и по три dry-run. Он выполняется в роли reconciler
с теми же CPU/RAM-лимитами. Сохраняются длительность, LIST/HEAD/GET/bytes counters,
process RSS high-water и полный список решений. Raw сохраняется в JSON,
сводка — в experiment_summary.json, графики — в PNG.
### Результаты согласованности
Источник: `evidence/reconcile_faults.json`, `s3_write_failure.json`.
| Сценарий | Наблюдение и результат |
| --- | --- |
| S3 PUT запрещён IAM policy | API вернула 503; в PG осталась uploading, объекта нет |
| Компенсация отказа PUT | Dry-run сохранил запись, apply удалил просроченную запись без объекта; 0.035159 / 0.027420 s |
| PG отказала при UPDATE ready после PUT | API вернула 503; uploading и объект существуют. Немедленная сверка за 0.027833 s ничего не удалила |
| Старый orphan + незавершённый upload + missing используемого файла | Dry-run нашёл delete_orphan, delete_stale_upload, mark_missing; applied=false для всех, данные не изменены |
| Apply того же плана | Те же три действия применены; ссылки и метаданные missing сохранены |
| Свежие, используемые, активная lease и удерживаемый lock | Защитные проверки пройдены, вложения не удалены |
| Возврат оригинала оператором | После возврата fixture выполнен GET 111817 bytes и проверен SHA-256; restore_ready |
| Повторный apply | actions пуст; новых исправлений нет, история и исходные bytes доступны |
| Проход | Время, ms | Действий |
| --- | --- | --- |
| dry_run | 47.514 | 3 |
| apply | 53.742 | 3 |
| restore | 72.341 | 1 |
| repeat | 37.725 | 0 |
Ошибки внедрены на обеих сторонах реальными отказами IAM/SQL, а не подменой
ответов клиента. Исправление безопасно в проверенных состояниях: свежие и
используемые файлы не удаляются, пропавшие bytes не подменяются выдуманным файлом.
В тесте отдельно подтверждён возврат оригинала оператором и проверка его checksum.
### Результаты десяти одновременных загрузок
Два размера PNG, по три серии из 10 одновременно стартующих клиентов.
Каждая серия достигла обеих API, успешно загрузила 10 файлов и скачала их
вторым участником с проверкой исходных bytes и SHA-256. Итого 60/60, ошибок 0.
API пересоздавались перед каждой серией; лимиты каждой реплики — 1 CPU/512 MiB.
| PNG | Размер, bytes | Повтор | Burst RPS | p95 upload, ms | Max RSS одной API, MiB | Max cgroup peak одной API, MiB |
| --- | --- | --- | --- | --- | --- | --- |
| 256×256 | 197173 | 1 | 50.001 | 199.251 | 129.832 | 124.703 |
| 256×256 | 197173 | 2 | 69.096 | 143.348 | 111.172 | 116.461 |
| 256×256 | 197173 | 3 | 66.421 | 149.041 | 111.160 | 106.207 |
| 1024×1024 | 3151221 | 1 | 24.810 | 402.166 | 127.586 | 122.039 |
| 1024×1024 | 3151221 | 2 | 23.200 | 429.619 | 125.387 | 131.141 |
| 1024×1024 | 3151221 | 3 | 22.433 | 444.786 | 125.195 | 129.051 |
Медианы по трём сериям: для 256×256 — 66.421 burst RPS и p95 149.041 ms;
для 1024×1024 — 23.200 burst RPS и p95 429.619 ms. Здесь медиана p95 —
медиана трёх отдельных p95, а не общий p95 всех 30 запросов.
Максимальный process RSS среди всех серий — 129.832 MiB, максимальный cgroup
peak — 131.141 MiB, заметно ниже лимита 512 MiB на API. Это максимум одной
реплики, а не одновременная сумма памяти двух контейнеров. Cgroup peak измерен
через cgroup v1 `memory.max_usage_in_bytes`; RSS получен через `ru_maxrss`.
Пик включает запуск процесса, регистрацию и вход перед upload, поэтому более
крупный файл не обязан дать больший абсолютный RSS. Максимальный прирост
RSS high-water между снимками before/after составил 4.797 MiB для 256×256
и 18.668 MiB для 1024×1024. Это прирост счётчика максимума, не точное выделение
памяти только под содержимое файла. Burst RPS описывает короткую серию из 10
запросов и не заменяет продолжительный throughput-тест.
![Пик памяти API при десяти одновременных загрузках](evidence/upload_memory.png)
### Стоимость обхода
Отдельный контейнер роли reconciler, лимиты 0.5 CPU/256 MiB. На каждом размере
выполнен прогрев и три измеренных dry-run. Источник: `evidence/reconcile_cost.json`.
| Объектов и записей PG | Три прохода, s | Медиана, s | LIST / HEAD за проход | GET содержимого / bytes | Max RSS процесса, MiB |
| --- | --- | --- | --- | --- | --- |
| 100 | 0.692223, 0.696161, 0.808335 | 0.696161 | 1 / 100 | 0 / 0 | 68.250 |
| 500 | 3.238328, 3.476147, 3.516724 | 3.476147 | 1 / 500 | 0 / 0 | 69.797 |
| 1000 | 6.510459, 6.586175, 6.482201 | 6.510459 | 1 / 1000 | 0 / 0 | 71.859 |
На каждом проходе просмотрены ровно N объектов и N метаданных, исправлений 0.
Содержимое здоровых файлов не скачивалось: 0 GET и 0 bytes означают именно
содержимое объектов, а не отсутствие сетевого трафика LIST/HEAD и SQL.
При росте количества объектов в 10 раз медианное время увеличилось примерно
в 9.35 раза: 0.696161 → 6.510459 s. На измеренном диапазоне стоимость близка
к линейной: один LIST и N HEAD, плюс запросы/блокировки PostgreSQL. Лишнее
чтение оригиналов не требуется для обычного прохода; полный SHA-256 проверяется
при восстановлении missing или несогласованности HEAD.
RSS здесь — накопительный максимум одного процесса, который последовательно
готовил и обходил все три набора. Поэтому значения памяти по размерам не являются
независимыми измерениями чистой памяти одного прохода. Максимум 71.859 MiB
не достиг лимита reconciler 256 MiB. На большие объёмы эти числа не экстраполирую.
![Стоимость обхода двух хранилищ](evidence/reconcile_cost.png)
Вывод: на этом стенде гипотеза подтверждена для воспроизведённых состояний.
Reconciler обнаружил оба вида рассогласования, безопасно выполнил компенсацию,
сохранил свежие/используемые вложения; повторная сверка не внесла новых изменений.
Стоимость проверенного обхода растёт вместе с числом объектов; выполнены
отдельные измерения RAM для 10 конкурентных upload.
## Ограничения и источники
Один host, PG, LB и S3 остаются SPOF. Две API не защищают от потери S3-диска.
Повреждённые/отсутствующие оригиналы требуют восстановления из внешней копии.
Полный проход хранит inventory/IDs в памяти, его стоимость растёт с числом объектов;
для больших объёмов нужны paging/checkpoints и отдельный бюджет обслуживания.
Обычный ready проверяется по HEAD/метаданным; полный checksum bytes выполняется
при восстановлении missing или несогласованности HEAD. Это не постоянный full scrub.
Истёкшие сессии/ключи и неиспользуемые ready-вложения автоматически не удаляются.
Grace может задержать очистку; lease не является универсальным distributed transaction.
Измерения выполняются на одном компьютере; мониторинг/контрольные запросы также
потребляют его ресурсы. RAM-серия охватывает файлы 197173 и 3151221 bytes;
она не измеряет пик для каждого допустимого файла до 10 MiB/40 млн пикселей.
Защита предельного размера и пикселей отдельно проверена отрицательными тестами. Для длинной нагрузки результаты нужно проверять отдельно.
HTTP допускается только в локальном изолированном стенде. Прямой presigned upload
и pipeline thumbnail не реализованы, поскольку выбран трек 2 lab4.
Источники: README/COURSE/OpenAPI lab4 и собственное решение lab3;
[MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-04-22T22-12-26Z),
[mc release](https://github.com/minio/mc/releases/tag/RELEASE.2025-04-16T18-13-26Z),
[S3 PutObject](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object.html),
[Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html),
[PostgreSQL locks](https://www.postgresql.org/docs/16/explicit-locking.html#ADVISORY-LOCKS).
+143
View File
@@ -0,0 +1,143 @@
services:
db:
image: postgres:16.9-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets: [db_password]
volumes: [postgres_data:/var/lib/postgresql/data]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 3s
retries: 30
cache:
image: redis:7.4.3-bookworm
command: [redis-server, --appendonly, 'yes', --appendfsync, everysec]
volumes: [redis_data:/data]
cpus: 0.5
mem_limit: 128m
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 3s
retries: 30
migrate:
build: .
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
cpus: 0.5
mem_limit: 128m
depends_on:
db: {condition: service_healthy}
healthcheck: {disable: true}
s3:
image: mtusi2026lab4-minio:2025-04-22
build:
context: .
dockerfile: infra/s3.Dockerfile
args:
COMPONENT: minio
RELEASE: RELEASE.2025-04-22T22-12-26Z
SHA256: 53e2a2cb16c5366ea6fbbc479c19ddb4c6a0948273e752f740fb1fbf27bb817c
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
MINIO_BROWSER: 'off'
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://localhost:9000/minio/health/live",timeout=2)']
interval: 2s
timeout: 3s
retries: 30
s3-bootstrap:
image: mtusi2026lab4-mc:2025-04-16
build:
context: .
dockerfile: infra/s3.Dockerfile
args:
COMPONENT: mc
RELEASE: RELEASE.2025-04-16T18-13-26Z
SHA256: ac90da87a35641be5a0ac75d49de5161ddb47d629b5ba01261b0ae9e00aea15f
entrypoint: [/bin/sh, /bootstrap.sh]
volumes: ['./scripts/bootstrap_s3.sh:/bootstrap.sh:ro', './infra:/policies:ro']
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
environment:
S3_ENDPOINT: http://s3:9000
depends_on:
s3: {condition: service_healthy}
cpus: 0.5
mem_limit: 128m
api:
build: .
scale: 2
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://cache:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX: ${DB_POOL_MAX:-8}
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_REGION: us-east-1
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: '10485760'
MAX_IMAGE_PIXELS: '40000000'
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
dns_opt: ['timeout:1', 'attempts:1']
cpus: ${API_CPU:-1.0}
mem_limit: ${API_MEMORY:-512m}
stop_grace_period: 10s
depends_on:
migrate: {condition: service_completed_successfully}
cache: {condition: service_healthy}
s3-bootstrap: {condition: service_completed_successfully}
healthcheck:
test: [CMD, python, /app/scripts/healthcheck.py]
interval: 2s
timeout: 4s
retries: 30
reconciler:
build: .
command: [python, -m, src.reconcile, --apply, --loop, --interval, '60', --grace-seconds, '600']
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
cpus: 0.5
mem_limit: 256m
depends_on:
migrate: {condition: service_completed_successfully}
s3-bootstrap: {condition: service_completed_successfully}
healthcheck: {disable: true}
lb:
image: haproxy:3.0.11-bookworm
ports: ['127.0.0.1:${APP_PORT:-8080}:8080']
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
cpus: 0.5
mem_limit: 128m
depends_on:
api: {condition: service_healthy}
volumes:
postgres_data:
redis_data:
s3_data:
secrets:
db_password: {file: .secrets/db_password}
database_url: {file: .secrets/database_url}
s3_root_user: {file: .secrets/s3_root_user}
s3_root_password: {file: .secrets/s3_root_password}
s3_access_key_id: {file: .secrets/s3_access_key_id}
s3_secret_access_key: {file: .secrets/s3_secret_access_key}
+247
View File
@@ -0,0 +1,247 @@
{
"status": "PASS",
"uploader": "48fedaf342e1:632ba2ab",
"downloader": "9425d69912fc:665365c7",
"recovery_seconds": 1.3700443690031534,
"failed_attempts": 0,
"original_size": 111817,
"checksum_verified": true,
"stop": {
"target": "48fedaf342e1:632ba2ab",
"stopped_container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"before": [
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:24:22.143921145Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:25:41.660780116Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:25:39.453529659Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:25:39.454265647Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
],
"after": [
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:24:22.143921145Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": false,
"started_at": "2026-10-16T19:25:41.660780116Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:25:39.453529659Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:25:39.454265647Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
}
}
+118
View File
@@ -0,0 +1,118 @@
[
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:24:22.143921145Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:24:21.834698967Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:22:16.896719789Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:22:16.884675125Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
+118
View File
@@ -0,0 +1,118 @@
[
{
"id": "3ce028ba0a279a365934fdddf6924260b13d8f3c3d15be9fb2b328c548fc03df",
"name": "mtusi2026lab4-api-1",
"hostname": "3ce028ba0a27",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:22:19.108920225Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "fd8770db9dfad55cb3f54b902061ea0d0e146399c2e02ccd74e1a7b87f7d0883",
"name": "mtusi2026lab4-api-2",
"hostname": "fd8770db9dfa",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:22:19.397716604Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:16:39.322959984Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:16:39.315215096Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a922458bb583dd8f163ddab3a44bd8d7a80784955bb1dd84d3971a02c0c933d9",
"name": "mtusi2026lab4-lb-1",
"hostname": "a922458bb583",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:22:22.158189871Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:22:16.896719789Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "7a63ecf93930384af9215ceca99f3634ecaf661258aac3410af9dd8ecfe49516",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "7a63ecf93930",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:22:19.108876624Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:16:39.315456952Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:22:16.884675125Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
+118
View File
@@ -0,0 +1,118 @@
[
{
"id": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975",
"name": "mtusi2026lab4-api-3",
"hostname": "d66f39ddc1de",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:30:35.417805574Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa",
"name": "mtusi2026lab4-api-4",
"hostname": "7d0a6457eb21",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:30:35.105182778Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:27:50.22191294Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:27:57.6646694Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:31:59.037730504Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:32:00.920195433Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:28:15.845174201Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:31:59.038314101Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
+21
View File
@@ -0,0 +1,21 @@
{
"redis": {
"status": "PASS",
"elapsed_seconds": 7.07943196000997
},
"postgres": {
"status": "PASS",
"readiness_and_request_status": [
503,
503
]
},
"s3": {
"status": "PASS",
"readiness_and_download_status": [
503,
503
]
},
"recovery": "old session and confirmed message preserved"
}
+123
View File
@@ -0,0 +1,123 @@
{
"docker": "26.1.1",
"compose": "2.27.0-desktop.2",
"containers": [
{
"id": "355cebdc2affd62651fc7da80af09cae310fe235f8bcf83165a5d559b9f752c6",
"name": "mtusi2026lab4-api-3",
"hostname": "355cebdc2aff",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:28:38.177524516Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "90f387d120f8481a19384b5b630eef38ae04062b1064cf57ec330e7f847d8e50",
"name": "mtusi2026lab4-api-4",
"hostname": "90f387d120f8",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:28:37.808617738Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:27:50.22191294Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:27:57.6646694Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:28:19.512821485Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:28:21.5678737Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:28:15.845174201Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:28:19.512884457Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
],
"client_cpus": 16
}
+37
View File
@@ -0,0 +1,37 @@
{
"reconciler_cost": [
{
"objects": 100,
"repetitions": 3,
"median_seconds": 0.696161,
"heads_per_scan": [
100,
100,
100
]
},
{
"objects": 500,
"repetitions": 3,
"median_seconds": 3.476147,
"heads_per_scan": [
500,
500,
500
]
},
{
"objects": 1000,
"repetitions": 3,
"median_seconds": 6.510459,
"heads_per_scan": [
1000,
1000,
1000
]
}
],
"concurrent_uploads": 10,
"verified_originals": 60,
"status": "PASS"
}
File diff suppressed because it is too large Load Diff
+9
View File
@@ -0,0 +1,9 @@
{
"status": "PASS",
"concurrent_requests": 32,
"observed_instances": 2,
"messages_after_race": 1,
"conflict": 409,
"invalid_key": 400,
"logout": 401
}
+238
View File
@@ -0,0 +1,238 @@
{
"target": "9425d69912fc:665365c7",
"stopped_container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"before": [
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:24:22.143921145Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:26:35.987840207Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:26:34.060806615Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:26:34.060879403Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
],
"after": [
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": false,
"started_at": "2026-10-16T19:24:22.143921145Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:26:35.987840207Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:26:34.060806615Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:26:34.060879403Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
}
+2
View File
@@ -0,0 +1,2 @@
BASE_URL="http://localhost:8080" python3 student-tests/availability.py
PASS: both API failures and rolling replacement, confirmations preserved
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
BASE_URL="http://localhost:8080" python3 student-tests/dependencies.py
PASS: Redis fallback, PostgreSQL/S3 unready, old session and original preserved
+31
View File
@@ -0,0 +1,31 @@
CPU=16 RAM_BYTES=8187711488
Client: Docker Engine - Community
Version: 28.0.1
API version: 1.45 (downgraded from 1.48)
Go version: go1.23.6
Git commit: 068a01e
Built: Wed Feb 26 10:41:08 2025
OS/Arch: linux/amd64
Context: default
Server: Docker Desktop
Engine:
Version: 26.1.1
API version: 1.45 (minimum version 1.24)
Go version: go1.21.9
Git commit: ac2de55
Built: Tue Apr 30 11:48:28 2024
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.6.31
GitCommit: e377cd56a71523140ca6ae87e30244719194a521
runc:
Version: 1.1.12
GitCommit: v1.1.12-0-g51d5e94
docker-init:
Version: 0.19.0
GitCommit: de40ad0
Docker Compose version v2.27.0-desktop.2
Python 3.10.12
Linux LAPTOP-66HKAM8B 5.15.146.1-microsoft-standard-WSL2 #1 SMP Thu Jan 11 04:09:03 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux
+11
View File
@@ -0,0 +1,11 @@
BASE_URL="http://localhost:8080" python3 scripts/experiment.py
PASS: private bucket, limited credentials, closed console
PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection
Upload memory: 256x256, repeat 1/3
Upload memory: 256x256, repeat 2/3
Upload memory: 256x256, repeat 3/3
Upload memory: 1024x1024, repeat 1/3
Upload memory: 1024x1024, repeat 2/3
Upload memory: 1024x1024, repeat 3/3
Reconciler cost: datasets 100, 500, 1000, three scans each
Experiment completed; results saved in evidence/
+20
View File
@@ -0,0 +1,20 @@
bash scripts/test_expiry.sh
Container mtusi2026lab4-api-4 Recreate
Container mtusi2026lab4-api-3 Recreate
Container mtusi2026lab4-api-4 Recreated
Container mtusi2026lab4-api-3 Recreated
Container mtusi2026lab4-api-3 Starting
Container mtusi2026lab4-api-3 Started
Container mtusi2026lab4-api-4 Starting
Container mtusi2026lab4-api-4 Started
API ready: 5e906f839a98:4bde0bc0, 8bb802a890f0:0923a18a
PASS: short TTL expiry
Container mtusi2026lab4-api-3 Recreate
Container mtusi2026lab4-api-4 Recreate
Container mtusi2026lab4-api-4 Recreated
Container mtusi2026lab4-api-3 Recreated
Container mtusi2026lab4-api-4 Starting
Container mtusi2026lab4-api-4 Started
Container mtusi2026lab4-api-3 Starting
Container mtusi2026lab4-api-3 Started
API ready: 355cebdc2aff:2f7b7cb5, 90f387d120f8:d12f4b2b
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 3, "first_recovery_seconds": 1.524}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
BASE_URL="http://localhost:8080" python3 student-tests/file_failover.py
PASS: upload API stopped, original downloaded by other API with existing token
+5
View File
@@ -0,0 +1,5 @@
python3 student-tests/attachments.py --base-url "http://localhost:8080"
PASS: PNG/JPEG, bytes/pixels limits, filename/MIME, ACL, attachment message replay
BASE_URL="http://localhost:8080" python3 scripts/experiment.py --mode privacy
PASS: private bucket, limited credentials, closed console
Experiment completed; results saved in evidence/
+2
View File
@@ -0,0 +1,2 @@
BASE_URL="http://localhost:8080" python3 scripts/stack.py wait --count 2
API ready: 7d0a6457eb21:3ee5bf95, d66f39ddc1de:9d084897
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
BASE_URL="http://localhost:8080" python3 scripts/stack.py wait --count 2
API ready: 3ce028ba0a27:df4472a0, fd8770db9dfa:6ce14f15
+10
View File
@@ -0,0 +1,10 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab4-api-1 mtusi2026lab4-api "sh -c 'exec uvicorn…" api 47 seconds ago Up 33 seconds (healthy) 8080/tcp
mtusi2026lab4-api-2 mtusi2026lab4-api "sh -c 'exec uvicorn…" api 47 seconds ago Up 33 seconds (healthy) 8080/tcp
mtusi2026lab4-cache-1 redis:7.4.3-bookworm "docker-entrypoint.s…" cache 6 minutes ago Up 6 minutes (healthy) 6379/tcp
mtusi2026lab4-db-1 postgres:16.9-bookworm "docker-entrypoint.s…" db 6 minutes ago Up 6 minutes (healthy) 5432/tcp
mtusi2026lab4-lb-1 haproxy:3.0.11-bookworm "docker-entrypoint.s…" lb 46 seconds ago Up 30 seconds 127.0.0.1:8080->8080/tcp
mtusi2026lab4-migrate-1 mtusi2026lab4-migrate "python -m src.migra…" migrate 47 seconds ago Exited (0) 34 seconds ago
mtusi2026lab4-reconciler-1 mtusi2026lab4-reconciler "python -m src.recon…" reconciler 47 seconds ago Up 33 seconds 8080/tcp
mtusi2026lab4-s3-1 mtusi2026lab4-minio:2025-04-22 "/bin/sh /entrypoint…" s3 6 minutes ago Up 6 minutes (healthy)
mtusi2026lab4-s3-bootstrap-1 mtusi2026lab4-mc:2025-04-16 "/bin/sh /bootstrap.…" s3-bootstrap 6 minutes ago Exited (0) 34 seconds ago
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+260
View File
@@ -0,0 +1,260 @@
bash scripts/start.sh
#0 building with "default" instance using docker driver
#1 [s3 internal] load build definition from s3.Dockerfile
#1 transferring dockerfile: 922B done
#1 DONE 0.0s
#2 [migrate internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.04kB done
#2 DONE 0.0s
#3 [s3] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0
#3 ...
#4 [migrate internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm
#4 DONE 1.0s
#3 [s3] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0
#3 DONE 1.0s
#5 [s3] docker-image://docker.io/docker/dockerfile:1.7.0@sha256:dbbd5e059e8a07ff7ea6233b213b36aa516b4c53c645f1817a4dd18b83cbea56
#5 CACHED
#6 [migrate internal] load .dockerignore
#6 transferring context: 205B done
#6 DONE 0.0s
#7 [migrate 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
#7 DONE 0.0s
#8 [migrate internal] load build context
#8 transferring context: 2.44kB done
#8 DONE 0.0s
#9 [migrate 2/12] WORKDIR /app
#9 CACHED
#10 [migrate 3/12] COPY requirements.txt .
#10 CACHED
#11 [migrate 4/12] RUN pip install --no-cache-dir --requirement requirements.txt && groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app
#11 CACHED
#12 [migrate 5/12] COPY --chown=app:app src ./src
#12 CACHED
#13 [migrate 6/12] COPY --chown=app:app migrations ./migrations
#13 CACHED
#14 [migrate 7/12] COPY --chown=app:app scripts ./scripts
#14 DONE 0.1s
#15 [migrate 8/12] COPY --chown=app:app student-tests ./student-tests
#15 DONE 0.1s
#16 [migrate 9/12] COPY --chown=app:app tests ./tests
#16 DONE 0.1s
#4 [s3 internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm
#4 ...
#17 [migrate 10/12] COPY --chown=app:app contracts ./contracts
#17 DONE 0.1s
#18 [migrate 11/12] COPY --chown=app:app lab.json ./lab.json
#18 DONE 0.0s
#7 [s3 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
#7 DONE 0.0s
#4 [s3 internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm
#4 DONE 1.2s
#19 [s3 internal] load .dockerignore
#19 transferring context: 205B done
#19 DONE 0.0s
#20 [s3 internal] load build context
#20 transferring context: 241B done
#20 DONE 0.0s
#21 [migrate 12/12] RUN mkdir -p evidence && chown app:app evidence
#21 ...
#22 [s3 2/3] RUN python - <<'PY'
#22 CACHED
#23 [s3 3/3] COPY infra/minio-entrypoint.sh /entrypoint.sh
#23 CACHED
#24 [s3] exporting to image
#24 exporting layers done
#24 writing image sha256:6737b5a1fceb3f5d6035e6ea1ea9c32f3dffbfdb7912203727f68d41bfe41ba4 done
#24 naming to docker.io/library/mtusi2026lab4-minio:2025-04-22 done
#24 DONE 0.0s
#21 [migrate 12/12] RUN mkdir -p evidence && chown app:app evidence
#21 DONE 0.4s
#25 [s3-bootstrap internal] load build definition from s3.Dockerfile
#25 transferring dockerfile: 922B done
#25 DONE 0.0s
#26 [migrate] exporting to image
#26 exporting layers 0.1s done
#26 writing image sha256:6203bb8001386f7e6b955bbcd5e3770c21a56231807036e5aeea33c25510bc97 done
#26 naming to docker.io/library/mtusi2026lab4-migrate done
#26 DONE 0.1s
#3 [s3-bootstrap] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0
#3 DONE 1.3s
#5 [s3-bootstrap] docker-image://docker.io/docker/dockerfile:1.7.0@sha256:dbbd5e059e8a07ff7ea6233b213b36aa516b4c53c645f1817a4dd18b83cbea56
#5 CACHED
#4 [s3-bootstrap internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm
#4 DONE 1.5s
#27 [s3-bootstrap internal] load .dockerignore
#27 transferring context: 205B done
#27 DONE 0.0s
#7 [s3-bootstrap 1/3] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
#7 DONE 0.0s
#28 [s3-bootstrap internal] load build context
#28 transferring context: 241B done
#28 DONE 0.0s
#29 [s3-bootstrap 2/3] RUN python - <<'PY'
#29 CACHED
#30 [s3-bootstrap 3/3] COPY infra/minio-entrypoint.sh /entrypoint.sh
#30 CACHED
#31 [s3-bootstrap] exporting to image
#31 exporting layers done
#31 writing image sha256:0f35e9da12772a90417ce81511aab2c03dcc88a5134dbb734e13e12a2aa4afa4 done
#31 naming to docker.io/library/mtusi2026lab4-mc:2025-04-16 done
#31 DONE 0.0s
#32 [reconciler internal] load build definition from Dockerfile
#32 transferring dockerfile: 1.04kB done
#32 DONE 0.0s
#33 [api internal] load build definition from Dockerfile
#33 transferring dockerfile: 1.04kB done
#33 DONE 0.0s
#4 [api internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm
#4 DONE 1.8s
#34 [api internal] load .dockerignore
#34 transferring context: 205B done
#34 DONE 0.0s
#35 [reconciler internal] load .dockerignore
#35 transferring context: 205B done
#35 DONE 0.0s
#7 [reconciler 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
#7 DONE 0.0s
#36 [api internal] load build context
#36 transferring context: 427.92kB 0.0s done
#36 DONE 0.0s
#37 [reconciler internal] load build context
#37 transferring context: 427.92kB 0.0s done
#37 DONE 0.0s
#9 [api 2/12] WORKDIR /app
#9 CACHED
#38 [api 6/12] COPY --chown=app:app migrations ./migrations
#38 CACHED
#39 [api 9/12] COPY --chown=app:app tests ./tests
#39 CACHED
#40 [api 3/12] COPY requirements.txt .
#40 CACHED
#41 [api 11/12] COPY --chown=app:app lab.json ./lab.json
#41 CACHED
#42 [api 8/12] COPY --chown=app:app student-tests ./student-tests
#42 CACHED
#43 [api 10/12] COPY --chown=app:app contracts ./contracts
#43 CACHED
#44 [api 5/12] COPY --chown=app:app src ./src
#44 CACHED
#45 [api 7/12] COPY --chown=app:app scripts ./scripts
#45 CACHED
#46 [api 4/12] RUN pip install --no-cache-dir --requirement requirements.txt && groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app
#46 CACHED
#47 [reconciler 12/12] RUN mkdir -p evidence && chown app:app evidence
#47 CACHED
#48 [api] exporting to image
#48 exporting layers done
#48 writing image sha256:ee25a4e40dec7eb92c2d6ca1b5c0cbc473cf40883f6e6c24bc59346bbed17911 done
#48 naming to docker.io/library/mtusi2026lab4-api 0.0s done
#48 DONE 0.0s
#49 [reconciler] exporting to image
#49 exporting layers done
#49 writing image sha256:fe4ad675e910216faecbc2b15a9850bd8f1dc78b7f78b987e099614ed137bb62 done
#49 naming to docker.io/library/mtusi2026lab4-reconciler done
#49 DONE 0.0s
Container mtusi2026lab4-s3-1 Running
Container mtusi2026lab4-cache-1 Running
Container mtusi2026lab4-db-1 Running
Container mtusi2026lab4-s3-bootstrap-1 Created
Container mtusi2026lab4-migrate-1 Recreate
Container mtusi2026lab4-migrate-1 Recreated
Container mtusi2026lab4-api-1 Recreate
Container mtusi2026lab4-reconciler-1 Recreate
Container mtusi2026lab4-api-2 Recreate
Container mtusi2026lab4-api-2 Recreated
Container mtusi2026lab4-api-1 Recreated
Container mtusi2026lab4-lb-1 Recreate
Container mtusi2026lab4-lb-1 Recreated
Container mtusi2026lab4-reconciler-1 Recreated
Container mtusi2026lab4-s3-1 Waiting
Container mtusi2026lab4-db-1 Waiting
Container mtusi2026lab4-s3-1 Healthy
Container mtusi2026lab4-s3-bootstrap-1 Starting
Container mtusi2026lab4-db-1 Healthy
Container mtusi2026lab4-migrate-1 Starting
Container mtusi2026lab4-s3-bootstrap-1 Started
Container mtusi2026lab4-migrate-1 Started
Container mtusi2026lab4-migrate-1 Waiting
Container mtusi2026lab4-s3-bootstrap-1 Waiting
Container mtusi2026lab4-s3-bootstrap-1 Waiting
Container mtusi2026lab4-migrate-1 Waiting
Container mtusi2026lab4-cache-1 Waiting
Container mtusi2026lab4-cache-1 Healthy
Container mtusi2026lab4-migrate-1 Exited
Container mtusi2026lab4-migrate-1 Exited
Container mtusi2026lab4-s3-bootstrap-1 Exited
Container mtusi2026lab4-reconciler-1 Starting
Container mtusi2026lab4-s3-bootstrap-1 Exited
Container mtusi2026lab4-api-1 Starting
Container mtusi2026lab4-reconciler-1 Started
Container mtusi2026lab4-api-1 Started
Container mtusi2026lab4-api-2 Starting
Container mtusi2026lab4-api-2 Started
Container mtusi2026lab4-api-1 Waiting
Container mtusi2026lab4-api-2 Waiting
Container mtusi2026lab4-api-1 Healthy
Container mtusi2026lab4-api-2 Healthy
Container mtusi2026lab4-lb-1 Starting
Container mtusi2026lab4-lb-1 Started
API ready: 3ce028ba0a27:df4472a0, fd8770db9dfa:6ce14f15
+10
View File
@@ -0,0 +1,10 @@
python3 student-tests/test_api.py --base-url "http://localhost:8080"
PASS: malformed JSON/media type, duplicate username, blank message, outsider access, pagination
PASS: 10 clients created exactly 100 unique messages
python3 student-tests/checks.py --base-url "http://localhost:8080"
PASS: registration race, invalid input, cursor scope/types on two API, read-after-write, logout
python3 student-tests/idempotency.py --base-url "http://localhost:8080"
PASS: shared idempotency, scope, conflict and auth on replay
docker exec "$(docker compose ps --status running -q api | head -n 1)" python student-tests/database.py
PASS: rollback of chat + members; migration reapplied twice
PASS: persisted key expiry, atomic message/result rollback
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+8
View File
@@ -0,0 +1,8 @@
{
"status": "PASS",
"anonymous_get": 403,
"bucket_list": "filtered",
"create_bucket": "AccessDenied",
"console": "not reachable",
"attachment_id": "0cecaeeb-42a0-4f63-9281-51ac7df57414"
}
+194
View File
@@ -0,0 +1,194 @@
{
"status": "PASS",
"scans": [
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 0.692223,
"counters": {
"objects_scanned": 100,
"metadata_scanned": 100,
"list_requests": 1,
"head_requests": 100,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 70963200,
"dataset_added": 100,
"repetition": 1
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 0.696161,
"counters": {
"objects_scanned": 100,
"metadata_scanned": 100,
"list_requests": 1,
"head_requests": 100,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 71294976,
"dataset_added": 100,
"repetition": 2
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 0.808335,
"counters": {
"objects_scanned": 100,
"metadata_scanned": 100,
"list_requests": 1,
"head_requests": 100,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 71565312,
"dataset_added": 100,
"repetition": 3
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 3.238328,
"counters": {
"objects_scanned": 500,
"metadata_scanned": 500,
"list_requests": 1,
"head_requests": 500,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 72646656,
"dataset_added": 500,
"repetition": 1
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 3.476147,
"counters": {
"objects_scanned": 500,
"metadata_scanned": 500,
"list_requests": 1,
"head_requests": 500,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 72916992,
"dataset_added": 500,
"repetition": 2
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 3.516724,
"counters": {
"objects_scanned": 500,
"metadata_scanned": 500,
"list_requests": 1,
"head_requests": 500,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 73187328,
"dataset_added": 500,
"repetition": 3
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 6.510459,
"counters": {
"objects_scanned": 1000,
"metadata_scanned": 1000,
"list_requests": 1,
"head_requests": 1000,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 74539008,
"dataset_added": 1000,
"repetition": 1
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 6.586175,
"counters": {
"objects_scanned": 1000,
"metadata_scanned": 1000,
"list_requests": 1,
"head_requests": 1000,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 75079680,
"dataset_added": 1000,
"repetition": 2
},
{
"mode": "dry-run",
"grace_seconds": 600,
"chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed",
"elapsed_seconds": 6.482201,
"counters": {
"objects_scanned": 1000,
"metadata_scanned": 1000,
"list_requests": 1,
"head_requests": 1000,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [],
"process_peak_rss_bytes": 75350016,
"dataset_added": 1000,
"repetition": 3
}
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

+221
View File
@@ -0,0 +1,221 @@
{
"s3_failure_compensation": {
"dry_run": {
"mode": "dry-run",
"grace_seconds": 20,
"chat_scope": "4849767c-8c80-4469-ba2f-7b439782abc4",
"elapsed_seconds": 0.035159,
"counters": {
"objects_scanned": 0,
"metadata_scanned": 1,
"list_requests": 1,
"head_requests": 1,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [
{
"kind": "delete_stale_upload",
"attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c",
"object_key": "originals/4849767c-8c80-4469-ba2f-7b439782abc4/d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c",
"applied": false,
"reason": "lease_expired"
}
],
"process_peak_rss_bytes": 70615040
},
"apply": {
"mode": "apply",
"grace_seconds": 20,
"chat_scope": "4849767c-8c80-4469-ba2f-7b439782abc4",
"elapsed_seconds": 0.02742,
"counters": {
"objects_scanned": 0,
"metadata_scanned": 1,
"list_requests": 1,
"head_requests": 1,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 0,
"protected_active": 0,
"protected_used": 0
},
"actions": [
{
"kind": "delete_stale_upload",
"attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c",
"object_key": "originals/4849767c-8c80-4469-ba2f-7b439782abc4/d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c",
"applied": true,
"reason": "lease_expired"
}
],
"process_peak_rss_bytes": 70946816
},
"metadata_removed": true
},
"after_put_db_failure": {
"http_status": 503,
"metadata": "uploading",
"object": "present",
"immediate_cleanup": {
"mode": "apply",
"grace_seconds": 20,
"chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44",
"elapsed_seconds": 0.027833,
"counters": {
"objects_scanned": 2,
"metadata_scanned": 2,
"list_requests": 1,
"head_requests": 1,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 1,
"protected_active": 0,
"protected_used": 1
},
"actions": [],
"process_peak_rss_bytes": 71266304
}
},
"status": "PASS",
"dry_run": {
"mode": "dry-run",
"grace_seconds": 20,
"chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44",
"elapsed_seconds": 0.047514,
"counters": {
"objects_scanned": 7,
"metadata_scanned": 6,
"list_requests": 1,
"head_requests": 5,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 2,
"protected_active": 3,
"protected_used": 1
},
"actions": [
{
"kind": "delete_orphan",
"attachment_id": "4baae95b-26db-42df-9dfa-263e7b8c10a1",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/4baae95b-26db-42df-9dfa-263e7b8c10a1",
"applied": false
},
{
"kind": "delete_stale_upload",
"attachment_id": "633b27f4-a695-4381-af03-8d6f53d0b055",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/633b27f4-a695-4381-af03-8d6f53d0b055",
"applied": false,
"reason": "lease_expired"
},
{
"kind": "mark_missing",
"attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"applied": false,
"reason": "object_missing"
}
],
"process_peak_rss_bytes": 71266304
},
"apply": {
"mode": "apply",
"grace_seconds": 20,
"chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44",
"elapsed_seconds": 0.053742,
"counters": {
"objects_scanned": 7,
"metadata_scanned": 6,
"list_requests": 1,
"head_requests": 5,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 2,
"protected_active": 3,
"protected_used": 1
},
"actions": [
{
"kind": "delete_orphan",
"attachment_id": "4baae95b-26db-42df-9dfa-263e7b8c10a1",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/4baae95b-26db-42df-9dfa-263e7b8c10a1",
"applied": true
},
{
"kind": "delete_stale_upload",
"attachment_id": "633b27f4-a695-4381-af03-8d6f53d0b055",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/633b27f4-a695-4381-af03-8d6f53d0b055",
"applied": true,
"reason": "lease_expired"
},
{
"kind": "mark_missing",
"attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"applied": true,
"reason": "object_missing"
}
],
"process_peak_rss_bytes": 71536640
},
"restore": {
"mode": "apply",
"grace_seconds": 20,
"chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44",
"elapsed_seconds": 0.072341,
"counters": {
"objects_scanned": 6,
"metadata_scanned": 5,
"list_requests": 1,
"head_requests": 3,
"get_requests": 1,
"bytes_read": 111817,
"protected_fresh": 2,
"protected_active": 3,
"protected_used": 1
},
"actions": [
{
"kind": "restore_ready",
"attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e",
"applied": true,
"reason": "verified_original_restored"
}
],
"process_peak_rss_bytes": 71536640
},
"repeat": {
"mode": "apply",
"grace_seconds": 20,
"chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44",
"elapsed_seconds": 0.037725,
"counters": {
"objects_scanned": 6,
"metadata_scanned": 5,
"list_requests": 1,
"head_requests": 3,
"get_requests": 0,
"bytes_read": 0,
"protected_fresh": 2,
"protected_active": 3,
"protected_used": 2
},
"actions": [],
"process_peak_rss_bytes": 71536640
},
"used_and_fresh_preserved": true,
"active_lease_and_lock_preserved": true,
"missing_original_restored_by_operator": true,
"s3_write_failure": {
"status": "PASS",
"http_status": 503,
"metadata": "uploading",
"object": "absent",
"chat_id": "4849767c-8c80-4469-ba2f-7b439782abc4",
"attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c"
}
}
File diff suppressed because it is too large Load Diff
+296
View File
@@ -0,0 +1,296 @@
{
"before": [
{
"id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63",
"name": "mtusi2026lab4-api-1",
"hostname": "9425d69912fc",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:26:57.313322452Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:26:35.987840207Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:26:55.368313114Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:26:55.368377904Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
],
"steps": [
[
{
"id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f",
"name": "mtusi2026lab4-api-2",
"hostname": "48fedaf342e1",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:26:35.987840207Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2",
"name": "mtusi2026lab4-api-3",
"hostname": "b89ca7b5c7e8",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:27:04.675168495Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
}
],
[
{
"id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2",
"name": "mtusi2026lab4-api-3",
"hostname": "b89ca7b5c7e8",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:27:04.675168495Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039",
"name": "mtusi2026lab4-api-4",
"hostname": "26996fc7b6ac",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:27:09.192493456Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
}
]
],
"after": [
{
"id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2",
"name": "mtusi2026lab4-api-3",
"hostname": "b89ca7b5c7e8",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:27:04.675168495Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039",
"name": "mtusi2026lab4-api-4",
"hostname": "26996fc7b6ac",
"service": "api",
"running": true,
"started_at": "2026-10-16T19:27:09.192493456Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2",
"name": "mtusi2026lab4-cache-1",
"hostname": "1f03107c8f59",
"service": "cache",
"running": true,
"started_at": "2026-10-16T19:24:14.51497638Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"6379/tcp": null
}
},
{
"id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973",
"name": "mtusi2026lab4-db-1",
"hostname": "11b3a463b436",
"service": "db",
"running": true,
"started_at": "2026-10-16T19:24:14.515575034Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {
"5432/tcp": null
}
},
{
"id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc",
"name": "mtusi2026lab4-lb-1",
"hostname": "a01916435f0d",
"service": "lb",
"running": true,
"started_at": "2026-10-16T19:24:25.265114396Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
}
},
{
"id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9",
"name": "mtusi2026lab4-migrate-1",
"hostname": "425a7c4ea232",
"service": "migrate",
"running": false,
"started_at": "2026-10-16T19:26:55.368313114Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
},
{
"id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc",
"name": "mtusi2026lab4-reconciler-1",
"hostname": "f223fdbf3dfa",
"service": "reconciler",
"running": true,
"started_at": "2026-10-16T19:24:26.040659771Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 268435456,
"published_ports": {
"8080/tcp": null
}
},
{
"id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045",
"name": "mtusi2026lab4-s3-1",
"hostname": "68e73eb491e5",
"service": "s3",
"running": true,
"started_at": "2026-10-16T19:24:14.517842259Z",
"cpu_limit": 1.0,
"memory_limit_bytes": 536870912,
"published_ports": {}
},
{
"id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c",
"name": "mtusi2026lab4-s3-bootstrap-1",
"hostname": "34e29dc315a1",
"service": "s3-bootstrap",
"running": false,
"started_at": "2026-10-16T19:26:55.368377904Z",
"cpu_limit": 0.5,
"memory_limit_bytes": 134217728,
"published_ports": {}
}
]
}
+8
View File
@@ -0,0 +1,8 @@
{
"status": "PASS",
"http_status": 503,
"metadata": "uploading",
"object": "absent",
"chat_id": "4849767c-8c80-4469-ba2f-7b439782abc4",
"attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c"
}
+7
View File
@@ -0,0 +1,7 @@
{
"status": "PASS",
"sent_bytes": 16,
"declared_bytes": 4096,
"metadata": 0,
"objects": 0
}
+443
View File
@@ -0,0 +1,443 @@
{
"runs": [
{
"dimension": 256,
"size_bytes": 197173,
"concurrent_uploads": 10,
"repetition": 1,
"elapsed_seconds": 0.2,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 50.00050475719273,
"successful_latency_ms": {
"p50": 177.823,
"p95": 199.251,
"p99": 199.251
},
"latency_samples_ms": [
199.251,
176.934,
178.783,
176.489,
179.393,
177.823,
195.595,
165.586,
180.372,
167.019
],
"before": [
{
"container": "81a289979a98a3cfe8ed140fe6e9d44dc9e4c1d97238731bf0e792ce6d8dab24",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 133656576,
"container_peak_bytes": 127946752,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "7de9c777c1bdd46d4a945010a99f20e9e57cc1bc0d77e575d7d2668f8ff51f8a",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 112111616,
"container_peak_bytes": 106348544,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "81a289979a98a3cfe8ed140fe6e9d44dc9e4c1d97238731bf0e792ce6d8dab24",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 136138752,
"container_peak_bytes": 130760704,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "7de9c777c1bdd46d4a945010a99f20e9e57cc1bc0d77e575d7d2668f8ff51f8a",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 116420608,
"container_peak_bytes": 111153152,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
},
{
"dimension": 256,
"size_bytes": 197173,
"concurrent_uploads": 10,
"repetition": 2,
"elapsed_seconds": 0.145,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 69.09566154263844,
"successful_latency_ms": {
"p50": 118.179,
"p95": 143.348,
"p99": 143.348
},
"latency_samples_ms": [
143.348,
114.799,
118.179,
115.119,
115.609,
124.187,
115.907,
126.252,
141.024,
119.354
],
"before": [
{
"container": "2ca6d753d6a7308991464a7b96b23507766831bbc80a10c716f419956b2fef9d",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 111525888,
"container_peak_bytes": 105918464,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "e1ccb5bd30f8e154ae47a83d163a92b29a3723440dac86dd1c3f01ba9f75df0f",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 114376704,
"container_peak_bytes": 108515328,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "2ca6d753d6a7308991464a7b96b23507766831bbc80a10c716f419956b2fef9d",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 116072448,
"container_peak_bytes": 110772224,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "e1ccb5bd30f8e154ae47a83d163a92b29a3723440dac86dd1c3f01ba9f75df0f",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 116572160,
"container_peak_bytes": 122118144,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
},
{
"dimension": 256,
"size_bytes": 197173,
"concurrent_uploads": 10,
"repetition": 3,
"elapsed_seconds": 0.151,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 66.42146959922258,
"successful_latency_ms": {
"p50": 124.273,
"p95": 149.041,
"p99": 149.041
},
"latency_samples_ms": [
125.702,
118.463,
121.036,
135.132,
142.124,
149.041,
87.277,
138.344,
86.5,
124.273
],
"before": [
{
"container": "e08871d00731b8a760906ea6fb8be77d81107a9e1dfbb3cd33ea760ace92c1de",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 111759360,
"container_peak_bytes": 105930752,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "328d37747428a59aae4130e2dafe03f39b5fff9dde561fe7f4751256a75073e0",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 111529984,
"container_peak_bytes": 105852928,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "e08871d00731b8a760906ea6fb8be77d81107a9e1dfbb3cd33ea760ace92c1de",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 116510720,
"container_peak_bytes": 111316992,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "328d37747428a59aae4130e2dafe03f39b5fff9dde561fe7f4751256a75073e0",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 116559872,
"container_peak_bytes": 111366144,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
},
{
"dimension": 1024,
"size_bytes": 3151221,
"concurrent_uploads": 10,
"repetition": 1,
"elapsed_seconds": 0.403,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 24.810339037866473,
"successful_latency_ms": {
"p50": 339.61,
"p95": 402.166,
"p99": 402.166
},
"latency_samples_ms": [
210.903,
265.17,
375.366,
402.166,
400.501,
266.088,
339.61,
374.81,
210.982,
342.662
],
"before": [
{
"container": "be5e8022f8422cd906a56e1bbaf9de4301a02e6485be30fe98217355b251257b",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 114208768,
"container_peak_bytes": 118149120,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "d75aff6f8b9488e9f7c2b33e0653c495efec77601af7589249fa0cb77ac88026",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 113872896,
"container_peak_bytes": 108003328,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "be5e8022f8422cd906a56e1bbaf9de4301a02e6485be30fe98217355b251257b",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 133783552,
"container_peak_bytes": 127967232,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "d75aff6f8b9488e9f7c2b33e0653c495efec77601af7589249fa0cb77ac88026",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 131272704,
"container_peak_bytes": 126140416,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
},
{
"dimension": 1024,
"size_bytes": 3151221,
"concurrent_uploads": 10,
"repetition": 2,
"elapsed_seconds": 0.431,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 23.199689911744827,
"successful_latency_ms": {
"p50": 319.572,
"p95": 429.619,
"p99": 429.619
},
"latency_samples_ms": [
319.572,
294.825,
374.821,
429.619,
377.772,
296.36,
215.412,
423.914,
231.341,
398.349
],
"before": [
{
"container": "33c568324dc6c6ba022889f9dc1fe9cf8ed4a064b0aba2a0b14b8a8e3ae2d213",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 113623040,
"container_peak_bytes": 108158976,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "b81f0a676713c37f0481157f4552020b9eff8823a183451c7303a5334aa6c6d0",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 111923200,
"container_peak_bytes": 106242048,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "33c568324dc6c6ba022889f9dc1fe9cf8ed4a064b0aba2a0b14b8a8e3ae2d213",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 131477504,
"container_peak_bytes": 134270976,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "b81f0a676713c37f0481157f4552020b9eff8823a183451c7303a5334aa6c6d0",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 131133440,
"container_peak_bytes": 137510912,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
},
{
"dimension": 1024,
"size_bytes": 3151221,
"concurrent_uploads": 10,
"repetition": 3,
"elapsed_seconds": 0.446,
"successful_uploads": 10,
"failed_attempts": 0,
"burst_successful_rps": 22.432872376785337,
"successful_latency_ms": {
"p50": 354.611,
"p95": 444.786,
"p99": 444.786
},
"latency_samples_ms": [
364.278,
444.786,
229.569,
233.565,
387.33,
419.847,
271.838,
354.611,
277.369,
398.801
],
"before": [
{
"container": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 112033792,
"container_peak_bytes": 106389504,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 111742976,
"container_peak_bytes": 106307584,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"after": [
{
"container": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 131276800,
"container_peak_bytes": 125980672,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
},
{
"container": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa",
"limits": {
"cpu": 1.0,
"ram": 536870912
},
"process_peak_rss_bytes": 130502656,
"container_peak_bytes": 135319552,
"container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"
}
],
"observed_instances": 2,
"original_checksums_verified": true
}
],
"measurement": "process high-water RSS and cgroup peak if available"
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

+37
View File
@@ -0,0 +1,37 @@
global
log stdout format raw local0
maxconn 2048
defaults
mode http
log global
option httplog
timeout connect 500ms
timeout client 30s
timeout server 50s
timeout check 1s
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 1
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold nx 1s
hold other 1s
hold refused 1s
hold timeout 1s
hold obsolete 1s
frontend public
bind :8080
http-request deny if { path_beg /internal/ }
default_backend messenger
backend messenger
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1
server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
exec minio "$@"
+10
View File
@@ -0,0 +1,10 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"],
"Condition": {"StringLike": {"s3:prefix": ["originals/*"]}}},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]}
]
}
+10
View File
@@ -0,0 +1,10 @@
{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": ["s3:GetBucketLocation"], "Resource": ["arn:aws:s3:::messenger-attachments"]},
{"Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::messenger-attachments"],
"Condition": {"StringLike": {"s3:prefix": ["originals/*"]}}},
{"Effect": "Allow", "Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]}
]
}
+27
View File
@@ -0,0 +1,27 @@
# syntax=docker/dockerfile:1.7.0
FROM python:3.12.10-slim-bookworm
ARG COMPONENT
ARG RELEASE
ARG SHA256
RUN python - <<'PY'
import hashlib
import os
from pathlib import Path
import shutil
import urllib.request
component = os.environ['COMPONENT']
release = os.environ['RELEASE']
assert component in {'minio', 'mc'}
name = f'{component}.linux-amd64.{release}'
url = f'https://github.com/minio/{component}/releases/download/{release}/{name}'
target = Path('/usr/local/bin') / component
with urllib.request.urlopen(url, timeout=180) as response, target.open('wb') as file:
shutil.copyfileobj(response, file)
with target.open('rb') as file:
digest = hashlib.file_digest(file, 'sha256').hexdigest()
assert digest == os.environ['SHA256'], 'Unexpected binary checksum'
target.chmod(0o755)
PY
COPY infra/minio-entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/bin/sh", "/entrypoint.sh"]
+36
View File
@@ -0,0 +1,36 @@
-- Сначала схема и ограничения, затем запросы приложения.
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_order_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
-- Equality по chat_id и range/order по двум полям курсора.
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
CREATE TABLE sessions (
token_hash char(64) PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
request_key varchar(128) NOT NULL,
body_hash char(64) NOT NULL,
response jsonb NOT NULL,
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, request_key)
);
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
+41
View File
@@ -0,0 +1,41 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename varchar(255) NOT NULL,
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 char(64) NOT NULL,
object_key text NOT NULL UNIQUE,
status text NOT NULL CHECK (status IN ('uploading', 'ready', 'missing')),
lease_until timestamptz NOT NULL,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, position),
UNIQUE (message_id, attachment_id)
);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
CREATE FUNCTION check_message_content() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF length(btrim(NEW.text))=0 AND NOT EXISTS (
SELECT 1 FROM message_attachments WHERE message_id=NEW.id
) THEN
RAISE EXCEPTION 'message needs text or attachment' USING ERRCODE='23514';
END IF;
RETURN NEW;
END;
$$;
-- Проверяем в конце транзакции, когда связи вложений уже вставлены.
CREATE CONSTRAINT TRIGGER message_content
AFTER INSERT OR UPDATE ON messages DEFERRABLE INITIALLY DEFERRED
FOR EACH ROW EXECUTE FUNCTION check_message_content();
-- Прежние текстовые повторы остаются совместимыми с расширенной схемой Message.
UPDATE message_requests SET response=response || '{"attachment_ids": []}'::jsonb
WHERE NOT response ? 'attachment_ids';
+2
View File
@@ -0,0 +1,2 @@
Pillow==12.0.0
matplotlib==3.9.4
+18
View File
@@ -0,0 +1,18 @@
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
fastapi==0.115.12
h11==0.16.0
idna==3.20
pydantic==2.11.5
pydantic_core==2.33.2
starlette==0.46.2
typing-inspection==0.4.4
typing_extensions==4.16.0
uvicorn==0.34.2
psycopg[binary]==3.2.9
psycopg-pool==3.3.3
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.40.18
Pillow==12.0.0
+156
View File
@@ -0,0 +1,156 @@
"""Один профиль нагрузки для всех серий. Только синтетические данные, без токенов в отчёте."""
from concurrent.futures import ThreadPoolExecutor
from collections import Counter
import math
from pathlib import Path
import sys
import threading
import time
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure
def context(base):
c = Client(base)
alice, bob, _, chat = c.bootstrap()
return dict(actor=alice, reader=bob, chat=chat)
def percentile(values, p):
if not values:
return None
return round(sorted(values)[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3)
def measure(base, ctx, duration, concurrency, action=None, retry_writes=False):
gate = threading.Event()
started = time.monotonic()
deadline = started + duration
action_events = []
def perform_action():
nonlocal deadline
time.sleep(2)
start = time.monotonic()
try:
info = action()
deadline = max(deadline, time.monotonic() + 5)
action_events.append(dict(start_seconds=start-started, end_seconds=time.monotonic()-started,
result=info))
except Exception as exc:
action_events.append(dict(start_seconds=start-started, error=str(exc)))
def worker(worker):
c = Client(base, timeout=3)
gate.wait()
latencies, confirmed, bins = [], [], Counter()
errors, instances, failures, operation_counts = Counter(), Counter(), [], Counter()
step, pending = worker, None
while time.monotonic() < deadline or (action_thread and action_thread.is_alive()):
if time.monotonic() - started > 180:
raise RuntimeError('Infrastructure action exceeded workload time limit')
begin = time.monotonic()
# Один и тот же цикл: 60% POST, 20% история, 20% профиль.
operation = 'write' if pending or step % 5 < 3 else ('history' if step % 5 == 3 else 'profile')
operation_counts[operation] += 1
try:
if operation == 'write':
key, text = pending or (uuid.uuid4().hex, 'x' * 128)
response = c.message(ctx['actor'], ctx['chat'], text, key=key)
confirmed.append(response['json'])
pending = None
elif operation == 'history':
response = c.call('GET', '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=50',
actor=ctx['reader'], schema='MessagePage')
else:
response = c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
elapsed = time.monotonic() - begin
latencies.append(elapsed)
instances[response['headers']['X-Instance-Id']] += 1
bins[int(time.monotonic() - started)] += 1
except Failure as exc:
# Сообщения ошибки клиента не содержат credentials/body.
errors[str(exc)] += 1
failures.append(round(time.monotonic()-started, 3))
if operation == 'write' and retry_writes:
pending = (key, text)
step += 1
# Последний timeout мог произойти после COMMIT. Уточняем результат тем же
# ключом вне измеряемого окна, иначе проверка ошибочно сочтёт его дубликатом.
finished = time.monotonic()
resolved = 0
if pending:
for attempt in range(10):
try:
confirmed.append(c.message(ctx['actor'], ctx['chat'], pending[1], key=pending[0])['json'])
pending = None
resolved = 1
break
except Failure:
time.sleep(.5)
if pending:
raise RuntimeError('Final uncertain write could not be resolved')
return dict(latencies=latencies, confirmed=confirmed, errors=errors, instances=instances,
failures=failures, bins=bins, operation_counts=operation_counts,
finished=finished, resolved_outside_window=resolved)
action_thread = threading.Thread(target=perform_action) if action else None
with ThreadPoolExecutor(max_workers=concurrency) as pool:
jobs = [pool.submit(worker, i) for i in range(concurrency)]
started = time.monotonic()
deadline = started + duration
if action_thread:
action_thread.start()
gate.set()
results = [job.result() for job in jobs]
elapsed = max(r['finished'] for r in results) - started
if action_thread:
action_thread.join(timeout=120)
if action_thread.is_alive():
raise RuntimeError('Infrastructure action did not finish')
if any('error' in e for e in action_events):
raise RuntimeError(str(action_events))
values = [x for r in results for x in r['latencies']]
errors, instances, bins, operation_counts = Counter(), Counter(), Counter(), Counter()
for r in results:
errors.update(r['errors']); instances.update(r['instances'])
bins.update(r['bins']); operation_counts.update(r['operation_counts'])
successes, failed = len(values), sum(errors.values())
report = dict(duration_requested=duration, elapsed_seconds=round(elapsed, 3), concurrency=concurrency,
profile='closed-loop: 60% POST text128, 20% first50 history, 20% profile',
retry_writes=retry_writes, successful_requests=successes, failed_attempts=failed,
uncertain_writes_resolved_after_window=sum(r['resolved_outside_window'] for r in results),
successful_rps=round(successes/elapsed, 3), error_ratio=failed/max(1, successes+failed),
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
instances=dict(instances), errors=dict(errors), operations=dict(operation_counts),
successful_per_second=dict(sorted(bins.items())),
failure_seconds=[x for r in results for x in r['failures']],
latency_samples_ms=[round(x*1000, 3) for x in values], action_events=action_events)
if not successes:
raise RuntimeError('No successful requests')
return report, [m for r in results for m in r['confirmed']]
def verify_confirmations(base, ctx, confirmations):
from urllib.parse import quote
c = Client(base)
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100'
stored = {}
while True:
page = c.call('GET', path, actor=ctx['reader'], schema='MessagePage')['json']
for message in page['items']:
if message['id'] in stored:
raise AssertionError('Duplicate ID in history')
stored[message['id']] = message
if page['next_cursor'] is None:
break
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'])
if not all(stored.get(m['id']) == m for m in confirmations):
raise AssertionError('Confirmed message missing after infrastructure action')
# В fault-сценарии нет seed. Каждый новый ключ должен породить одну строку.
expected = {m['id'] for m in confirmations}
if set(stored) - expected:
raise AssertionError('Retry created an extra message or left an unconfirmed result')
return dict(confirmed_messages=len(expected), stored_messages=len(stored), status='PASS')
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
endpoint="${S3_ENDPOINT:-http://s3:9000}"
bucket="messenger-attachments"
export MC_CONFIG_DIR=/tmp/mc-config
attempt=0
until mc alias set local "$endpoint" "$(cat /run/secrets/s3_root_user)" \
"$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
attempt=$((attempt + 1))
test "$attempt" -lt 60 || exit 1
sleep 1
done
case "${1:-init}" in
init)
mc mb --ignore-existing "local/$bucket" >/dev/null
mc anonymous set none "local/$bucket" >/dev/null
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" \
"$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
echo 'Private bucket and limited API credentials ready'
;;
deny-put)
mc admin policy create local messenger-api /policies/s3-no-put-policy.json >/dev/null
echo 'Synthetic S3 write failure enabled'
;;
allow-put)
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
echo 'Normal private bucket policy restored'
;;
*) exit 2 ;;
esac
+184
View File
@@ -0,0 +1,184 @@
"""Трек 2: реальные отказы, безопасная сверка и стоимость обхода; плюс RAM upload."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import io
import json
import os
from pathlib import Path
import random
import statistics
import sys
import threading
import time
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
from scripts.stack import command, containers, scale, save
from scripts.benchmark import percentile
def api_run(*arguments):
running = [row for row in containers('api') if row['running']]
if not running:
raise RuntimeError('No running API container')
return command(['docker', 'exec', running[0]['id'], 'python', *arguments])
def memory_stats():
output = []
for row in containers('api'):
if row['running']:
source = 'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/memory",timeout=5).read().decode())'
values = json.loads(command(['docker', 'exec', row['id'], 'python', '-c', source]))
output.append(dict(container=row['id'], limits=dict(cpu=row['cpu_limit'], ram=row['memory_limit_bytes']), **values))
return output
def file_checks():
result = json.loads(api_run('student-tests/storage_scenario.py', 'privacy'))
save('private_bucket.json', result)
interrupted = json.loads(api_run('student-tests/storage_scenario.py', 'interrupted'))
save('upload_interrupted.json', interrupted)
print('PASS: private bucket, limited credentials, closed console')
def storage_checks():
active = any(row['running'] for row in containers('reconciler'))
command(['docker', 'compose', 'stop', 'reconciler'])
try:
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'deny-put'])
try:
failure = json.loads(api_run('student-tests/storage_scenario.py', 's3-failure'))
save('s3_write_failure.json', failure)
finally:
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'allow-put'])
result = json.loads(api_run('student-tests/storage_scenario.py', 'reconcile', '--failed-chat-id', failure['chat_id']))
result['s3_write_failure'] = failure
save('reconcile_faults.json', result)
print('PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection')
finally:
if active:
command(['docker', 'compose', 'start', 'reconciler'])
def memory_experiment(base, repeats):
results = []
for dimension in [256, 1024]:
rng = random.Random(41)
image = Image.frombytes('RGB', (dimension, dimension), rng.randbytes(dimension*dimension*3))
output = io.BytesIO(); image.save(output, format='PNG'); image.close()
data = output.getvalue()
digest = hashlib.sha256(data).hexdigest()
for repeat in range(repeats):
print(f'Upload memory: {dimension}x{dimension}, repeat {repeat+1}/{repeats}', flush=True)
scale(2, 1, '512m', 8) # Новые процессы: high-water mark относится к этому случаю.
c = Client(base, timeout=60)
alice, bob, _, chat = c.bootstrap()
before = memory_stats()
gate = threading.Barrier(11)
def upload(_):
local = Client(base, timeout=60)
gate.wait()
start = time.monotonic()
response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=noise.png',
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
return response, time.monotonic()-start
with ThreadPoolExecutor(max_workers=10) as pool:
jobs = [pool.submit(upload, index) for index in range(10)]
start = time.monotonic(); gate.wait()
batch = [job.result() for job in jobs]
elapsed = time.monotonic()-start
after = memory_stats()
instances = {response['headers']['X-Instance-Id'] for response, _ in batch}
assert len(instances) == 2, 'Memory burst did not reach both replicas'
for response, _ in batch:
att = response['json']
assert att['sha256'] == digest and att['size_bytes'] == len(data)
content = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob)['bytes']
assert content == data
values = [latency for _, latency in batch]
results.append(dict(dimension=dimension, size_bytes=len(data), concurrent_uploads=10,
repetition=repeat+1, elapsed_seconds=round(elapsed, 3), successful_uploads=10,
failed_attempts=0, burst_successful_rps=10/elapsed,
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
latency_samples_ms=[round(v*1000, 3) for v in values],
before=before, after=after, observed_instances=len(instances),
original_checksums_verified=True))
save('upload_memory.json', dict(runs=results, measurement='process high-water RSS and cgroup peak if available'))
return results
def cost_experiment(sizes):
active = any(row['running'] for row in containers('reconciler'))
command(['docker', 'compose', 'stop', 'reconciler'])
try:
print('Reconciler cost: datasets '+', '.join(map(str, sizes))+', three scans each', flush=True)
output = command(['docker', 'compose', 'run', '--rm', '--no-deps', '-e', 'BASE_URL=http://lb:8080',
'reconciler', 'python', 'student-tests/storage_scenario.py', 'cost',
'--sizes', *map(str, sizes)], timeout=900)
result = json.loads(output)
save('reconcile_cost.json', result)
return result['scans']
finally:
if active:
command(['docker', 'compose', 'start', 'reconciler'])
def plots(cost, uploads):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
sizes = sorted({row['dataset_added'] for row in cost})
seconds = [statistics.median(row['elapsed_seconds'] for row in cost if row['dataset_added']==size) for size in sizes]
fig, ax = plt.subplots(figsize=(6, 4))
ax.plot(sizes, seconds, marker='o')
ax.set(xlabel='Objects + metadata', ylabel='Median scan, seconds', title='Reconciler cost')
ax.grid(alpha=.3); fig.tight_layout()
fig.savefig(ROOT/'evidence/reconcile_cost.png', dpi=150); plt.close(fig)
dimensions = sorted({row['dimension'] for row in uploads})
peaks = [max(api['process_peak_rss_bytes'] for row in uploads if row['dimension']==d for api in row['after'])/1024**2 for d in dimensions]
fig, ax = plt.subplots(figsize=(6, 4))
ax.bar([f'{d}x{d}' for d in dimensions], peaks)
ax.set(ylabel='Peak API process RSS, MiB', title='10 concurrent uploads')
fig.tight_layout(); fig.savefig(ROOT/'evidence/upload_memory.png', dpi=150); plt.close(fig)
def main():
p = argparse.ArgumentParser()
p.add_argument('--mode', choices=['privacy', 'faults', 'memory', 'cost', 'all'], default='all')
p.add_argument('--repeats', type=int, default=3)
p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000])
args = p.parse_args()
if args.repeats < 1 or sorted(set(args.sizes)) != args.sizes or min(args.sizes) < 1:
p.error('Positive repeats and increasing distinct dataset sizes required')
base = os.getenv('BASE_URL', 'http://localhost:8080')
save('experiment_environment.json', dict(docker=command(['docker', 'version', '--format', '{{.Server.Version}}']),
compose=command(['docker', 'compose', 'version', '--short']),
containers=containers(), client_cpus=os.cpu_count()))
if args.mode in {'privacy', 'all'}:
file_checks()
if args.mode in {'faults', 'all'}:
storage_checks()
uploads = memory_experiment(base, args.repeats) if args.mode in {'memory', 'all'} else []
cost = cost_experiment(args.sizes) if args.mode in {'cost', 'all'} else []
if uploads and cost:
plots(cost, uploads)
groups = []
for size in args.sizes:
cases = [row for row in cost if row['dataset_added'] == size]
groups.append(dict(objects=size, repetitions=len(cases),
median_seconds=statistics.median(row['elapsed_seconds'] for row in cases),
heads_per_scan=[row['counters']['head_requests'] for row in cases]))
save('experiment_summary.json', dict(reconciler_cost=groups, concurrent_uploads=10,
verified_originals=sum(row['successful_uploads'] for row in uploads),
status='PASS'))
print('Experiment completed; results saved in evidence/')
if __name__ == '__main__':
main()
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env python3
import json
import os
import urllib.request
host = "127.0.0.1"
port = os.getenv("APP_PORT", "8080")
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
payload = json.load(response)
if response.status != 200 or payload != {"status": "ok"}:
raise SystemExit(1)
+44
View File
@@ -0,0 +1,44 @@
"""Редкие снимки Docker CPU/RAM и ожиданий PostgreSQL; overhead отражён в методике."""
import json
import threading
import time
from scripts.stack import command, containers
class Monitor:
def __init__(self):
self.samples = []
self.stop = threading.Event()
self.thread = threading.Thread(target=self.loop)
def __enter__(self):
self.started = time.monotonic()
self.thread.start()
return self
def __exit__(self, *args):
self.stop.set()
self.thread.join(timeout=120)
def loop(self):
while not self.stop.is_set():
try:
ids = [r['id'] for r in containers() if r['running'] and r['service'] != 'migrate']
raw = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids])
stats = [json.loads(line) for line in raw.splitlines()]
query = "SELECT coalesce(json_agg(x),'[]') FROM (SELECT state,wait_event_type," \
"wait_event,count(*) FROM pg_stat_activity WHERE backend_type='client backend' " \
"AND pid<>pg_backend_pid() GROUP BY state,wait_event_type,wait_event) x"
waits = json.loads(command(['docker', 'compose', 'exec', '-T', 'db',
'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query]))
pools = []
for row in containers('api'):
if row['running']:
output = command(['docker', 'exec', row['id'], 'python', '-c',
'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/pool",timeout=3).read().decode())'])
pools.append(dict(instance=row['hostname'], stats=json.loads(output)))
self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3),
docker=stats, postgres=waits, pools=pools))
except Exception as exc:
self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3), error=str(exc)))
self.stop.wait(3)
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose stop reconciler lb api cache db s3
docker compose up -d --wait db cache s3
docker compose run --rm --no-deps migrate
docker compose run --rm --no-deps s3-bootstrap
docker compose up -d --no-deps --wait api
docker compose up -d --no-deps lb
python3 scripts/stack.py wait --count 2
docker compose up -d --no-deps reconciler
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose start api
python3 scripts/stack.py wait --count 2
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
mkdir -p .secrets
chmod 700 .secrets
python3 - <<'PY'
from pathlib import Path
import secrets
password_file = Path('.secrets/db_password')
if not password_file.exists():
password_file.write_text(secrets.token_hex(24))
password = password_file.read_text().strip()
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
for name in ['s3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key']:
file = Path('.secrets') / name
if not file.exists():
file.write_text(secrets.token_hex(16 if name.endswith(('user', 'id')) else 32))
for file in Path('.secrets').iterdir():
file.chmod(0o644 if file.name in {'database_url', 's3_access_key_id', 's3_secret_access_key'} else 0o600)
if not Path('.env').exists():
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\nAPI_CPU=1.0\nAPI_MEMORY=512m\n')
print('Локальные настройки подготовлены; секреты не выводятся.')
PY
+129
View File
@@ -0,0 +1,129 @@
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
def command(args, env=None, timeout=120):
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout)
if result.returncode:
raise RuntimeError(result.stderr[-2000:])
return result.stdout.strip()
def containers(service=None):
args = ['docker', 'compose', 'ps', '-a', '-q']
if service:
args.append(service)
ids = command(args).split()
if not ids:
return []
# Сохраняем только нужные поля, без environment с возможными секретами.
rows = json.loads(command(['docker', 'inspect', *ids]))
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
service=r['Config']['Labels']['com.docker.compose.service'],
running=r['State']['Running'], started_at=r['State']['StartedAt'],
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
memory_limit_bytes=r['HostConfig']['Memory'],
published_ports=r['NetworkSettings']['Ports']) for r in rows]
def save(name, data):
path = ROOT / 'evidence' / name
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
def wait(count, base=None):
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
deadline = time.monotonic() + 75
while time.monotonic() < deadline:
observed = set()
for _ in range(max(12, count * 8)):
try:
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
observed.add(response.headers['X-Instance-Id'])
except (OSError, ValueError):
break
if len(observed) == count and None not in observed:
return sorted(observed)
time.sleep(.5)
raise RuntimeError(f'Не дождались {count} готовых API через LB')
def stop(instance):
hostname = instance.split(':')[0]
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
if len(found) != 1:
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
before = containers()
command(['docker', 'stop', '--time', '3', found[0]['id']])
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
save('last_stop.json', result)
return result
def scale(count, cpu, memory, pool):
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
'--scale', f'api={count}', 'api'], env)
instances = wait(count)
rows = containers('api')
if len(rows) != count or not all(r['running'] for r in rows):
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
for row in rows:
if abs(row['cpu_limit'] - cpu) > .001:
raise RuntimeError('Docker не применил CPU limit')
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
raise RuntimeError('Docker не применил RAM limit')
return dict(instances=instances, containers=rows)
def rolling():
old = [r['id'] for r in containers('api') if r['running']]
if len(old) != 2:
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
before = containers()
command(['docker', 'compose', 'build', 'api'])
steps = []
for container in old:
# Не пересоздаём одновременно старые контейнеры при изменении image.
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
wait(3)
command(['docker', 'stop', '--time', '10', container])
command(['docker', 'rm', container])
wait(2)
steps.append(containers('api'))
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
def main():
p = argparse.ArgumentParser()
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
p.add_argument('--count', type=int, default=2)
p.add_argument('--instance')
p.add_argument('--output', default='containers.json')
args = p.parse_args()
if args.action == 'wait':
print('API ready:', ', '.join(wait(args.count)))
elif args.action == 'snapshot':
save(args.output, containers())
print('Container identities and limits saved')
elif args.action == 'stop':
if not args.instance:
p.error('--instance required')
stop(args.instance)
print('Selected API stopped')
else:
rolling()
print('API replaced one at a time')
if __name__ == '__main__':
main()
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose up -d --build
python3 scripts/stack.py wait --count 2
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
python3 scripts/stack.py stop --instance "$TARGET_INSTANCE_ID"
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
restore() {
docker compose up -d --no-deps --force-recreate --scale api=2 api
python3 scripts/stack.py wait --count 2
}
trap restore EXIT
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate --scale api=2 api
python3 scripts/stack.py wait --count 2
python3 student-tests/checks.py --expiry
+17
View File
@@ -0,0 +1,17 @@
"""Ожидание готовности API без сторонних библиотек."""
import os
import time
import urllib.request
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
for attempt in range(90):
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
print('API ready')
break
except OSError:
pass
time.sleep(1)
else:
raise SystemExit('API не стал готов за 90 попыток')
View File
+157
View File
@@ -0,0 +1,157 @@
"""Ограниченный binary upload, проверка изображения и сохранение в двух хранилищах."""
import asyncio
import base64
from datetime import datetime
import hashlib
import os
import tempfile
import threading
import unicodedata
import uuid
import warnings
from fastapi import Request
from fastapi.responses import StreamingResponse
from PIL import Image, ImageFile, PngImagePlugin, UnidentifiedImageError
from pydantic import BaseModel, ConfigDict
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from src.config import MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, UPLOAD_TIMEOUT
from src.errors import ApiProblem
from src.storage import ObjectStore, PREFIX, unlock_attachment
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
ImageFile.LOAD_TRUNCATED_IMAGES = False
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
PngImagePlugin.MAX_TEXT_MEMORY = 2 * 1024 * 1024
storage = ObjectStore()
upload_slots = asyncio.Semaphore(4)
decode_slots = threading.Semaphore(1)
class Attachment(BaseModel):
model_config = ConfigDict(extra='ignore')
id: uuid.UUID
chat_id: uuid.UUID
owner_id: uuid.UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list = []
error_code: str | None = None
created_at: datetime
updated_at: datetime
def public_attachment(row):
return Attachment(**row).model_dump(mode='json')
def filename_valid(name):
if (not 1 <= len(name) <= 255 or not name.strip() or name in {'.', '..'} or
any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in name)):
raise ApiProblem(400, 'invalid_request', 'Некорректное отображаемое имя файла')
def validate_image(file, declared_type):
# Декодируется максимум одно изображение на процесс, даже при нескольких upload.
with decode_slots, warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
try:
file.seek(0)
# Pillow может закрыть свой file object. Дубликат FD сохраняет исходный spool.
with os.fdopen(os.dup(file.fileno()), 'rb') as probe, Image.open(probe, formats=['PNG', 'JPEG']) as image:
if image.width * image.height > MAX_IMAGE_PIXELS:
raise ApiProblem(413, 'payload_too_large', 'Слишком много пикселей')
detected = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format]
if detected != declared_type:
raise ApiProblem(415, 'unsupported_media_type', 'MIME не совпадает с содержимым')
image.verify()
file.seek(0)
with os.fdopen(os.dup(file.fileno()), 'rb') as probe, Image.open(probe, formats=['PNG', 'JPEG']) as image:
image.load() # JPEG verify не проверяет все декодируемые данные.
image.close()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise ApiProblem(413, 'payload_too_large', 'Слишком много пикселей') from None
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
raise ApiProblem(415, 'unsupported_media_type', 'Повреждённое или неподдерживаемое изображение') from None
file.seek(0)
def persist(file, pool, chat_id, owner_id, filename, mime, size, sha256, md5):
attachment_id = uuid.uuid4()
key = PREFIX + str(chat_id) + '/' + str(attachment_id)
with pool.connection() as db:
# Session lock сохраняется между COMMIT намерения и записью ready.
# Reconciler использует тот же lock и не удаляет активную загрузку.
db.execute('SELECT pg_advisory_lock(hashtextextended(%s,0))', ('attachment:' + str(attachment_id),))
db.commit()
try:
db.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,status,lease_until) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'uploading',"
"clock_timestamp()+interval '120 seconds')",
(attachment_id, chat_id, owner_id, filename, mime, size, sha256, key))
db.commit() # Устойчивое намерение ДО S3 PUT.
storage.put(file, key, size, mime, sha256, md5)
row = db.execute("UPDATE attachments SET status='ready', updated_at=clock_timestamp(),"
"error_code=NULL WHERE id=%s RETURNING *", (attachment_id,)).fetchone()
db.commit() # 201 разрешён только после S3 PUT и COMMIT.
return public_attachment(row)
finally:
# При ошибке остаётся uploading с lease. Его безопасно обработает reconciler.
db.rollback()
unlock_attachment(db, attachment_id)
db.commit()
async def receive(request: Request, pool, chat_id, owner_id, filename):
filename_valid(filename)
mime = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if mime not in {'image/png', 'image/jpeg'}:
raise ApiProblem(415, 'unsupported_media_type', 'Нужен PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isdecimal():
raise ApiProblem(400, 'invalid_request', 'Некорректный Content-Length')
if int(length) > MAX_UPLOAD_BYTES:
raise ApiProblem(413, 'payload_too_large', 'Лимит файла 10 MiB')
try:
async with asyncio.timeout(UPLOAD_TIMEOUT), upload_slots:
# На диске только временный bounded spool, не оригинал для будущего download.
with tempfile.TemporaryFile() as file:
size, sha, md5 = 0, hashlib.sha256(), hashlib.md5()
async for chunk in request.stream():
size += len(chunk)
if size > MAX_UPLOAD_BYTES:
raise ApiProblem(413, 'payload_too_large', 'Лимит файла 10 MiB')
await run_in_threadpool(file.write, chunk)
sha.update(chunk); md5.update(chunk)
if not size:
raise ApiProblem(415, 'unsupported_media_type', 'Пустой файл')
await run_in_threadpool(validate_image, file, mime)
return await run_in_threadpool(persist, file, pool, chat_id, owner_id, filename,
mime, size, sha.hexdigest(), base64.b64encode(md5.digest()).decode())
except TimeoutError:
raise ApiProblem(503, 'unavailable', 'Время загрузки истекло') from None
except ClientDisconnect:
raise ApiProblem(400, 'invalid_request', 'Передача файла прервана') from None
def download(row):
result = storage.get(row['object_key'])
body = result['Body']
if result['ContentLength'] != row['size_bytes']:
body.close()
raise ApiProblem(503, 'unavailable', 'Оригинал временно недоступен')
def chunks():
try:
while data := body.read(64 * 1024):
yield data
finally:
body.close()
return StreamingResponse(chunks(), media_type=row['content_type'],
headers={'Content-Length': str(row['size_bytes'])})
+22
View File
@@ -0,0 +1,22 @@
"""Настройки из окружения или Docker secrets; секреты не печатаем."""
import os
from pathlib import Path
def setting(name, default=None):
file = os.getenv(name + '_FILE')
return Path(file).read_text().strip() if file else os.getenv(name, default)
DATABASE_URL = setting('DATABASE_URL')
CACHE_URL = setting('CACHE_URL', 'redis://localhost:6379/0')
SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
POOL_MAX = int(os.getenv('DB_POOL_MAX', '8'))
S3_ENDPOINT = setting('S3_ENDPOINT', 'http://s3:9000')
S3_REGION = setting('S3_REGION', 'us-east-1')
S3_BUCKET = setting('S3_BUCKET', 'messenger-attachments')
S3_ACCESS_KEY = setting('S3_ACCESS_KEY_ID')
S3_SECRET_KEY = setting('S3_SECRET_ACCESS_KEY')
MAX_UPLOAD_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_IMAGE_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
UPLOAD_TIMEOUT = int(os.getenv('UPLOAD_TIMEOUT_SECONDS', '45'))
+6
View File
@@ -0,0 +1,6 @@
class ApiProblem(Exception):
def __init__(self, status, code, message, challenge=None):
self.status = status
self.code = code
self.message = message
self.challenge = challenge
+637
View File
@@ -0,0 +1,637 @@
from __future__ import annotations
import base64
import json
import logging
import os
import re
import time
import uuid
from datetime import datetime, timedelta, timezone
from contextlib import asynccontextmanager
import hashlib
import secrets
import socket
import resource
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from starlette.exceptions import HTTPException
from src.config import DATABASE_URL, CACHE_URL, SESSION_TTL, POOL_MAX
from src.errors import ApiProblem
from src import attachments
from botocore.exceptions import BotoCoreError, ClientError
from typing import Annotated
from fastapi import Depends, FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from starlette.concurrency import run_in_threadpool
logging.basicConfig(
level=os.getenv("LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger("messenger")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def as_json(value: BaseModel) -> dict:
return value.model_dump(mode="json")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class User(StrictModel):
id: uuid.UUID
username: str
display_name: str
created_at: datetime
class CreateUser(StrictModel):
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("password")
@classmethod
def password_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("password must not contain only whitespace")
return value
@field_validator("display_name")
@classmethod
def display_name_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("display_name must not contain only whitespace")
return value
class Chat(StrictModel):
id: uuid.UUID
title: str
member_ids: list[uuid.UUID]
created_at: datetime
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def title_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("title must not contain only whitespace")
return value
@field_validator("member_ids")
@classmethod
def member_ids_are_unique(cls, value: list[uuid.UUID]) -> list[uuid.UUID]:
if len(value) != len(set(value)):
raise ValueError("member_ids must be unique")
return value
class Message(StrictModel):
id: uuid.UUID
chat_id: uuid.UUID
sender_id: uuid.UUID
text: str
created_at: datetime
attachment_ids: list[uuid.UUID] = Field(default_factory=list)
class CreateMessage(StrictModel):
text: str = Field(max_length=4000)
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
@model_validator(mode='after')
def valid_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('Нужен текст или вложение')
if len(self.attachment_ids) != len(set(self.attachment_ids)):
raise ValueError('Повторяющиеся attachment_ids')
return self
class Store:
"""Сессии и метаданные в PostgreSQL; Redis — только кеш профилей."""
def __init__(self) -> None:
self.pool = ConnectionPool(
DATABASE_URL, min_size=1, max_size=POOL_MAX, timeout=2, open=False,
kwargs={"row_factory": dict_row, "connect_timeout": 2,
"options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"},
)
self.cache = redis.Redis.from_url(
CACHE_URL, decode_responses=True, socket_connect_timeout=0.25, socket_timeout=0.25,
)
def start(self) -> None:
self.pool.open()
self.pool.wait(timeout=10)
def close(self) -> None:
self.pool.close()
self.cache.close()
def chat(self, db, chat_id: uuid.UUID) -> Chat | None:
row = db.execute("SELECT * FROM chats WHERE id=%s", (chat_id,)).fetchone()
if row is None:
return None
members = db.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",
(chat_id,)).fetchall()
return Chat(**row, member_ids=[member["user_id"] for member in members])
def chat_messages(self, db, chat_id: uuid.UUID, limit: int, after) -> list[Message]:
# В lab1 здесь были общий список/список чата и bisect_right.
# Теперь границу ищет B-tree PostgreSQL (chat_id, created_at, id).
query = ("SELECT m.*, COALESCE((SELECT array_agg(a.attachment_id ORDER BY a.position) "
"FROM message_attachments a WHERE a.message_id=m.id),'{}'::uuid[]) AS attachment_ids "
"FROM messages m WHERE chat_id=%s")
params = [chat_id]
if after is not None:
query += " AND (created_at,id)>(%s,%s)"
params.extend(after)
rows = db.execute(query + " ORDER BY created_at,id LIMIT %s", params + [limit + 1]).fetchall()
return [Message(**row) for row in rows]
INSTANCE_ID = os.getenv("INSTANCE_ID") or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}"
store = Store()
passwords = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
dummy_hash = passwords.hash(secrets.token_urlsafe(32))
@asynccontextmanager
async def lifespan(app):
store.start()
yield
store.close()
app = FastAPI(title="MTUCI PostgreSQL messenger", version="4.0.0", lifespan=lifespan,
docs_url=None, redoc_url=None, openapi_url=None)
def request_id(request: Request) -> str:
return request.state.request_id
def error_response(request: Request, status: int, code: str, message: str, challenge: str | None = None) -> JSONResponse:
rid = getattr(request.state, "request_id", str(uuid.uuid4()))
headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID}
if challenge:
headers["WWW-Authenticate"] = challenge
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": rid}},
headers=headers,
)
@app.middleware("http")
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid.uuid4())
started = time.perf_counter()
status = 500
try:
# Без токена отказываем ДО обращения к БД и проверки JSON.
if request.url.path.startswith("/api/v1/") and request.url.path not in {
"/api/v1/users", "/api/v1/auth/sessions"
}:
scheme, _, token = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "bearer" or not token or len(token) > 256:
status = 401
return error_response(request, 401, "unauthorized", "Требуется Bearer",
'Bearer realm="messenger"')
json_endpoints = {
"/api/v1/users",
"/api/v1/chats",
}
requires_json = request.method == "POST" and (
request.url.path in json_endpoints
or re.fullmatch(r"/api/v1/chats/[^/]+/messages", request.url.path) is not None
)
media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if requires_json and media_type != "application/json":
status = 415
return error_response(
request,
415,
"unsupported_media_type",
"Требуется Content-Type: application/json",
)
if requires_json:
# Ограниченный JSON; после body() Starlette повторно отдаёт его маршруту.
raw = await request.body()
if len(raw) > 65536:
status = 413
return error_response(request, 413, "payload_too_large", "Запрос слишком большой")
response = await call_next(request)
status = response.status_code
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
finally:
logger.info(
"request method=%s path=%s status=%s duration_ms=%.3f request_id=%s",
request.method,
request.url.path,
status,
(time.perf_counter() - started) * 1000,
request.state.request_id,
)
@app.exception_handler(ApiProblem)
async def api_problem_handler(request: Request, exc: ApiProblem):
return error_response(request, exc.status, exc.code, exc.message, exc.challenge)
@app.exception_handler(RequestValidationError)
async def validation_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Запрос не соответствует контракту")
@app.exception_handler(Exception)
async def unexpected_error_handler(request: Request, exc: Exception):
logger.error("unexpected_error request_id=%s", request_id(request))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
@app.exception_handler(psycopg.Error)
@app.exception_handler(PoolTimeout)
async def database_error_handler(request: Request, exc: Exception):
return error_response(request, 503, "unavailable", "Хранилище временно недоступно")
@app.exception_handler(BotoCoreError)
@app.exception_handler(ClientError)
async def object_store_error_handler(request: Request, exc: Exception):
return error_response(request, 503, 'unavailable', 'Объектное хранилище временно недоступно')
@app.exception_handler(HTTPException)
async def http_error_handler(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Неверный маршрут или метод")
def unauthorized(basic=False):
challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
raise ApiProblem(401, "unauthorized", "Нужна действительная идентификация", challenge)
def token_hash(token: str) -> str:
return hashlib.sha256(token.encode()).hexdigest()
def current_user(authorization: Annotated[str | None, Header()] = None) -> User:
scheme, _, token = (authorization or "").partition(" ")
if scheme.lower() != "bearer" or not token or len(token) > 256:
unauthorized()
with store.pool.connection() as db:
session = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()",
(token_hash(token),)).fetchone()
if session is None:
unauthorized()
key = "user:" + str(session["user_id"])
cache_available = True
try:
cached = store.cache.get(key)
if cached:
user = User.model_validate_json(cached)
if user.id == session["user_id"]:
return user
except (redis.RedisError, ValueError):
cache_available = False
row = db.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s",
(session["user_id"],)).fetchone()
user = User(**row)
if cache_available:
try:
store.cache.setex(key, 60, user.model_dump_json())
except redis.RedisError:
pass
return user
def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, uuid.UUID]:
try:
if not cursor or len(cursor) > 1024:
raise ValueError
padded = cursor + "=" * (-len(cursor) % 4)
payload = json.loads(base64.b64decode(padded.encode(), altchars=b"-_", validate=True).decode())
if not isinstance(payload, dict) or set(payload) != {"scope", "created_at", "id"}:
raise ValueError
if payload["scope"] != expected_scope:
raise ValueError
if not isinstance(payload["created_at"], str) or not isinstance(payload["id"], str):
raise ValueError
created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00"))
item_id = uuid.UUID(payload["id"])
if created_at.tzinfo is None:
raise ValueError
return created_at, item_id
except (ValueError, TypeError, KeyError, json.JSONDecodeError, UnicodeError):
raise ApiProblem(400, "invalid_request", "Некорректный cursor") from None
def encode_cursor(scope: str, created_at: datetime, item_id: uuid.UUID) -> str:
payload = {
"scope": scope,
"created_at": created_at.isoformat().replace("+00:00", "Z"),
"id": str(item_id),
}
return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode().rstrip("=")
def page(items: list, limit: int, scope: str) -> dict:
# SQL уже выбрал limit+1 строк после курсора; весь список не загружаем.
selected = items[:limit]
next_cursor = None
if len(items) > limit and selected:
last = selected[-1]
next_cursor = encode_cursor(scope, last.created_at, last.id)
return {"items": [as_json(item) for item in selected], "next_cursor": next_cursor}
def visible_chat(chat_id: uuid.UUID, user: User) -> Chat:
with store.pool.connection() as db:
member = db.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
(chat_id, user.id)).fetchone()
if member is None:
raise ApiProblem(404, "not_found", "Объект не найден")
return store.chat(db, chat_id)
def page_parameters(request: Request, scope: str, limit: int, cursor: str | None):
# Query(ge/le) сохранён из lab1, добавлена проверка дублей и формы limit.
if set(request.query_params) - {"limit", "cursor"} or any(
len(request.query_params.getlist(k)) > 1 for k in request.query_params
):
raise ApiProblem(400, "invalid_request", "Некорректные параметры страницы")
if not re.fullmatch(r"[0-9]{1,3}", request.query_params.get("limit", "50")):
raise ApiProblem(400, "invalid_request", "Некорректный limit")
return decode_cursor(cursor, scope) if cursor is not None else None
@app.get("/health/live")
def liveness():
return {"status": "ok"}
@app.get("/health/ready")
def readiness():
with store.pool.connection() as db:
db.execute("SELECT 1")
attachments.storage.ready()
return {"status": "ok"}
@app.get("/internal/pool")
def pool_statistics():
# Диагностика доступна только внутри Docker network: LB закрывает этот путь.
return store.pool.get_stats()
@app.get('/internal/memory')
def memory_statistics():
result = {'process_peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024}
from pathlib import Path
for name in ['/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes']:
path = Path(name)
if path.is_file():
result['container_peak_bytes'] = int(path.read_text().strip())
result['container_peak_source'] = name
break
return result
@app.post("/api/v1/users", status_code=201)
def register_user(body: CreateUser):
# Синхронные маршруты FastAPI исполняет в worker thread, не в event loop.
hashed = passwords.hash(body.password)
try:
with store.pool.connection() as db:
row = db.execute(
"INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) "
"RETURNING id,username,display_name,created_at",
(uuid.uuid4(), body.username, body.display_name, hashed),
).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiProblem(409, "conflict", "Username уже занят") from None
return as_json(User(**row))
@app.post("/api/v1/auth/sessions", status_code=201)
def create_session(request: Request, authorization: Annotated[str | None, Header()] = None):
if request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers:
raise ApiProblem(400, "invalid_request", "Вход не принимает тело запроса")
scheme, _, encoded = (authorization or "").partition(" ")
try:
if scheme.lower() != "basic" or len(encoded) > 2048:
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
raise ValueError
except (ValueError, UnicodeError):
unauthorized(basic=True)
with store.pool.connection() as db:
row = db.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
try:
passwords.verify(row["password_hash"] if row else dummy_hash, password)
except (VerificationError, InvalidHashError):
unauthorized(basic=True)
if row is None:
unauthorized(basic=True)
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL)
with store.pool.connection() as db:
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)",
(token_hash(token), row["id"], expires_at))
user = User(**{k: row[k] for k in ("id", "username", "display_name", "created_at")})
return {"token": token, "token_type": "Bearer", "user": as_json(user),
"expires_at": expires_at.isoformat().replace("+00:00", "Z")}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def delete_session(user: Annotated[User, Depends(current_user)],
authorization: Annotated[str, Header()]):
token = authorization.partition(" ")[2]
with store.pool.connection() as db:
db.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),))
return Response(status_code=204)
@app.get("/api/v1/users/me")
def get_current_user(user: Annotated[User, Depends(current_user)]):
return as_json(user)
@app.post("/api/v1/chats", status_code=201)
def create_chat(body: CreateChat, user: Annotated[User, Depends(current_user)]):
if user.id not in body.member_ids:
raise ApiProblem(400, "invalid_request", "Текущий пользователь должен быть участником")
with store.pool.connection() as db:
found = db.execute("SELECT id FROM users WHERE id=ANY(%s)", (body.member_ids,)).fetchall()
if len(found) != len(body.member_ids):
raise ApiProblem(400, "invalid_request", "Указан неизвестный участник")
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *",
(uuid.uuid4(), body.title)).fetchone()
with db.cursor() as cur:
cur.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",
[(row["id"], member) for member in body.member_ids])
chat = store.chat(db, row["id"])
return as_json(chat) # COMMIT уже выполнен для чата и всех участников.
@app.get("/api/v1/chats")
def list_chats(
request: Request,
user: Annotated[User, Depends(current_user)],
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
):
scope = f"chats:{user.id}"
after = page_parameters(request, scope, limit, cursor)
query = "SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"
params = [user.id]
if after is not None:
query += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
with store.pool.connection() as db:
rows = db.execute(query + " ORDER BY c.created_at,c.id LIMIT %s", params + [limit + 1]).fetchall()
chats = [store.chat(db, row["id"]) for row in rows]
return page(chats, limit, scope)
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)]):
return as_json(visible_chat(chat_id, user))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
request: Request,
chat_id: uuid.UUID,
body: CreateMessage,
user: Annotated[User, Depends(current_user)],
idempotency_key: Annotated[str | None, Header()] = None,
):
if (not idempotency_key or
not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) or
len(request.headers.getlist("idempotency-key")) != 1):
raise ApiProblem(400, "invalid_request", "Нужен корректный Idempotency-Key")
visible_chat(chat_id, user) # Повтор тоже проверяет сессию и участие.
canonical = {'text': body.text}
if body.attachment_ids:
canonical['attachment_ids'] = [str(value) for value in body.attachment_ids]
payload = json.dumps(canonical, ensure_ascii=False, sort_keys=True,
separators=(",", ":"))
digest = hashlib.sha256(payload.encode()).hexdigest()
lock_scope = f"{user.id}:POST:{chat_id}:{idempotency_key}"
with store.pool.connection() as db:
# Общая блокировка в PostgreSQL, а не локальный Lock одной реплики.
# Коллизия хеша только сериализует независимые запросы; ключи сравниваются полностью.
db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (lock_scope,))
previous = db.execute(
"SELECT * FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s",
(user.id, chat_id, idempotency_key),
).fetchone()
if previous and previous["expires_at"] > utc_now():
if previous["body_hash"] != digest:
raise ApiProblem(409, "conflict", "Ключ уже использован с другим сообщением")
result = previous["response"]
else:
for attachment_id in body.attachment_ids:
found = db.execute('SELECT chat_id,status FROM attachments WHERE id=%s FOR KEY SHARE',
(attachment_id,)).fetchone()
if not found or found['chat_id'] != chat_id:
raise ApiProblem(404, 'not_found', 'Вложение недоступно')
if found['status'] != 'ready':
raise ApiProblem(409, 'conflict', 'Вложение ещё недоступно')
if previous:
db.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s",
(user.id, chat_id, idempotency_key))
row = db.execute(
"INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(uuid.uuid4(), chat_id, user.id, body.text),
).fetchone()
for position, attachment_id in enumerate(body.attachment_ids):
db.execute('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
(row['id'], attachment_id, position))
result = as_json(Message(**row, attachment_ids=body.attachment_ids))
db.execute(
"INSERT INTO message_requests(user_id,chat_id,request_key,body_hash,response,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",
(user.id, chat_id, idempotency_key, digest, Jsonb(result)),
)
# Сообщение и результат повтора зафиксированы одной транзакцией до ответа.
return result
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(
request: Request,
chat_id: uuid.UUID,
user: Annotated[User, Depends(current_user)],
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
):
visible_chat(chat_id, user)
scope = f"messages:{chat_id}"
after = page_parameters(request, scope, limit, cursor)
with store.pool.connection() as db:
messages = store.chat_messages(db, chat_id, limit, after)
return page(messages, limit, scope)
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
async def upload_attachment(request: Request, chat_id: uuid.UUID,
user: Annotated[User, Depends(current_user)], filename: str = Query()):
await run_in_threadpool(visible_chat, chat_id, user)
return await attachments.receive(request, store.pool, chat_id, user.id, filename)
def accessible_attachment(attachment_id, user):
with store.pool.connection() as db:
row = db.execute('SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
'WHERE a.id=%s AND m.user_id=%s', (attachment_id, user.id)).fetchone()
if not row:
raise ApiProblem(404, 'not_found', 'Вложение недоступно')
return row
@app.get('/api/v1/attachments/{attachment_id}')
def attachment_metadata(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)]):
row = accessible_attachment(attachment_id, user)
if row['status'] != 'ready':
raise ApiProblem(503, 'unavailable', 'Вложение временно недоступно')
return attachments.public_attachment(row)
@app.get('/api/v1/attachments/{attachment_id}/content')
def attachment_content(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)],
variant: str = Query('original')):
row = accessible_attachment(attachment_id, user)
if variant != 'original':
raise ApiProblem(404, 'not_found', 'Вариант отсутствует')
if row['status'] != 'ready':
raise ApiProblem(503, 'unavailable', 'Вложение временно недоступно')
return attachments.download(row)
+26
View File
@@ -0,0 +1,26 @@
"""Версионируемые миграции: один файл = одна атомарная транзакция."""
from pathlib import Path
import psycopg
from src.config import DATABASE_URL
def migrate():
with psycopg.connect(DATABASE_URL) as db:
# Миграции запускает отдельный сервис. Lock защищает от второго запуска оператором.
db.execute('SELECT pg_advisory_lock(20260002)')
db.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
'(version text PRIMARY KEY, applied_at timestamptz DEFAULT now())')
db.commit()
for file in sorted(Path('migrations').glob('*.sql')):
with db.transaction():
applied = db.execute('SELECT 1 FROM schema_migrations WHERE version=%s',
(file.name,)).fetchone()
if not applied:
db.execute(file.read_text())
db.execute('INSERT INTO schema_migrations(version) VALUES (%s)', (file.name,))
print('Applied', file.name)
db.execute('SELECT pg_advisory_unlock(20260002)')
if __name__ == '__main__':
migrate()
+193
View File
@@ -0,0 +1,193 @@
"""Сверка двух хранилищ. Dry-run по умолчанию, apply меняет только проверенные записи."""
import argparse
from datetime import datetime, timezone
import hashlib
import json
import resource
import time
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
from src.config import DATABASE_URL
from src.storage import ObjectStore, PREFIX, lock_attachment, unlock_attachment
def checksum_matches(storage, row, counters):
result = storage.get(row['object_key'])
counters['get_requests'] += 1
size, digest = 0, hashlib.sha256()
try:
while chunk := result['Body'].read(64 * 1024):
size += len(chunk)
counters['bytes_read'] += len(chunk)
if size > row['size_bytes']:
return False
digest.update(chunk)
finally:
result['Body'].close()
return size == row['size_bytes'] and digest.hexdigest() == row['sha256']
def run(grace=600, apply=False, chat_id=None):
if grace < 1:
raise ValueError('Grace period must be positive')
started = time.monotonic()
now = datetime.now(timezone.utc)
storage = ObjectStore()
counters = dict(objects_scanned=0, metadata_scanned=0, list_requests=0, head_requests=0,
get_requests=0, bytes_read=0, protected_fresh=0,
protected_active=0, protected_used=0)
actions = []
def old(timestamp):
return (now - timestamp).total_seconds() >= grace
def head(key):
counters['head_requests'] += 1
return storage.head(key)
def action(kind, row, reason=None):
item = dict(kind=kind, attachment_id=str(row['id']), object_key=row['object_key'], applied=apply)
if reason:
item['reason'] = reason
actions.append(item)
def release(db, attachment_id):
db.rollback()
unlock_attachment(db, attachment_id)
db.commit()
try:
with psycopg.connect(DATABASE_URL, row_factory=dict_row, connect_timeout=2,
options='-c statement_timeout=10000 -c lock_timeout=2000') as db:
# LIST завершается успешно до любой модификации. Ошибка зависимости не равна missing.
prefix = PREFIX if chat_id is None else PREFIX + str(UUID(str(chat_id))) + '/'
inventory = list(storage.objects(prefix, counters))
for obj in inventory:
counters['objects_scanned'] += 1
try:
attachment_id = UUID(obj['Key'].rsplit('/', 1)[-1])
except ValueError:
continue # Не трогаем неизвестный формат ключа.
if not lock_attachment(db, attachment_id, wait=False):
counters['protected_active'] += 1
db.commit()
continue
db.commit()
try:
row = db.execute('SELECT id FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone()
if row:
continue
current = head(obj['Key']) # Не полагаемся только на старый LIST snapshot.
if not current:
continue
if not old(current['LastModified']):
counters['protected_fresh'] += 1
continue
action('delete_orphan', dict(id=attachment_id, object_key=obj['Key']))
if apply:
storage.delete(obj['Key'])
finally:
release(db, attachment_id)
if chat_id is None:
ids = db.execute('SELECT id FROM attachments ORDER BY id').fetchall()
else:
ids = db.execute('SELECT id FROM attachments WHERE chat_id=%s ORDER BY id', (chat_id,)).fetchall()
db.commit()
for identifier in ids:
attachment_id = identifier['id']
counters['metadata_scanned'] += 1
if not lock_attachment(db, attachment_id, wait=False):
counters['protected_active'] += 1
db.commit()
continue
db.commit()
try:
row = db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',
(attachment_id,)).fetchone()
if not row:
continue
if not old(row['created_at']):
counters['protected_fresh'] += 1
continue
if row['status'] == 'uploading' and row['lease_until'] > now:
counters['protected_active'] += 1
continue
used = db.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',
(attachment_id,)).fetchone()
current = head(row['object_key'])
if row['status'] == 'uploading':
if used:
counters['protected_used'] += 1
continue
if current and not old(current['LastModified']):
counters['protected_fresh'] += 1
continue
action('delete_stale_upload', row, 'lease_expired')
if apply:
if current:
storage.delete(row['object_key'])
db.execute('DELETE FROM attachments WHERE id=%s', (attachment_id,))
db.commit()
continue
if not current:
if row['status'] != 'missing':
action('mark_missing', row, 'object_missing')
if apply:
db.execute("UPDATE attachments SET status='missing',error_code='object_missing',"
"updated_at=clock_timestamp() WHERE id=%s", (attachment_id,))
db.commit()
continue
matches = current['ContentLength'] == row['size_bytes']
if matches and (row['status'] == 'missing' or current.get('Metadata', {}).get('sha256') != row['sha256']):
matches = checksum_matches(storage, row, counters)
if not matches:
if row['status'] != 'missing':
action('mark_missing', row, 'object_corrupt')
if apply:
db.execute("UPDATE attachments SET status='missing',error_code='object_corrupt',"
"updated_at=clock_timestamp() WHERE id=%s", (attachment_id,))
db.commit()
continue
if row['status'] == 'missing':
action('restore_ready', row, 'verified_original_restored')
if apply:
db.execute("UPDATE attachments SET status='ready',error_code=NULL,"
"updated_at=clock_timestamp() WHERE id=%s", (attachment_id,))
db.commit()
elif used:
counters['protected_used'] += 1
finally:
release(db, attachment_id)
finally:
storage.client.close()
return dict(mode='apply' if apply else 'dry-run', grace_seconds=grace, chat_scope=str(chat_id) if chat_id else None,
elapsed_seconds=round(time.monotonic()-started, 6), counters=counters, actions=actions,
process_peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024)
def main():
p = argparse.ArgumentParser()
p.add_argument('--apply', action='store_true')
p.add_argument('--grace-seconds', type=int, default=600)
p.add_argument('--loop', action='store_true')
p.add_argument('--interval', type=int, default=60)
p.add_argument('--chat-id', type=UUID, help='Optional safe scope for one chat')
args = p.parse_args()
while True:
try:
print(json.dumps(run(args.grace_seconds, args.apply, args.chat_id), ensure_ascii=False), flush=True)
except Exception:
# Не выдаём DSN, ключи и текст исключений SDK в логи.
print(json.dumps({'status': 'FAILED', 'reason': 'dependency_or_scan_error'}), flush=True)
if not args.loop:
raise SystemExit(1)
if not args.loop:
break
time.sleep(max(1, args.interval))
if __name__ == '__main__':
main()
+57
View File
@@ -0,0 +1,57 @@
"""S3-клиент с ограниченными credentials; ключи назначаются приложением."""
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from src.config import S3_ENDPOINT, S3_REGION, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY
PREFIX = 'originals/'
class ObjectStore:
def __init__(self):
self.client = boto3.client(
's3', endpoint_url=S3_ENDPOINT, region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'}, request_checksum_calculation='when_required',
response_checksum_validation='when_required'))
def ready(self):
self.client.list_objects_v2(Bucket=S3_BUCKET, Prefix=PREFIX, MaxKeys=1)
def put(self, file, key, size, mime, sha256, content_md5):
file.seek(0)
self.client.put_object(Bucket=S3_BUCKET, Key=key, Body=file, ContentLength=size,
ContentType=mime, Metadata={'sha256': sha256}, ContentMD5=content_md5)
def head(self, key):
try:
return self.client.head_object(Bucket=S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response['Error']['Code'] in {'404', 'NoSuchKey', 'NotFound'}:
return None
raise
def get(self, key):
return self.client.get_object(Bucket=S3_BUCKET, Key=key)
def delete(self, key):
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
def objects(self, prefix=PREFIX, counters=None):
pages = self.client.get_paginator('list_objects_v2')
for page in pages.paginate(Bucket=S3_BUCKET, Prefix=prefix):
if counters is not None:
counters['list_requests'] += 1
yield from page.get('Contents', [])
def lock_attachment(db, attachment_id, wait=True):
function = 'pg_advisory_lock' if wait else 'pg_try_advisory_lock'
return db.execute(f'SELECT {function}(hashtextextended(%s,0)) AS locked',
('attachment:' + str(attachment_id),)).fetchone()['locked']
def unlock_attachment(db, attachment_id):
db.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',
('attachment:' + str(attachment_id),))
+87
View File
@@ -0,0 +1,87 @@
"""Контракт файлов: реальные изображения, лимиты, ACL и ключи сообщений."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import http.client
import io
from pathlib import Path
import struct
import sys
from urllib.parse import quote, urlsplit
import uuid
import zlib
from PIL import Image
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Attachment negative and concurrent checks')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/attachments?filename='
png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
headers = {'Content-Type': 'image/png'}
att = c.call('POST', path+'sample.png', 201, actor=alice, raw=png, headers=headers, schema='Attachment')['json']
check('object_key' not in att and 'url' not in att, 'private storage details leaked')
for filename in ['../a.png', 'a\\b.png', 'x\x00.png', 'x\n.png', '..', ' ']:
c.call('POST', path+quote(filename, safe=''), 400, actor=alice, raw=png, headers=headers)
for data, mime in [(b'not an image', 'image/png'), (png[:30], 'image/png'),
(png[:-12], 'image/png'), (png, 'image/jpeg'), (png, 'application/octet-stream')]:
c.call('POST', path+'invalid.png', 415, actor=alice, raw=data, headers={'Content-Type': mime})
jpeg = io.BytesIO()
Image.new('RGB', (64, 64), 'red').save(jpeg, format='JPEG')
data = jpeg.getvalue()
j = c.call('POST', path+'real.jpg', 201, actor=alice, raw=data,
headers={'Content-Type': 'image/jpeg'}, schema='Attachment')['json']
downloaded = c.call('GET', '/api/v1/attachments/'+j['id']+'/content', actor=bob)
check(downloaded['bytes'] == data, 'JPEG bytes changed')
c.call('POST', path+'broken.jpg', 415, actor=alice, raw=data[:-20], headers={'Content-Type': 'image/jpeg'})
# Валидный IHDR заявляет >40M пикселей. Отклонение происходит до декодирования.
ihdr = struct.pack('>IIBBBBB', 9000, 5000, 8, 2, 0, 0, 0)
header = b'\x89PNG\r\n\x1a\n' + struct.pack('>I', len(ihdr)) + b'IHDR' + ihdr
header += struct.pack('>I', zlib.crc32(b'IHDR'+ihdr))
header += png[33:]
c.call('POST', path+'huge-pixels.png', 413, actor=alice, raw=header, headers=headers)
# Отправляем только headers заведомо большого файла: доказываем ранний byte limit.
base = urlsplit(c.base)
connection = http.client.HTTPConnection(base.hostname, base.port or 80, timeout=10)
connection.putrequest('POST', path+'too-big.png')
connection.putheader('Authorization', 'Bearer '+alice['token'])
connection.putheader('Content-Type', 'image/png')
connection.putheader('Content-Length', str(10485761))
connection.endheaders()
response = connection.getresponse()
check(response.status == 413, 'oversized body was not rejected before reading')
response.read(); connection.close()
c.call('POST', path+'outsider.png', 404, actor=eve, raw=png, headers=headers)
for suffix in ['', '/content']:
c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve)
c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob)
other_chat = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json']
message_path = '/api/v1/chats/'+other_chat['id']+'/messages'
c.call('POST', message_path, 404, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
message_path = '/api/v1/chats/'+chat['id']+'/messages'
for ids, code in [([str(uuid.uuid4())], 404), ([att['id'], att['id']], 400), ([att['id']]*11, 400)]:
c.call('POST', message_path, code, actor=alice, body={'text': '', 'attachment_ids': ids},
headers={'Idempotency-Key': uuid.uuid4().hex})
key = uuid.uuid4().hex
def send(_):
return Client(args.base_url).message(bob, chat, '', key=key, attachments=[att['id']])
with ThreadPoolExecutor(max_workers=8) as pool:
replies = list(pool.map(send, range(16)))
check(all(r['json'] == replies[0]['json'] for r in replies), 'attachment replay created different messages')
check(len({r['headers']['X-Instance-Id'] for r in replies}) >= 2, 'attachment race did not cross replicas')
# Отсутствие attachment_ids и [] одинаковы и для хеша повтора.
text_key = uuid.uuid4().hex
text_message = c.message(alice, chat, 'Только текст', key=text_key)['json']
check(c.message(alice, chat, 'Только текст', key=text_key, attachments=[])['json'] == text_message,
'default [] changes key semantics')
check(att['sha256'] == hashlib.sha256(png).hexdigest(), 'wrong original checksum')
print('PASS: PNG/JPEG, bytes/pixels limits, filename/MIME, ACL, attachment message replay')
if __name__ == '__main__':
run(main)
+64
View File
@@ -0,0 +1,64 @@
"""Каждая реплика отдельно и поочерёдная замена под одним профилем нагрузки."""
import json
import os
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from scripts.benchmark import context, measure, verify_confirmations
from scripts.stack import containers, stop, wait, command, rolling, save
from scripts.monitor import Monitor
def main():
base = os.getenv('BASE_URL', 'http://localhost:8080')
aliases = wait(2, base)
results = []
for target in aliases:
ctx = context(base)
# Сохраняем подтверждённую запись до отказа вместе с подтверждениями нагрузки.
from scripts.benchmark import Client
original = Client(base).message(ctx['actor'], ctx['chat'], 'До отказа')['json']
attachment, content = Client(base).attachment(ctx['actor'], ctx['chat'])
try:
with Monitor() as monitor:
report, confirmations = measure(base, ctx, 16, 4,
action=lambda: stop(target), retry_writes=True)
event = report['action_events'][0]
successful_times = sorted(int(x) for x, count in report['successful_per_second'].items() if count)
first = next((x for x in successful_times if x > event['end_seconds']), None)
if first is None or first+1-event['start_seconds'] > 10:
raise AssertionError('Recovery exceeded 10 seconds')
stopped_host = target.split(':')[0]
if any(r['running'] for r in containers('api') if r['hostname'] == stopped_host):
raise AssertionError('Target is still running')
report['first_successful_bucket_after_stop_seconds'] = first
report['recovery_upper_bound_seconds'] = round(first+1-event['start_seconds'], 3)
report['verification'] = verify_confirmations(base, ctx, [original, *confirmations])
assert Client(base).call('GET', '/api/v1/attachments/'+attachment['id']+'/content',
actor=ctx['reader'])['bytes'] == content
report['attachment_original_verified'] = True
report['monitor'] = monitor.samples
report['target'] = target
results.append(report)
save('failures.json', results)
finally:
command(['docker', 'compose', 'start', 'api'])
wait(2, base)
ctx = context(base)
attachment, content = Client(base).attachment(ctx['actor'], ctx['chat'])
original = Client(base).message(ctx['actor'], ctx['chat'], '', attachments=[attachment['id']])['json']
with Monitor() as monitor:
report, confirmations = measure(base, ctx, 35, 4, action=rolling, retry_writes=True)
report['verification'] = verify_confirmations(base, ctx, [original, *confirmations])
assert Client(base).call('GET', '/api/v1/attachments/'+attachment['id']+'/content',
actor=ctx['reader'])['bytes'] == content
report['attachment_original_verified'] = True
report['monitor'] = monitor.samples
save('rolling.json', report)
print('PASS: both API failures and rolling replacement, confirmations preserved')
if __name__ == '__main__':
main()
+90
View File
@@ -0,0 +1,90 @@
"""Дополнения к публичному smoke. Не печатает и не сохраняет tokens."""
import base64
import concurrent.futures
import json
import sys
import time
from pathlib import Path
from uuid import uuid4
from urllib.parse import quote
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def check_cursor_types(c, actor, path, cursor, schema):
check(bool(cursor), 'test requires a real pagination cursor')
payload = json.loads(base64.urlsafe_b64decode(cursor + '=' * (-len(cursor) % 4)))
c.call('GET', path + '?cursor=' + quote(cursor, safe=''), actor=actor, schema=schema)
invalid = []
for field in ['created_at', 'id']:
for value in [1, 1.5, None, True, [], {}, '', 'invalid']:
invalid.append({**payload, field: value})
invalid.extend([None, 1, True, [], 'cursor'])
for value in invalid:
encoded = base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip('=')
instances = set()
# Проверяем один и тот же cursor на обеих API за round-robin LB.
for _ in range(8):
response = c.call('GET', path + '?cursor=' + quote(encoded, safe=''), 400, actor=actor)
check(response['json']['error']['code'] == 'invalid_request', 'incorrect cursor error code')
instances.add(response['headers']['X-Instance-Id'])
if len(instances) >= 2:
break
check(len(instances) >= 2, 'invalid cursor was not checked on two API instances')
def main():
def configure(p):
p.add_argument('--expiry', action='store_true', help='API must run with TTL=2')
args = arguments('Student API checks', configure)
c = Client(args.base_url)
alice, bob, eve, chat = c.bootstrap()
if args.expiry:
time.sleep(2.2)
c.call('GET', '/api/v1/users/me', 401, actor=alice)
print('PASS: short TTL expiry')
return
registration = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Гонка',
'password': 'Synthetic-password-42'}
# Для гонки регистрируем оба допустимых исхода через urllib.
import urllib.request
import urllib.error
def register(_):
req = urllib.request.Request(c.base + '/api/v1/users',
data=json.dumps(registration).encode(),
headers={'Content-Type': 'application/json'})
try:
with urllib.request.urlopen(req, timeout=10) as response:
return response.status
except urllib.error.HTTPError as error:
return error.code
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
codes = list(executor.map(register, range(8)))
check(codes.count(201) == 1 and codes.count(409) == 7, 'username race failed')
path = '/api/v1/chats/' + chat['id']
c.call('POST', '/api/v1/chats', 400, actor=alice,
body={'title': 'unknown member', 'member_ids': [alice['id'], str(uuid4())]})
for body in [{'text': 'x', 'sender_id': eve['id']}, {'text': 4}, {'text': ' '}]:
c.call('POST', path + '/messages', 400, actor=alice, body=body)
c.call('GET', path + '/messages?limit=true', 400, actor=alice)
c.call('GET', path + '/messages?cursor=not-a-cursor', 400, actor=alice)
for n in range(3):
c.message(alice, chat, 'read-after-write ' + str(n))
page = c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json']
other = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json']
c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' +
quote(page['next_cursor'], safe=''), 400, actor=alice)
check_cursor_types(c, alice, path + '/messages', page['next_cursor'], 'MessagePage')
chats = c.call('GET', '/api/v1/chats?limit=1', actor=alice, schema='ChatPage')['json']
check_cursor_types(c, alice, '/api/v1/chats', chats['next_cursor'], 'ChatPage')
history = c.call('GET', path + '/messages', actor=bob, schema='MessagePage')['json']['items']
check(len(history) == 3 and history[-1]['text'] == 'read-after-write 2', 'read-after-write failed')
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
c.call('GET', '/api/v1/users/me', 401, actor=alice)
print('PASS: registration race, invalid input, cursor scope/types on two API, read-after-write, logout')
if __name__ == '__main__':
run(main)
+61
View File
@@ -0,0 +1,61 @@
"""Проверка реального rollback после вставки чата и первого участника."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from uuid import uuid4
import psycopg
import os
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
from src.config import DATABASE_URL
from src.migrate import migrate
migrate()
migrate() # Повтор миграции не создаёт дубликаты.
with psycopg.connect(DATABASE_URL) as db:
user = db.execute('SELECT id FROM users LIMIT 1').fetchone()
assert user, 'Сначала выполните smoke, чтобы создать синтетического пользователя'
chat = uuid4()
try:
with db.transaction():
db.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (chat, 'rollback test'))
db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, user[0]))
# FK прерывает составную операцию ПОСЛЕ двух успешных INSERT.
db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, uuid4()))
except psycopg.errors.ForeignKeyViolation:
pass
else:
raise AssertionError('Expected foreign-key failure')
assert db.execute('SELECT count(*) FROM chats WHERE id=%s', (chat,)).fetchone()[0] == 0
assert db.execute('SELECT count(*) FROM chat_members WHERE chat_id=%s', (chat,)).fetchone()[0] == 0
assert db.execute('SELECT count(*) FROM schema_migrations').fetchone()[0] == 3
print('PASS: rollback of chat + members; migration reapplied twice')
c = Client(os.getenv('BASE_URL', 'http://localhost:8080'))
alice, bob, _, chat = c.bootstrap()
key = uuid4().hex
original = c.message(alice, chat, 'До истечения ключа', key=key)['json']
with psycopg.connect(DATABASE_URL) as db:
# Без ожидания суток: сдвигаем именно persisted expiry, а не clock приложения.
db.execute("UPDATE message_requests SET expires_at=now()-interval '1 second' "
"WHERE chat_id=%s AND request_key=%s", (chat['id'], key))
renewed = c.message(alice, chat, 'После истечения ключа', key=key)['json']
assert renewed['id'] != original['id']
with psycopg.connect(DATABASE_URL) as db:
assert db.execute('SELECT count(*) FROM message_requests WHERE chat_id=%s AND request_key=%s',
(chat['id'], key)).fetchone()[0] == 1
with psycopg.connect(DATABASE_URL) as db:
message = uuid4()
try:
with db.transaction():
db.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)',
(message, chat['id'], bob['id'], 'atomic rollback'))
# Ошибка сохранения результата должна откатить и сообщение.
db.execute("INSERT INTO message_requests VALUES (%s,%s,%s,%s,%s,now())",
(uuid4(), chat['id'], 'rollback', '0'*64, psycopg.types.json.Jsonb({})))
except psycopg.errors.ForeignKeyViolation:
pass
else:
raise AssertionError('Expected result-store failure')
assert db.execute('SELECT count(*) FROM messages WHERE id=%s', (message,)).fetchone()[0] == 0
print('PASS: persisted key expiry, atomic message/result rollback')
+71
View File
@@ -0,0 +1,71 @@
"""Redis необязателен для корректности; без PostgreSQL готовность снимается."""
import os
from pathlib import Path
import sys
import time
import urllib.request
import urllib.error
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from scripts.benchmark import context, Client
from scripts.stack import command, wait, save, containers
base = os.getenv('BASE_URL', 'http://localhost:8080')
c = Client(base)
ctx = context(base)
original = c.message(ctx['actor'], ctx['chat'], 'До отказов')['json']
attachment, content = c.attachment(ctx['actor'], ctx['chat'])
results = {}
try:
command(['docker', 'compose', 'stop', 'cache'])
started = time.monotonic()
for _ in range(6):
c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
c.message(ctx['actor'], ctx['chat'], 'Без Redis')
results['redis'] = dict(status='PASS', elapsed_seconds=time.monotonic()-started)
finally:
command(['docker', 'compose', 'up', '-d', '--wait', 'cache'])
try:
command(['docker', 'compose', 'stop', 'db'])
# Даём LB время исключить все неготовые API. Его собственный 503 может быть HTML.
time.sleep(4)
codes = []
for path in ['/health/ready', '/api/v1/users/me']:
request = urllib.request.Request(base+path, headers={'Authorization': 'Bearer '+ctx['actor']['token']})
try:
with urllib.request.urlopen(request, timeout=8) as response:
codes.append(response.status)
except urllib.error.HTTPError as exc:
codes.append(exc.code)
assert codes == [503, 503], codes
results['postgres'] = dict(status='PASS', readiness_and_request_status=codes)
finally:
command(['docker', 'compose', 'up', '-d', '--wait', 'db'])
wait(2, base)
active = any(row['running'] for row in containers('reconciler'))
command(['docker', 'compose', 'stop', 'reconciler'])
try:
command(['docker', 'compose', 'stop', 's3'])
time.sleep(4)
codes = []
paths = ['/health/ready', '/api/v1/attachments/'+attachment['id']+'/content']
for path in paths:
request = urllib.request.Request(base+path, headers={'Authorization': 'Bearer '+ctx['actor']['token']})
try:
with urllib.request.urlopen(request, timeout=10) as response:
codes.append(response.status)
except urllib.error.HTTPError as exc:
codes.append(exc.code)
assert codes == [503, 503], codes
results['s3'] = dict(status='PASS', readiness_and_download_status=codes)
finally:
command(['docker', 'compose', 'up', '-d', '--wait', 's3'])
wait(2, base)
if active:
command(['docker', 'compose', 'start', 'reconciler'])
page = c.call('GET', '/api/v1/chats/'+ctx['chat']['id']+'/messages', actor=ctx['reader'], schema='MessagePage')['json']
assert original in page['items']
c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
assert c.call('GET', '/api/v1/attachments/'+attachment['id']+'/content', actor=ctx['reader'])['bytes'] == content
results['recovery'] = 'old session and confirmed message preserved'
save('dependencies.json', results)
print('PASS: Redis fallback, PostgreSQL/S3 unready, old session and original preserved')
+43
View File
@@ -0,0 +1,43 @@
"""Останавливаем именно API, принявшую upload, и скачиваем оригинал другой репликой."""
import hashlib
import os
from pathlib import Path
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure
from scripts.stack import stop, wait, save, command
base = os.getenv('BASE_URL', 'http://localhost:8080')
c = Client(base, timeout=3)
alice, bob, _, chat = c.bootstrap()
data = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
response = c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
target = response['headers']['X-Instance-Id']
att = response['json']
c.message(alice, chat, '', attachments=[att['id']])
started = time.monotonic()
errors = 0
try:
stopped = stop(target)
while True:
try:
downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob)
alias = downloaded['headers']['X-Instance-Id']
assert alias != target and downloaded['bytes'] == data
assert hashlib.sha256(downloaded['bytes']).hexdigest() == att['sha256']
break
except Failure:
errors += 1
if time.monotonic()-started > 10:
raise
time.sleep(.2)
save('attachment_failover.json', dict(status='PASS', uploader=target, downloader=alias,
recovery_seconds=time.monotonic()-started, failed_attempts=errors,
original_size=len(data), checksum_verified=True, stop=stopped))
print('PASS: upload API stopped, original downloaded by other API with existing token')
finally:
command(['docker', 'compose', 'start', 'api'])
wait(2, base)
+58
View File
@@ -0,0 +1,58 @@
"""Проверки общего результата повтора через LB и несколько API."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Shared idempotency through LB')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
key = uuid.uuid4().hex
text = 'Один результат для конкурентных повторов'
def send(_):
return Client(args.base_url, args.ca_file).message(alice, chat, text, key=key)
with ThreadPoolExecutor(max_workers=16) as pool:
responses = list(pool.map(send, range(32)))
first = responses[0]['json']
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'replay differs')
instances = {r['headers']['X-Instance-Id'] for r in responses}
check(len(instances) >= 2, 'race did not cross API replicas')
path = '/api/v1/chats/' + chat['id'] + '/messages'
history = c.call('GET', path, actor=bob, schema='MessagePage')['json']['items']
check(len(history) == 1 and history[0] == first, 'concurrent replay created duplicates')
c.call('POST', path, 409, body={'text': 'Другой текст'}, actor=alice,
headers={'Idempotency-Key': key})
for invalid in ['', 'bad key', 'a' * 129]:
c.call('POST', path, 400, body={'text': text}, actor=alice,
headers={'Idempotency-Key': invalid})
c.call('POST', path, 400, body={'text': text}, actor=alice)
# Тот же ключ независим для другого пользователя и другого чата.
other = c.message(bob, chat, text, key=key)['json']
check(other['id'] != first['id'], 'key scope must include user')
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Second', 'member_ids': [alice['id']]}, schema='Chat')['json']
check(c.message(alice, chat2, text, key=key)['json']['id'] != first['id'], 'key scope must include chat')
# Пробелы JSON не изменяют смысл текста и не должны менять результат.
replay = c.call('POST', path, 201, actor=alice, raw=json.dumps({'text': text}, indent=2).encode(),
headers={'Content-Type': 'application/json', 'Idempotency-Key': key}, schema='Message')
check(replay['json'] == first, 'JSON formatting changed idempotency')
c.call('POST', path, 404, actor=eve, body={'text': text}, headers={'Idempotency-Key': key})
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
c.call('POST', path, 401, actor=alice, body={'text': text}, headers={'Idempotency-Key': key})
result = dict(status='PASS', concurrent_requests=32, observed_instances=len(instances),
messages_after_race=len(history), conflict=409, invalid_key=400, logout=401)
Path('evidence').mkdir(exist_ok=True)
Path('evidence/idempotency.json').write_text(json.dumps(result, indent=2))
print('PASS: shared idempotency, scope, conflict and auth on replay')
if __name__ == '__main__':
run(main)
+237
View File
@@ -0,0 +1,237 @@
"""Настоящие fault injection в PostgreSQL и S3, затем dry-run/apply/повтор."""
import argparse
import base64
from datetime import datetime, timezone
import hashlib
import http.client
import io
import json
import os
from pathlib import Path
import sys
import time
import urllib.request
import urllib.error
import uuid
import psycopg
from psycopg.rows import dict_row
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
from src.config import DATABASE_URL, S3_ENDPOINT, S3_BUCKET
from src.storage import ObjectStore, PREFIX, lock_attachment, unlock_attachment
from src.reconcile import run
def main():
p = argparse.ArgumentParser()
p.add_argument('mode', choices=['privacy', 'interrupted', 's3-failure', 'reconcile', 'cost'])
p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000])
p.add_argument('--failed-chat-id', type=uuid.UUID)
args = p.parse_args()
c = Client(os.getenv('BASE_URL', 'http://localhost:8080'), timeout=30)
alice, bob, _, chat = c.bootstrap()
data = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
digest = hashlib.sha256(data).hexdigest()
md5 = base64.b64encode(hashlib.md5(data).digest()).decode()
storage = ObjectStore()
records = {}
def scan(grace=20, apply=False):
return run(grace, apply, chat['id'])
def put(key):
storage.put(io.BytesIO(data), key, len(data), 'image/png', digest, md5)
def upload(name, code=201):
return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+name,
code, actor=alice, raw=data, headers={'Content-Type': 'image/png'},
schema='Attachment' if code == 201 else None)
def row(identifier):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
return db.execute('SELECT * FROM attachments WHERE id=%s', (identifier,)).fetchone()
def seed(status='ready', active=False):
identifier = uuid.uuid4()
key = PREFIX + chat['id'] + '/' + str(identifier)
put(key)
with psycopg.connect(DATABASE_URL) as db:
db.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,status,lease_until,created_at) VALUES (%s,%s,%s,'synthetic.png','image/png',"
"%s,%s,%s,%s,clock_timestamp()+%s*interval '1 second',clock_timestamp()-interval '15 minutes')",
(identifier, chat['id'], alice['id'], len(data), digest, key, status, 120 if active else -900))
return dict(id=identifier, key=key)
if args.mode == 'interrupted':
from urllib.parse import urlsplit
base = urlsplit(c.base)
connection = http.client.HTTPConnection(base.hostname, base.port or 80, timeout=5)
connection.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png')
connection.putheader('Authorization', 'Bearer '+alice['token'])
connection.putheader('Content-Type', 'image/png')
connection.putheader('Content-Length', '4096')
connection.endheaders()
connection.send(data[:16])
connection.close() # Не отправляем остаток и не завершаем корректный HTTP body.
time.sleep(.5)
with psycopg.connect(DATABASE_URL) as db:
count = db.execute('SELECT count(*) FROM attachments WHERE chat_id=%s', (chat['id'],)).fetchone()[0]
objects = list(storage.objects(PREFIX+chat['id']+'/'))
assert count == 0 and not objects
print(json.dumps(dict(status='PASS', sent_bytes=16, declared_bytes=4096, metadata=0, objects=0)))
return
if args.mode == 'privacy':
original = upload('private.png')['json']
key = row(original['id'])['object_key']
try:
urllib.request.urlopen(S3_ENDPOINT+'/'+S3_BUCKET+'/'+key, timeout=5)
except urllib.error.HTTPError as exc:
assert exc.code == 403
else:
raise AssertionError('Anonymous object GET succeeded')
buckets = storage.client.list_buckets().get('Buckets', [])
assert all(bucket['Name'] == S3_BUCKET for bucket in buckets), 'Foreign bucket listed'
# MinIO может вернуть отфильтрованный ListBuckets, а не AccessDenied.
try:
storage.client.create_bucket(Bucket='forbidden-'+uuid.uuid4().hex)
except Exception as exc:
assert getattr(exc, 'response', {}).get('Error', {}).get('Code') == 'AccessDenied'
else:
raise AssertionError('API credential has bucket administration rights')
from urllib.parse import urlsplit
endpoint = urlsplit(S3_ENDPOINT)
console = f'{endpoint.scheme}://{endpoint.hostname}:9001/'
try:
urllib.request.urlopen(console, timeout=2)
except urllib.error.URLError:
pass
else:
raise AssertionError('Admin console is reachable from another container')
print(json.dumps(dict(status='PASS', anonymous_get=403, bucket_list='filtered', create_bucket='AccessDenied',
console='not reachable', attachment_id=original['id'])))
return
if args.mode == 's3-failure':
# Оператор временно установил deny PUT policy; GET/LIST/readiness доступны.
upload('fault-s3.png', 503)
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
failed = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-s3.png'",
(chat['id'],)).fetchone()
assert failed and failed['status'] == 'uploading'
assert storage.head(failed['object_key']) is None
print(json.dumps(dict(status='PASS', http_status=503, metadata='uploading', object='absent',
chat_id=chat['id'], attachment_id=str(failed['id']))))
return
if args.mode == 'cost':
results, made = [], 0
for size in args.sizes:
while made < size:
seed()
made += 1
scan(600, False) # Прогрев обхода.
for repeat in range(3):
report = scan(600, False)
report.update(dataset_added=size, repetition=repeat+1)
results.append(report)
print(json.dumps(dict(status='PASS', scans=results)))
return
# Используемое вложение: сообщение и готовый объект должны пережить очистку.
if args.failed_chat_id:
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
failure = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-s3.png'",
(args.failed_chat_id,)).fetchone()
assert failure and failure['status'] == 'uploading'
db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes',"
"lease_until=clock_timestamp()-interval '15 minutes' WHERE id=%s", (failure['id'],))
dry_failure = run(20, False, args.failed_chat_id)
assert row(failure['id'])
fixed_failure = run(20, True, args.failed_chat_id)
assert row(failure['id']) is None and storage.head(failure['object_key']) is None
records['s3_failure_compensation'] = dict(dry_run=dry_failure, apply=fixed_failure, metadata_removed=True)
used = upload('used.png')['json']
message = c.message(bob, chat, '', attachments=[used['id']])['json']
used_row = row(used['id'])
with psycopg.connect(DATABASE_URL) as db:
db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s",
(used['id'],))
# Ошибка UPDATE ready ПОСЛЕ успешного S3 PUT. Только тестовое имя файла.
with psycopg.connect(DATABASE_URL) as db:
db.execute("CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS $$ "
"BEGIN IF NEW.filename='fault-db.png' AND NEW.status='ready' THEN "
"RAISE EXCEPTION 'synthetic DB failure'; END IF; RETURN NEW; END; $$")
db.execute('CREATE TRIGGER lab4_test_fail BEFORE UPDATE ON attachments '
'FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()')
try:
upload('fault-db.png', 503)
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db:
failed = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-db.png'",
(chat['id'],)).fetchone()
assert failed and failed['status'] == 'uploading' and storage.head(failed['object_key'])
immediate = scan(20, True)
assert row(failed['id']) and storage.head(failed['object_key'])
records['after_put_db_failure'] = dict(http_status=503, metadata='uploading', object='present',
immediate_cleanup=immediate)
with psycopg.connect(DATABASE_URL) as db:
db.execute("UPDATE attachments SET lease_until=clock_timestamp()-interval '15 minutes',"
"created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s", (failed['id'],))
finally:
with psycopg.connect(DATABASE_URL) as db:
db.execute('DROP TRIGGER IF EXISTS lab4_test_fail ON attachments')
db.execute('DROP FUNCTION IF EXISTS lab4_fail_ready()')
missing = upload('missing.png')['json']
missing_message = c.message(alice, chat, '', attachments=[missing['id']])['json']
missing_row = row(missing['id'])
storage.delete(missing_row['object_key'])
with psycopg.connect(DATABASE_URL) as db:
db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s",
(missing['id'],))
orphan_key = PREFIX + chat['id'] + '/' + str(uuid.uuid4())
put(orphan_key)
active = seed('uploading', active=True)
locked = seed('uploading')
time.sleep(20.2) # Настоящий возраст объекта S3, не фиктивный LastModified.
fresh_key = PREFIX + chat['id'] + '/' + str(uuid.uuid4())
put(fresh_key)
fresh = upload('fresh.png')['json']
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as guard:
lock_attachment(guard, locked['id']); guard.commit()
try:
dry = scan(20, False)
assert storage.head(orphan_key) and row(failed['id'])
assert row(missing['id'])['status'] == 'ready'
applied = scan(20, True)
assert storage.head(orphan_key) is None
assert row(failed['id']) is None and storage.head(failed['object_key']) is None
assert row(missing['id'])['status'] == 'missing'
assert row(used['id']) and storage.head(used_row['object_key'])
assert row(active['id']) and storage.head(active['key'])
assert row(locked['id']) and storage.head(locked['key'])
assert storage.head(fresh_key) and row(fresh['id'])
c.call('GET', '/api/v1/attachments/'+missing['id']+'/content', 503, actor=bob)
# После возврата именно исходных bytes reconciler может вернуть ready.
put(missing_row['object_key'])
repaired = scan(20, True)
assert row(missing['id'])['status'] == 'ready'
assert c.call('GET', '/api/v1/attachments/'+missing['id']+'/content', actor=bob)['bytes'] == data
repeated = scan(20, True)
assert not repeated['actions'], 'Repeat still changes already corrected state'
page = c.call('GET', '/api/v1/chats/'+chat['id']+'/messages', actor=bob, schema='MessagePage')['json']
assert message in page['items'] and missing_message in page['items']
finally:
unlock_attachment(guard, locked['id']); guard.commit()
records.update(status='PASS', dry_run=dry, apply=applied, restore=repaired, repeat=repeated,
used_and_fresh_preserved=True, active_lease_and_lock_preserved=True,
missing_original_restored_by_operator=True)
print(json.dumps(records))
if __name__ == '__main__':
main()
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Black-box grading evidence beyond the public smoke test."""
from concurrent.futures import ThreadPoolExecutor
import json
import sys
import uuid
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
def collect_messages(client, actor, chat, page_size=17):
items = []
cursor = None
while True:
path = f"/api/v1/chats/{chat['id']}/messages?limit={page_size}"
if cursor:
path += "&cursor=" + quote(cursor, safe="")
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items
def main():
args = arguments("Student negative, pagination and concurrency tests")
client = Client(args.base_url, args.ca_file)
alice, bob, outsider, chat = client.bootstrap()
client.call(
"POST",
"/api/v1/users",
400,
raw=b'{"username":',
headers={"Content-Type": "application/json"},
)
client.call(
"POST",
"/api/v1/users",
415,
raw=b'{"username":"wrong_media","display_name":"Wrong"}',
headers={"Content-Type": "text/plain"},
)
client.call("POST", "/api/v1/users", 409, body=alice["registration"])
client.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
400,
actor=alice,
body={"text": " \t"},
)
client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=outsider)
prefix = "race-" + uuid.uuid4().hex + "-"
def send(worker_id):
local = Client(args.base_url, args.ca_file)
return [
local.message(alice, chat, f"{prefix}{worker_id}-{index}")["json"]
for index in range(10)
]
with ThreadPoolExecutor(max_workers=10) as pool:
created = [message for batch in pool.map(send, range(10)) for message in batch]
check(len(created) == 100, "not all concurrent requests succeeded")
check(len({message["id"] for message in created}) == 100, "duplicate message ids returned")
history = collect_messages(client, bob, chat)
concurrent_history = [message for message in history if message["text"].startswith(prefix)]
check(len(concurrent_history) == 100, "history does not contain exactly 100 concurrent messages")
check(len({message["id"] for message in concurrent_history}) == 100, "history contains duplicates")
check(
history == sorted(history, key=lambda message: (message["created_at"], uuid.UUID(message["id"]).int)),
"multi-page history is out of order",
)
print("PASS: malformed JSON/media type, duplicate username, blank message, outsider access, pagination")
print("PASS: 10 clients created exactly 100 unique messages")
if __name__ == "__main__":
run(main)