From 16709c91b3e06028b10d72095fb1e5d2bacbc5ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A8=D0=B5=D0=BF?= =?UTF-8?q?=D1=82=D1=83=D0=BD=D0=BE=D0=B2?= Date: Fri, 16 Oct 2026 22:47:54 +0300 Subject: [PATCH] lab4 --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 30 + Makefile | 32 + REPORT.md | 420 +- compose.yaml | 143 + evidence/attachment_failover.json | 247 + evidence/containers_after_restart.json | 118 + evidence/containers_before.json | 118 + evidence/containers_final.json | 118 + evidence/dependencies.json | 21 + evidence/experiment_environment.json | 123 + evidence/experiment_summary.json | 37 + evidence/failures.json | 5359 ++++++++++++++++ evidence/idempotency.json | 9 + evidence/last_stop.json | 238 + evidence/local_availability.txt | 2 + evidence/local_check.txt | 2 + evidence/local_dependencies.txt | 2 + evidence/local_environment.txt | 31 + evidence/local_experiment.txt | 11 + evidence/local_expiry.txt | 20 + evidence/local_failover.txt | 4 + evidence/local_file_failover.txt | 2 + evidence/local_files.txt | 5 + evidence/local_final_ready.txt | 2 + evidence/local_final_smoke.txt | 2 + evidence/local_persistence.txt | 3 + evidence/local_ready.txt | 2 + evidence/local_services.txt | 10 + evidence/local_smoke.txt | 2 + evidence/local_smoke_after_failures.txt | 2 + evidence/local_start.txt | 260 + evidence/local_student_checks.txt | 10 + evidence/local_validate.txt | 2 + evidence/private_bucket.json | 8 + evidence/reconcile_cost.json | 194 + evidence/reconcile_cost.png | Bin 0 -> 36285 bytes evidence/reconcile_faults.json | 221 + evidence/rolling.json | 7828 +++++++++++++++++++++++ evidence/rolling_containers.json | 296 + evidence/s3_write_failure.json | 8 + evidence/upload_interrupted.json | 7 + evidence/upload_memory.json | 443 ++ evidence/upload_memory.png | Bin 0 -> 27582 bytes infra/haproxy.cfg | 37 + infra/minio-entrypoint.sh | 5 + infra/s3-api-policy.json | 10 + infra/s3-no-put-policy.json | 10 + infra/s3.Dockerfile | 27 + migrations/001_initial.sql | 36 + migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 41 + requirements-experiment.txt | 2 + requirements.txt | 18 + scripts/benchmark.py | 156 + scripts/bootstrap_s3.sh | 32 + scripts/experiment.py | 184 + scripts/healthcheck.py | 12 + scripts/monitor.py | 44 + scripts/restart.sh | 11 + scripts/restore-api.sh | 5 + scripts/setup.sh | 23 + scripts/stack.py | 129 + scripts/start.sh | 5 + scripts/stop-one.sh | 5 + scripts/test_expiry.sh | 11 + scripts/wait_ready.py | 17 + src/__init__.py | 0 src/attachments.py | 157 + src/config.py | 22 + src/errors.py | 6 + src/main.py | 637 ++ src/migrate.py | 26 + src/reconcile.py | 193 + src/storage.py | 57 + student-tests/attachments.py | 87 + student-tests/availability.py | 64 + student-tests/checks.py | 90 + student-tests/database.py | 61 + student-tests/dependencies.py | 71 + student-tests/file_failover.py | 43 + student-tests/idempotency.py | 58 + student-tests/storage_scenario.py | 237 + student-tests/test_api.py | 87 + 85 files changed, 19098 insertions(+), 22 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/attachment_failover.json create mode 100644 evidence/containers_after_restart.json create mode 100644 evidence/containers_before.json create mode 100644 evidence/containers_final.json create mode 100644 evidence/dependencies.json create mode 100644 evidence/experiment_environment.json create mode 100644 evidence/experiment_summary.json create mode 100644 evidence/failures.json create mode 100644 evidence/idempotency.json create mode 100644 evidence/last_stop.json create mode 100644 evidence/local_availability.txt create mode 100644 evidence/local_check.txt create mode 100644 evidence/local_dependencies.txt create mode 100644 evidence/local_environment.txt create mode 100644 evidence/local_experiment.txt create mode 100644 evidence/local_expiry.txt create mode 100644 evidence/local_failover.txt create mode 100644 evidence/local_file_failover.txt create mode 100644 evidence/local_files.txt create mode 100644 evidence/local_final_ready.txt create mode 100644 evidence/local_final_smoke.txt create mode 100644 evidence/local_persistence.txt create mode 100644 evidence/local_ready.txt create mode 100644 evidence/local_services.txt create mode 100644 evidence/local_smoke.txt create mode 100644 evidence/local_smoke_after_failures.txt create mode 100644 evidence/local_start.txt create mode 100644 evidence/local_student_checks.txt create mode 100644 evidence/local_validate.txt create mode 100644 evidence/private_bucket.json create mode 100644 evidence/reconcile_cost.json create mode 100644 evidence/reconcile_cost.png create mode 100644 evidence/reconcile_faults.json create mode 100644 evidence/rolling.json create mode 100644 evidence/rolling_containers.json create mode 100644 evidence/s3_write_failure.json create mode 100644 evidence/upload_interrupted.json create mode 100644 evidence/upload_memory.json create mode 100644 evidence/upload_memory.png create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-entrypoint.sh create mode 100644 infra/s3-api-policy.json create mode 100644 infra/s3-no-put-policy.json create mode 100644 infra/s3.Dockerfile create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 requirements-experiment.txt create mode 100644 requirements.txt create mode 100644 scripts/benchmark.py create mode 100755 scripts/bootstrap_s3.sh create mode 100644 scripts/experiment.py create mode 100644 scripts/healthcheck.py create mode 100644 scripts/monitor.py create mode 100755 scripts/restart.sh create mode 100755 scripts/restore-api.sh create mode 100755 scripts/setup.sh create mode 100644 scripts/stack.py create mode 100755 scripts/start.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/test_expiry.sh create mode 100644 scripts/wait_ready.py create mode 100644 src/__init__.py create mode 100644 src/attachments.py create mode 100644 src/config.py create mode 100644 src/errors.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/reconcile.py create mode 100644 src/storage.py create mode 100644 student-tests/attachments.py create mode 100644 student-tests/availability.py create mode 100644 student-tests/checks.py create mode 100644 student-tests/database.py create mode 100644 student-tests/dependencies.py create mode 100644 student-tests/file_failover.py create mode 100644 student-tests/idempotency.py create mode 100644 student-tests/storage_scenario.py create mode 100644 student-tests/test_api.py diff --git a/.dockerignore b/.dockerignore index bc16e55..a097d39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ .env.* !.env.example secrets +.secrets certs volumes data diff --git a/.gitignore b/.gitignore index b5a7189..9e8c10d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env.* !.env.example secrets/ +.secrets/ certs/private/ *.key *.p12 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..99ae93d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.12.10-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --requirement requirements.txt \ + && groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app + +COPY --chown=app:app src ./src +COPY --chown=app:app migrations ./migrations +COPY --chown=app:app scripts ./scripts +COPY --chown=app:app student-tests ./student-tests +COPY --chown=app:app tests ./tests +COPY --chown=app:app contracts ./contracts +COPY --chown=app:app lab.json ./lab.json +RUN mkdir -p evidence && chown app:app evidence + +USER 10001:10001 +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD ["python", "/app/scripts/healthcheck.py"] + +CMD ["sh", "-c", "exec uvicorn src.main:app --host \"${APP_HOST}\" --port \"${APP_PORT}\" --workers 1 --timeout-graceful-shutdown 8"] diff --git a/Makefile b/Makefile index 5d86d17..7c19482 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,35 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: setup start ready student-checks test-availability test-expiry test-dependencies test-files test-file-failover test-storage experiment reconcile stop +setup: + bash scripts/setup.sh +start: + bash scripts/start.sh +ready: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/stack.py wait --count 2 +student-checks: + $(PYTHON) student-tests/test_api.py --base-url "$(BASE_URL)" + $(PYTHON) student-tests/checks.py --base-url "$(BASE_URL)" + $(PYTHON) student-tests/idempotency.py --base-url "$(BASE_URL)" + docker exec "$$(docker compose ps --status running -q api | head -n 1)" python student-tests/database.py +test-files: + $(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py --mode privacy +test-file-failover: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/file_failover.py +test-storage: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py --mode faults +test-availability: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/availability.py +test-dependencies: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/dependencies.py +test-expiry: + bash scripts/test_expiry.sh +experiment: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/experiment.py +reconcile: + docker compose run --rm --no-deps reconciler python -m src.reconcile +stop: + docker compose down diff --git a/REPORT.md b/REPORT.md index 9ad2b84..dba4d91 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,413 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4 — S3 и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Максим Шептунов, БВТ2301. +- Целевая оценка: 5; трек 2, «Согласованность двух хранилищ». +- Skeleton: `ec540a1a725862a8619fcb758daec4ad197f0dd5`. +- Проверки и эксперимент: 16.10.2026, локальный Docker-стенд в WSL2. +- Помощь ИИ: помощь с реализацией, проверками и разбором результатов. + +Ниже описано решение и результаты моего запуска. Числа рассчитаны по +`evidence/local_*` и исходным JSON; результаты эксперимента сохранены в evidence. +README, публичные tests и контракт skeleton сохранены. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2/BuildKit, Python 3.10+ на хосте и Linux/amd64 +для закреплённых S3-бинарников. В Windows — WSL Ubuntu с Docker Desktop. +Порт 8080 должен быть свободен. Сборка требует доступа к PyPI, Docker Hub +для базовых образов и GitHub Releases для MinIO/mc. -## Архитектура и решения +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements-dev.txt -r requirements-experiment.txt +chmod +x scripts/*.sh +make setup +make start +make ready +make check +make validate +make test +make student-checks +make test-files +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +bash scripts/restore-api.sh +make test-file-failover +make test-availability +make test-dependencies +make test-expiry +make experiment +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +API-зависимости устанавливаются внутри Docker из `requirements.txt`. +`setup.sh` генерирует `.env` и секреты в `.secrets`, не выводя значения. +Папка секретов 700; файлы, читаемые пользователем API через mount, 644 внутри +закрытой папки; root-credentials MinIO и пароль PG — 600. Secrets/.env исключены +из Git и build context. Настройки повторно не генерируют новые credentials. -## Доказательства критериев +Сервис `migrate` применяет SQL и завершается, `s3-bootstrap` создаёт приватный +bucket и ограниченного пользователя API и тоже завершается. Exited(0) для них +нормален. Затем работают две API, LB, PG, Redis, MinIO и reconciler. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Общий запуск выполняется без Compose `--wait`: у фонового reconciler нет healthcheck. +После `up` скрипт отдельно ждёт две готовые API через LB (`stack.py wait --count 2`). + +`make stop` сохраняет volumes. Намеренное удаление всех учебных данных: +`docker compose down -v`; оно не используется в тестах сохранности. + +Python-образ 3.12.10 slim-bookworm, PG 16.9 bookworm, Redis 7.4.3 bookworm, +HAProxy 3.0.11 bookworm. MinIO RELEASE.2025-04-22T22-12-26Z и mc +RELEASE.2025-04-16T18-13-26Z собираются в локальные образы через `infra/s3.Dockerfile`: +официальные GitHub Release binaries, проверка закреплённого SHA-256 до запуска. +Это не требует доступа к переставшим быть общедоступными образам minio/minio. +Все библиотеки закреплены по версиям. + +## Окружение измерений + +| Параметр | Значение | +| --- | --- | +| Дата | 16.10.2026 | +| Система | WSL2, Linux 5.15.146.1-microsoft-standard-WSL2, x86_64 | +| Ресурсы Docker | 16 логических CPU; 8187711488 bytes RAM, около 7.63 GiB | +| Docker Engine сервера | 26.1.1, Docker Desktop | +| Docker CLI | 28.0.1 | +| Docker Compose | 2.27.0-desktop.2 | +| Python на хосте | 3.10.12 | +| API | 2 реплики, каждая 1 CPU / 512 MiB, DB_POOL_MAX=8 | +| Reconciler | 0.5 CPU / 256 MiB | + +Источник: `evidence/local_environment.txt`, `experiment_environment.json`, +`containers_before.json`, `containers_final.json`. CPU/RAM-лимиты API также +сохранены отдельно для каждого повтора upload в `upload_memory.json`. +После всех сценариев снова видны две готовые API; финальный smoke прошёл. + +## Перенос и архитектура + +Основа — фактическое решение lab3 после проверок: PostgreSQL, общие сессии, +Argon2id, Redis-кеш, keyset-пагинация, две API за HAProxy, X-Instance-Id, +идемпотентность сообщений, рестарт и rolling. Сохранены исправления выбора +реально работающего контейнера вместо жёсткого индекса API и Redis fallback. +Исследование масштабирования lab3 повторно не запускается: в lab4 выбран другой трек. + +Также перенесено исправление проверки cursor из lab3: JSON должен быть объектом, +created_at и id — строками; неверные типы возвращают 400 invalid_request. +Регрессионные проверки охватывают историю и список чатов на обеих API. + +Добавлены MinIO, bootstrap bucket/IAM, вложения, третья миграция, +reconciler и сценарии файлов/RAM/двух хранилищ. Описание переноса включено +в этот отчёт, отдельного Markdown-файла миграции нет. + +Клиент → HAProxy → API ×2 → PostgreSQL + приватный S3. +PostgreSQL хранит метаданные, ACL, связи с сообщениями и результаты повторов; +S3 — оригинальные bytes. Redis — кеш публичных полей профиля, не источник сессий. +Для download авторизация и участие проверяются снова; случайный ключ не заменяет ACL. +Наружу опубликован только `127.0.0.1:8080` у LB. У API/S3 нет опубликованных портов. +Административная консоль MinIO выключена и привязана к loopback контейнера. + +API и reconciler используют общего ограниченного S3-пользователя, отдельного +от root. Его права: GET/PUT/DELETE в `messenger-attachments/originals/*` и LIST +этого namespace. Нет прав создания +bucket или администрирования IAM. Root credentials доступны S3 и bootstrap, +но не API/reconciler. Bootstrap повторно безопасен: bucket/private policy/user +создаются или приводятся к заданной конфигурации. + +### Контракт файла и сообщения + +POST attachment принимает raw bytes PNG/JPEG, не multipart. Лимит 10485760 bytes +проверяется по Content-Length до чтения и по счётчику потока, включая chunked body. +Проверяются отображаемое имя 1–255 символов, отсутствие разделителей пути +и управляющих символов, формат по содержимому и соответствие MIME. +Через Pillow проверяются структура и полноценное декодирование; площадь +ограничена 40000000 пикселей до load. PNG text metadata также ограничены. +Повреждение/MIME — 415, byte/pixel limit — 413, опасное имя — 400. + +Object key назначает сервер: `originals//`. +Filename не используется как путь. SHA-256 и размер описывают исходные bytes. +Валидация не перекодирует оригинал. Upload возвращает 201 ready только после +успешного S3 PUT и COMMIT метаданных. Публичный Attachment не содержит object key +или S3 URL. Download — авторизованный StreamingResponse с chunks 64 KiB, без redirect. + +Message содержит `attachment_ids`. Можно передать пустой текст при хотя бы одном +вложении; все ID уникальны и принадлежат этому чату. Любой участник может ссылаться +на доступное вложение, независимо от uploader. Чужое/неизвестное — 404, +неготовое — 409. Message и связи вложений сохраняются одной транзакцией. +Отложенный SQL trigger проверяет текст/связи при COMMIT. + +Идемпотентность lab3 сохранена. В хеш включается упорядоченный список непустых +attachment_ids; отсутствующее поле и [] равны. Для старых текстовых запросов +хеш не меняется. Миграция дополняет сохранённые старые ответы полем [] и +не удаляет сообщения/сессии. Разные JSON key order/whitespace не меняют смысл. + +### Память, таймауты, готовность + +Тело пишется потоково во временный bounded spool; это не постоянное хранилище +оригинала в API. TemporaryFile закрывается после запроса, download читает S3. +На процесс допускаются четыре активных upload, декодируется только одно +изображение одновременно. Остальные клиенты ждут, а не декодируют параллельно. +Нет `await request.body()` на весь файл. Дисковая запись/валидация выполняются +в thread pool, чтобы не блокировать event loop. + +API — по 1 CPU/512 MiB: RAM увеличена относительно lab3 для разрешённых больших +изображений. PG 1 CPU/512 MiB, MinIO 1 CPU/512 MiB, Redis/LB по 0.5 CPU/128 MiB, +reconciler 0.5 CPU/256 MiB. Фактические значения фиксируются через Docker inspect. +S3 connect/read timeout 2/5 s, SDK не повторяет PUT автоматически; upload timeout +45 s, LB server timeout 50 s. SQL/lock/pool timeouts 3/2/2 s для API. +Uvicorn graceful shutdown 8 s, Docker grace 10 s. Fault hook намеренно останавливает +выбранную API; после неопределённого message POST повторяется прежний ключ. + +Liveness проверяет процесс. Readiness проверяет PG и доступ к приватному S3 namespace; +Redis не критичен. HAProxy опрашивает ready с interval 1 s, check timeout 1 s, +fall 2/rise 1; Docker DNS обнаруживает актуальные реплики. Ориентир обнаружения +отказа — несколько секунд; фактическую границу ≤10 s измеряет failover. +Проверка LIST не обещает отсутствие любого будущего отказа диска или PUT. + +`.env` управляет APP_PORT, SESSION_TTL_SECONDS, API_CPU, API_MEMORY, DB_POOL_MAX. +S3 endpoint/bucket/region и пути credentials заданы в compose; MAX_* не превышают +контрактные значения. API_REPLICAS из исходного .env.example заменено Compose scale; +SESSION_STORE фиксирован как PG, CACHE_URL задан напрямую, TTL ключа — 24 h. + +## Согласованность PostgreSQL и S3 + +Общей транзакции между ними нет. Протокол upload: + +1. Полностью принять и проверить bounded файл. +2. Взять PostgreSQL session advisory lock на UUID вложения. +3. Сохранить устойчивую запись uploading с lease 120 s и COMMIT. +4. Выполнить S3 PUT с transport Content-MD5 и метаданными SHA-256. +5. Обновить ready и COMMIT, затем разрешить 201. +6. Освободить lock. После ошибки остаётся uploading для безопасной компенсации. + +Если PUT прошёл, а запись ready не прошла, объект учтён durable intent. +Если PUT отказал, остаётся uploading без объекта. Обрыв HTTP до полной валидации +не создаёт объектов/метаданных. Upload не имеет гарантии идемпотентности: +повтор его POST может создать другое вложение, как допускает контракт. + +Reconciler периодически запускается отдельным контейнером: каждые 60 s, +grace 600 s. Разовый запуск по умолчанию dry-run: + +```bash +make reconcile +docker compose run --rm --no-deps reconciler python -m src.reconcile --apply +``` + +Можно ограничить обход одним чатом через `--chat-id UUID`. Используются LIST, +повторный HEAD и свежий запрос PG под общей блокировкой, а не только старый snapshot. +Недоступность зависимости не считается отсутствием объекта. Dry-run не меняет данные. + +| Состояние | Безопасное действие | +| --- | --- | +| Объект без метаданных | Удалить только после grace, повторной проверки и отсутствия активного lock | +| Незавершённый uploading | Ждать lease/grace; при отсутствии ссылок очистить старый объект и запись | +| Используемый ready | Сохранить объект и метаданные | +| Слишком свежий объект/запись | Пропустить | +| Ready, но bytes отсутствуют | Сохранить метаданные/ссылки, пометить missing; API сообщает 503 | +| Исходные bytes возвращены | Проверить размер/SHA-256, затем восстановить ready | + +Reconciler не восстанавливает потерянные bytes из ничего. В сценарии оператор +возвращает оригинал из fixture; только после проверки он снова становится ready. +При исправлении missing ссылки из сообщений не удаляются. Повтор применённой +сверки не должен повторно менять уже исправленное состояние. + +## Проверки и доказательства + +| Критерий | Команда | Результат моего стенда | Evidence | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Структура/контракт | make check; make validate | PASS; структура и схемы, отдельно от HTTP | local_check.txt, local_validate.txt | +| Обязательный API | make test | PASS до/после отказов и в финале | local_smoke.txt, local_smoke_after_failures.txt, local_final_smoke.txt | +| Минимум lab3, SQL и ключи | make student-checks | PASS; 100 сообщений от 10 клиентов, 32 повтора → 1 сообщение | local_student_checks.txt, idempotency.json | +| Типы cursor | make student-checks | PASS; 400 invalid_request на обеих API, история и список чатов | local_student_checks.txt | +| PNG/JPEG, лимиты, ACL | make test-files | PASS; bytes/pixels, MIME, filename, чужие ID/чаты, replay | local_files.txt | +| Приватность bucket и права | make test-files; make experiment | PASS; anonymous GET 403, CreateBucket AccessDenied, console недоступна | private_bucket.json | +| Прерывание body | make test-files; make experiment | PASS; отправлено 16 из заявленных 4096 bytes, 0 объектов и записей | upload_interrupted.json | +| Полный restart | make test-persistence ACTION_SCRIPT=scripts/restart.sh | PASS; старый токен, сообщения и оригинал сохранились | local_persistence.txt, containers_before.json, containers_after_restart.json | +| Отказ выбранной API | make test-failover ACTION_SCRIPT=scripts/stop-one.sh | PASS; первая успешная запись после hook за 1.524 s | local_failover.txt | +| Стоп uploader → другая API | make test-file-failover | PASS; 111817 bytes, SHA-256 совпал, 1.370 s | attachment_failover.json | +| Каждая API и rolling | make test-availability | PASS; подтверждённые сообщения и оригиналы сохранены | local_availability.txt, failures.json, rolling.json, rolling_containers.json | +| Redis/PG/S3 outage | make test-dependencies | PASS; Redis fallback, PG/S3 → 503, затем восстановление | dependencies.json, local_dependencies.txt | +| Истечение сессии | make test-expiry | PASS; TTL=2 s, затем обычная конфигурация возвращена | local_expiry.txt | +| Ошибки двух хранилищ | make experiment, этап faults | PASS; dry-run/apply/restore/repeat и защита вложений | reconcile_faults.json, s3_write_failure.json | +| 10 concurrent upload, память | make experiment, этап memory | PASS; 6 серий, 60/60 оригиналов, 0 ошибок | upload_memory.json, upload_memory.png | +| Цена обхода | make experiment, этап cost | PASS; 100/500/1000, по 3 измерения после прогрева | reconcile_cost.json, reconcile_cost.png, experiment_summary.json | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Сохранность и переключение реплик -## Эксперименты и отказы +Snapshots до и после restart показывают новый `started_at` у PG, Redis, S3, +LB и обеих API. Для PG/Redis/S3 сохранены container ID; API/LB пересозданы. +Публичный persistence повторно прочитал данные и оригинал с прежней сессией. +`attachment_failover.json` фиксирует остановленный контейнер uploader и другой +`X-Instance-Id` скачавшей API. Контейнер uploader действительно остановлен в `after`. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Публичный failover: 122 успешные записи, 3 неуспешные попытки, первая успешная +запись после старта hook через 1.524 s. Отдельный файловый failover: 1.370 s, +0 неуспешных попыток скачивания, оригинал 111817 bytes совпал побайтно и по SHA-256. +Это разные сценарии и разные точки отсчёта, а не одно измерение. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Нагрузка | Успешных запросов | Неуспешных попыток | Доля ошибок | RPS успешных | p95, ms | Подтверждено / найдено сообщений | +| --- | --- | --- | --- | --- | --- | --- | +| Стоп API №1 | 2034 | 38 | 1.834% | 126.663 | 86.066 | 1228 / 1228 | +| Стоп API №2 | 1995 | 56 | 2.730% | 124.562 | 82.536 | 1212 / 1212 | +| Rolling двух API | 7020 | 176 | 2.446% | 200.515 | 71.279 | 4254 / 4254 | -## Ограничения и следующие шаги +Для остановки каждой API верхняя оценка восстановления по секундным bucket — +2.998 s, меньше требуемых 10 s. Во всех трёх сценариях оригинал также проверен. +Ошибки во время переключения не скрываются: клиенты повторяют неопределённые +message POST с прежним Idempotency-Key. Все подтверждения сверены с историей. +Нулевая потеря подтверждений не означает нулевую долю неуспешных попыток. +В rolling сохранены шаги замены контейнеров; это не бесшовное обновление без ошибок. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +При остановке Redis профиль и запись сообщения продолжили работать; серия +контрольных запросов заняла 7.079 s. Это длительность серии с fallback, не latency +одного запроса. При остановке PG readiness и запрос профиля вернули 503; +при остановке S3 readiness и download вернули 503. После возврата зависимостей +сохранились прежний токен, сообщение и оригинал. + +## Эксперимент на 5 + +Гипотеза: durable intent вместе с lease/lock/grace и повторной сверкой позволяет +исправлять разрыв между S3 и PG, сохраняя свежие, активные и используемые вложения. + +Ошибки реальные: временное удаление PUT из IAM-прав API; PostgreSQL trigger, +отказывающий UPDATE ready после S3 PUT; объект без метаданных; удаление bytes +используемого вложения. Все данные синтетические. Trigger удаляется в finally, +обычная IAM policy возвращается, фоновый reconciler на время сценария остановлен. +В тесте grace 20 s и контролируемые старые timestamps PG; возраст S3 orphan +измеряется настоящим ожиданием. Это не настройка production grace. + +Проверяются dry-run, apply, сохранность свежих/используемых объектов и активного +lease/lock, возвращение оригинала, повторный apply и сохранность истории. +Автоматическое восстановление S3/PG-кластеров не заявляется. + +RAM-серия: два размера noise PNG, 256×256 и 1024×1024; по десять одновременно +отправляющих клиентов, три повтора. API пересоздаются между случаями для нового +high-water mark. Измеряется peak RSS процесса API и cgroup peak при наличии +счётчика ядра, а не только редкие snapshots. Фиксируются реальные лимиты, aliases, +RPS конечного burst, ошибки, p50/p95/p99 и проверка оригиналов второго участника. +Burst RPS не является предельной sustained throughput сервера. + +Цена обхода: отдельный синтетический чат, 100/500/1000 объектов и метаданных, +прогрев полного прохода и по три dry-run. Он выполняется в роли reconciler +с теми же CPU/RAM-лимитами. Сохраняются длительность, LIST/HEAD/GET/bytes counters, +process RSS high-water и полный список решений. Raw сохраняется в JSON, +сводка — в experiment_summary.json, графики — в PNG. + +### Результаты согласованности + +Источник: `evidence/reconcile_faults.json`, `s3_write_failure.json`. + +| Сценарий | Наблюдение и результат | +| --- | --- | +| S3 PUT запрещён IAM policy | API вернула 503; в PG осталась uploading, объекта нет | +| Компенсация отказа PUT | Dry-run сохранил запись, apply удалил просроченную запись без объекта; 0.035159 / 0.027420 s | +| PG отказала при UPDATE ready после PUT | API вернула 503; uploading и объект существуют. Немедленная сверка за 0.027833 s ничего не удалила | +| Старый orphan + незавершённый upload + missing используемого файла | Dry-run нашёл delete_orphan, delete_stale_upload, mark_missing; applied=false для всех, данные не изменены | +| Apply того же плана | Те же три действия применены; ссылки и метаданные missing сохранены | +| Свежие, используемые, активная lease и удерживаемый lock | Защитные проверки пройдены, вложения не удалены | +| Возврат оригинала оператором | После возврата fixture выполнен GET 111817 bytes и проверен SHA-256; restore_ready | +| Повторный apply | actions пуст; новых исправлений нет, история и исходные bytes доступны | + +| Проход | Время, ms | Действий | +| --- | --- | --- | +| dry_run | 47.514 | 3 | +| apply | 53.742 | 3 | +| restore | 72.341 | 1 | +| repeat | 37.725 | 0 | + +Ошибки внедрены на обеих сторонах реальными отказами IAM/SQL, а не подменой +ответов клиента. Исправление безопасно в проверенных состояниях: свежие и +используемые файлы не удаляются, пропавшие bytes не подменяются выдуманным файлом. +В тесте отдельно подтверждён возврат оригинала оператором и проверка его checksum. + +### Результаты десяти одновременных загрузок + +Два размера PNG, по три серии из 10 одновременно стартующих клиентов. +Каждая серия достигла обеих API, успешно загрузила 10 файлов и скачала их +вторым участником с проверкой исходных bytes и SHA-256. Итого 60/60, ошибок 0. +API пересоздавались перед каждой серией; лимиты каждой реплики — 1 CPU/512 MiB. + +| PNG | Размер, bytes | Повтор | Burst RPS | p95 upload, ms | Max RSS одной API, MiB | Max cgroup peak одной API, MiB | +| --- | --- | --- | --- | --- | --- | --- | +| 256×256 | 197173 | 1 | 50.001 | 199.251 | 129.832 | 124.703 | +| 256×256 | 197173 | 2 | 69.096 | 143.348 | 111.172 | 116.461 | +| 256×256 | 197173 | 3 | 66.421 | 149.041 | 111.160 | 106.207 | +| 1024×1024 | 3151221 | 1 | 24.810 | 402.166 | 127.586 | 122.039 | +| 1024×1024 | 3151221 | 2 | 23.200 | 429.619 | 125.387 | 131.141 | +| 1024×1024 | 3151221 | 3 | 22.433 | 444.786 | 125.195 | 129.051 | + +Медианы по трём сериям: для 256×256 — 66.421 burst RPS и p95 149.041 ms; +для 1024×1024 — 23.200 burst RPS и p95 429.619 ms. Здесь медиана p95 — +медиана трёх отдельных p95, а не общий p95 всех 30 запросов. + +Максимальный process RSS среди всех серий — 129.832 MiB, максимальный cgroup +peak — 131.141 MiB, заметно ниже лимита 512 MiB на API. Это максимум одной +реплики, а не одновременная сумма памяти двух контейнеров. Cgroup peak измерен +через cgroup v1 `memory.max_usage_in_bytes`; RSS получен через `ru_maxrss`. + +Пик включает запуск процесса, регистрацию и вход перед upload, поэтому более +крупный файл не обязан дать больший абсолютный RSS. Максимальный прирост +RSS high-water между снимками before/after составил 4.797 MiB для 256×256 +и 18.668 MiB для 1024×1024. Это прирост счётчика максимума, не точное выделение +памяти только под содержимое файла. Burst RPS описывает короткую серию из 10 +запросов и не заменяет продолжительный throughput-тест. + +![Пик памяти API при десяти одновременных загрузках](evidence/upload_memory.png) + +### Стоимость обхода + +Отдельный контейнер роли reconciler, лимиты 0.5 CPU/256 MiB. На каждом размере +выполнен прогрев и три измеренных dry-run. Источник: `evidence/reconcile_cost.json`. + +| Объектов и записей PG | Три прохода, s | Медиана, s | LIST / HEAD за проход | GET содержимого / bytes | Max RSS процесса, MiB | +| --- | --- | --- | --- | --- | --- | +| 100 | 0.692223, 0.696161, 0.808335 | 0.696161 | 1 / 100 | 0 / 0 | 68.250 | +| 500 | 3.238328, 3.476147, 3.516724 | 3.476147 | 1 / 500 | 0 / 0 | 69.797 | +| 1000 | 6.510459, 6.586175, 6.482201 | 6.510459 | 1 / 1000 | 0 / 0 | 71.859 | + +На каждом проходе просмотрены ровно N объектов и N метаданных, исправлений 0. +Содержимое здоровых файлов не скачивалось: 0 GET и 0 bytes означают именно +содержимое объектов, а не отсутствие сетевого трафика LIST/HEAD и SQL. + +При росте количества объектов в 10 раз медианное время увеличилось примерно +в 9.35 раза: 0.696161 → 6.510459 s. На измеренном диапазоне стоимость близка +к линейной: один LIST и N HEAD, плюс запросы/блокировки PostgreSQL. Лишнее +чтение оригиналов не требуется для обычного прохода; полный SHA-256 проверяется +при восстановлении missing или несогласованности HEAD. + +RSS здесь — накопительный максимум одного процесса, который последовательно +готовил и обходил все три набора. Поэтому значения памяти по размерам не являются +независимыми измерениями чистой памяти одного прохода. Максимум 71.859 MiB +не достиг лимита reconciler 256 MiB. На большие объёмы эти числа не экстраполирую. + +![Стоимость обхода двух хранилищ](evidence/reconcile_cost.png) + +Вывод: на этом стенде гипотеза подтверждена для воспроизведённых состояний. +Reconciler обнаружил оба вида рассогласования, безопасно выполнил компенсацию, +сохранил свежие/используемые вложения; повторная сверка не внесла новых изменений. +Стоимость проверенного обхода растёт вместе с числом объектов; выполнены +отдельные измерения RAM для 10 конкурентных upload. + +## Ограничения и источники + +Один host, PG, LB и S3 остаются SPOF. Две API не защищают от потери S3-диска. +Повреждённые/отсутствующие оригиналы требуют восстановления из внешней копии. +Полный проход хранит inventory/IDs в памяти, его стоимость растёт с числом объектов; +для больших объёмов нужны paging/checkpoints и отдельный бюджет обслуживания. +Обычный ready проверяется по HEAD/метаданным; полный checksum bytes выполняется +при восстановлении missing или несогласованности HEAD. Это не постоянный full scrub. +Истёкшие сессии/ключи и неиспользуемые ready-вложения автоматически не удаляются. +Grace может задержать очистку; lease не является универсальным distributed transaction. + +Измерения выполняются на одном компьютере; мониторинг/контрольные запросы также +потребляют его ресурсы. RAM-серия охватывает файлы 197173 и 3151221 bytes; +она не измеряет пик для каждого допустимого файла до 10 MiB/40 млн пикселей. +Защита предельного размера и пикселей отдельно проверена отрицательными тестами. Для длинной нагрузки результаты нужно проверять отдельно. +HTTP допускается только в локальном изолированном стенде. Прямой presigned upload +и pipeline thumbnail не реализованы, поскольку выбран трек 2 lab4. + +Источники: README/COURSE/OpenAPI lab4 и собственное решение lab3; +[MinIO release](https://github.com/minio/minio/releases/tag/RELEASE.2025-04-22T22-12-26Z), +[mc release](https://github.com/minio/mc/releases/tag/RELEASE.2025-04-16T18-13-26Z), +[S3 PutObject](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/put_object.html), +[Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), +[PostgreSQL locks](https://www.postgresql.org/docs/16/explicit-locking.html#ADVISORY-LOCKS). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d2a6cea --- /dev/null +++ b/compose.yaml @@ -0,0 +1,143 @@ +services: + db: + image: postgres:16.9-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [postgres_data:/var/lib/postgresql/data] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 3s + retries: 30 + cache: + image: redis:7.4.3-bookworm + command: [redis-server, --appendonly, 'yes', --appendfsync, everysec] + volumes: [redis_data:/data] + cpus: 0.5 + mem_limit: 128m + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 3s + retries: 30 + migrate: + build: . + command: [python, -m, src.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + cpus: 0.5 + mem_limit: 128m + depends_on: + db: {condition: service_healthy} + healthcheck: {disable: true} + s3: + image: mtusi2026lab4-minio:2025-04-22 + build: + context: . + dockerfile: infra/s3.Dockerfile + args: + COMPONENT: minio + RELEASE: RELEASE.2025-04-22T22-12-26Z + SHA256: 53e2a2cb16c5366ea6fbbc479c19ddb4c6a0948273e752f740fb1fbf27bb817c + command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + MINIO_BROWSER: 'off' + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://localhost:9000/minio/health/live",timeout=2)'] + interval: 2s + timeout: 3s + retries: 30 + s3-bootstrap: + image: mtusi2026lab4-mc:2025-04-16 + build: + context: . + dockerfile: infra/s3.Dockerfile + args: + COMPONENT: mc + RELEASE: RELEASE.2025-04-16T18-13-26Z + SHA256: ac90da87a35641be5a0ac75d49de5161ddb47d629b5ba01261b0ae9e00aea15f + entrypoint: [/bin/sh, /bootstrap.sh] + volumes: ['./scripts/bootstrap_s3.sh:/bootstrap.sh:ro', './infra:/policies:ro'] + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + environment: + S3_ENDPOINT: http://s3:9000 + depends_on: + s3: {condition: service_healthy} + cpus: 0.5 + mem_limit: 128m + api: + build: . + scale: 2 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://cache:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_REGION: us-east-1 + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: '10485760' + MAX_IMAGE_PIXELS: '40000000' + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + dns_opt: ['timeout:1', 'attempts:1'] + cpus: ${API_CPU:-1.0} + mem_limit: ${API_MEMORY:-512m} + stop_grace_period: 10s + depends_on: + migrate: {condition: service_completed_successfully} + cache: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + healthcheck: + test: [CMD, python, /app/scripts/healthcheck.py] + interval: 2s + timeout: 4s + retries: 30 + reconciler: + build: . + command: [python, -m, src.reconcile, --apply, --loop, --interval, '60', --grace-seconds, '600'] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + cpus: 0.5 + mem_limit: 256m + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + healthcheck: {disable: true} + lb: + image: haproxy:3.0.11-bookworm + ports: ['127.0.0.1:${APP_PORT:-8080}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + cpus: 0.5 + mem_limit: 128m + depends_on: + api: {condition: service_healthy} +volumes: + postgres_data: + redis_data: + s3_data: +secrets: + db_password: {file: .secrets/db_password} + database_url: {file: .secrets/database_url} + s3_root_user: {file: .secrets/s3_root_user} + s3_root_password: {file: .secrets/s3_root_password} + s3_access_key_id: {file: .secrets/s3_access_key_id} + s3_secret_access_key: {file: .secrets/s3_secret_access_key} diff --git a/evidence/attachment_failover.json b/evidence/attachment_failover.json new file mode 100644 index 0000000..7c63a83 --- /dev/null +++ b/evidence/attachment_failover.json @@ -0,0 +1,247 @@ +{ + "status": "PASS", + "uploader": "48fedaf342e1:632ba2ab", + "downloader": "9425d69912fc:665365c7", + "recovery_seconds": 1.3700443690031534, + "failed_attempts": 0, + "original_size": 111817, + "checksum_verified": true, + "stop": { + "target": "48fedaf342e1:632ba2ab", + "stopped_container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "before": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:25:41.660780116Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:25:39.453529659Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:25:39.454265647Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "after": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": false, + "started_at": "2026-10-16T19:25:41.660780116Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:25:39.453529659Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:25:39.454265647Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ] + } +} diff --git a/evidence/containers_after_restart.json b/evidence/containers_after_restart.json new file mode 100644 index 0000000..1055689 --- /dev/null +++ b/evidence/containers_after_restart.json @@ -0,0 +1,118 @@ +[ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:21.834698967Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:22:16.896719789Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:22:16.884675125Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } +] diff --git a/evidence/containers_before.json b/evidence/containers_before.json new file mode 100644 index 0000000..4ea55de --- /dev/null +++ b/evidence/containers_before.json @@ -0,0 +1,118 @@ +[ + { + "id": "3ce028ba0a279a365934fdddf6924260b13d8f3c3d15be9fb2b328c548fc03df", + "name": "mtusi2026lab4-api-1", + "hostname": "3ce028ba0a27", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:22:19.108920225Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "fd8770db9dfad55cb3f54b902061ea0d0e146399c2e02ccd74e1a7b87f7d0883", + "name": "mtusi2026lab4-api-2", + "hostname": "fd8770db9dfa", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:22:19.397716604Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:16:39.322959984Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:16:39.315215096Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a922458bb583dd8f163ddab3a44bd8d7a80784955bb1dd84d3971a02c0c933d9", + "name": "mtusi2026lab4-lb-1", + "hostname": "a922458bb583", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:22:22.158189871Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:22:16.896719789Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "7a63ecf93930384af9215ceca99f3634ecaf661258aac3410af9dd8ecfe49516", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "7a63ecf93930", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:22:19.108876624Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:16:39.315456952Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:22:16.884675125Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } +] diff --git a/evidence/containers_final.json b/evidence/containers_final.json new file mode 100644 index 0000000..7243978 --- /dev/null +++ b/evidence/containers_final.json @@ -0,0 +1,118 @@ +[ + { + "id": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975", + "name": "mtusi2026lab4-api-3", + "hostname": "d66f39ddc1de", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:30:35.417805574Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa", + "name": "mtusi2026lab4-api-4", + "hostname": "7d0a6457eb21", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:30:35.105182778Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:27:50.22191294Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:27:57.6646694Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:31:59.037730504Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:32:00.920195433Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:28:15.845174201Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:31:59.038314101Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } +] diff --git a/evidence/dependencies.json b/evidence/dependencies.json new file mode 100644 index 0000000..a420090 --- /dev/null +++ b/evidence/dependencies.json @@ -0,0 +1,21 @@ +{ + "redis": { + "status": "PASS", + "elapsed_seconds": 7.07943196000997 + }, + "postgres": { + "status": "PASS", + "readiness_and_request_status": [ + 503, + 503 + ] + }, + "s3": { + "status": "PASS", + "readiness_and_download_status": [ + 503, + 503 + ] + }, + "recovery": "old session and confirmed message preserved" +} diff --git a/evidence/experiment_environment.json b/evidence/experiment_environment.json new file mode 100644 index 0000000..22c47e8 --- /dev/null +++ b/evidence/experiment_environment.json @@ -0,0 +1,123 @@ +{ + "docker": "26.1.1", + "compose": "2.27.0-desktop.2", + "containers": [ + { + "id": "355cebdc2affd62651fc7da80af09cae310fe235f8bcf83165a5d559b9f752c6", + "name": "mtusi2026lab4-api-3", + "hostname": "355cebdc2aff", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:28:38.177524516Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "90f387d120f8481a19384b5b630eef38ae04062b1064cf57ec330e7f847d8e50", + "name": "mtusi2026lab4-api-4", + "hostname": "90f387d120f8", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:28:37.808617738Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:27:50.22191294Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:27:57.6646694Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:28:19.512821485Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:28:21.5678737Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:28:15.845174201Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:28:19.512884457Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "client_cpus": 16 +} diff --git a/evidence/experiment_summary.json b/evidence/experiment_summary.json new file mode 100644 index 0000000..d84b6d2 --- /dev/null +++ b/evidence/experiment_summary.json @@ -0,0 +1,37 @@ +{ + "reconciler_cost": [ + { + "objects": 100, + "repetitions": 3, + "median_seconds": 0.696161, + "heads_per_scan": [ + 100, + 100, + 100 + ] + }, + { + "objects": 500, + "repetitions": 3, + "median_seconds": 3.476147, + "heads_per_scan": [ + 500, + 500, + 500 + ] + }, + { + "objects": 1000, + "repetitions": 3, + "median_seconds": 6.510459, + "heads_per_scan": [ + 1000, + 1000, + 1000 + ] + } + ], + "concurrent_uploads": 10, + "verified_originals": 60, + "status": "PASS" +} diff --git a/evidence/failures.json b/evidence/failures.json new file mode 100644 index 0000000..6d8884c --- /dev/null +++ b/evidence/failures.json @@ -0,0 +1,5359 @@ +[ + { + "duration_requested": 16, + "elapsed_seconds": 16.058, + "concurrency": 4, + "profile": "closed-loop: 60% POST text128, 20% first50 history, 20% profile", + "retry_writes": true, + "successful_requests": 2034, + "failed_attempts": 38, + "uncertain_writes_resolved_after_window": 0, + "successful_rps": 126.663, + "error_ratio": 0.01833976833976834, + "successful_latency_ms": { + "p50": 25.308, + "p95": 86.066, + "p99": 99.909 + }, + "instances": { + "9425d69912fc:665365c7": 1748, + "48fedaf342e1:4f9b3f90": 286 + }, + "errors": { + "POST /api/v1/chats/fe2d51a6-1866-47a3-9bea-a9448f027c1e/messages: expected [201], received 503": 25, + "GET /api/v1/chats/fe2d51a6-1866-47a3-9bea-a9448f027c1e/messages: expected [200], received 503": 6, + "GET /api/v1/users/me: expected [200], received 503": 7 + }, + "operations": { + "write": 1252, + "history": 406, + "profile": 414 + }, + "successful_per_second": { + "0": 179, + "1": 271, + "2": 159, + "3": 55, + "4": 143, + "5": 92, + "6": 137, + "7": 108, + "8": 118, + "9": 127, + "10": 56, + "11": 134, + "12": 128, + "13": 83, + "14": 125, + "15": 115, + "16": 4 + }, + "failure_seconds": [ + 2.495, + 2.497, + 2.524, + 2.541, + 2.543, + 2.576, + 2.591, + 2.642, + 2.687, + 2.775, + 3.379, + 2.488, + 2.49, + 2.519, + 2.548, + 2.549, + 2.579, + 2.611, + 2.666, + 2.694, + 2.755, + 2.757, + 2.784, + 3.354, + 2.509, + 2.567, + 2.6, + 2.65, + 2.685, + 2.744, + 3.346, + 2.521, + 2.559, + 2.589, + 2.62, + 2.661, + 2.69, + 3.367 + ], + "latency_samples_ms": [ + 15.69, + 10.736, + 20.491, + 14.405, + 11.071, + 17.201, + 17.915, + 58.723, + 15.97, + 8.191, + 73.625, + 86.714, + 19.379, + 69.575, + 15.024, + 16.5, + 11.354, + 9.176, + 54.621, + 5.771, + 88.892, + 17.035, + 12.236, + 13.274, + 6.534, + 65.929, + 9.493, + 14.02, + 16.97, + 8.754, + 13.215, + 10.459, + 8.382, + 21.501, + 7.981, + 11.668, + 14.438, + 12.747, + 16.264, + 7.896, + 12.148, + 14.826, + 15.058, + 14.702, + 6.003, + 12.785, + 13.08, + 16.028, + 19.55, + 8.99, + 14.91, + 13.299, + 16.691, + 15.349, + 8.167, + 11.84, + 15.433, + 14.778, + 15.982, + 10.777, + 16.446, + 14.419, + 15.411, + 12.898, + 10.809, + 18.527, + 15.945, + 13.048, + 16.21, + 7.285, + 14.216, + 11.563, + 14.959, + 20.279, + 6.161, + 11.183, + 86.358, + 20.601, + 19.288, + 7.38, + 14.741, + 15.994, + 16.022, + 18.625, + 11.258, + 15.641, + 16.049, + 13.843, + 16.424, + 7.818, + 14.083, + 14.48, + 12.764, + 13.667, + 9.756, + 16.864, + 10.735, + 11.467, + 15.192, + 5.985, + 12.967, + 15.664, + 11.652, + 14.065, + 9.55, + 16.589, + 15.626, + 13.456, + 13.671, + 5.97, + 14.573, + 12.627, + 10.441, + 12.644, + 7.523, + 15.854, + 16.304, + 22.918, + 15.268, + 8.866, + 24.729, + 29.416, + 15.966, + 10.037, + 5.753, + 10.131, + 11.788, + 22.257, + 12.179, + 6.513, + 17.767, + 18.967, + 35.419, + 19.305, + 7.523, + 22.83, + 26.607, + 14.244, + 14.669, + 7.535, + 15.46, + 12.539, + 12.566, + 16.417, + 9.501, + 25.414, + 15.257, + 31.229, + 13.823, + 48.889, + 43.279, + 85.39, + 81.672, + 13.788, + 5.249, + 73.35, + 27.172, + 92.099, + 74.846, + 31.141, + 98.53, + 68.899, + 23.054, + 14.209, + 29.337, + 48.332, + 28.058, + 30.274, + 33.29, + 31.039, + 25.007, + 38.302, + 26.362, + 9.088, + 24.254, + 44.867, + 28.732, + 23.519, + 10.058, + 43.427, + 22.794, + 26.639, + 46.736, + 12.519, + 24.203, + 19.876, + 48.266, + 26.318, + 12.344, + 24.366, + 41.203, + 26.491, + 21.48, + 32.998, + 25.986, + 23.242, + 49.299, + 25.51, + 10.348, + 24.153, + 41.171, + 28.381, + 22.36, + 12.551, + 53.894, + 25.316, + 56.341, + 28.251, + 70.956, + 24.855, + 89.781, + 91.725, + 29.977, + 66.961, + 24.067, + 97.607, + 85.454, + 24.545, + 11.868, + 28.726, + 46.106, + 27.063, + 21.422, + 27.46, + 27.218, + 24.186, + 25.733, + 43.872, + 16.683, + 26.16, + 40.529, + 25.973, + 24.597, + 13.219, + 32.909, + 30.338, + 27.464, + 45.382, + 13.235, + 22.052, + 25.603, + 42.436, + 29.677, + 11.82, + 26.955, + 46.921, + 21.935, + 21.731, + 10.368, + 58.94, + 23.794, + 41.855, + 27.055, + 14.601, + 30.768, + 50.453, + 32.58, + 47.95, + 7.963, + 30.782, + 56.068, + 29.892, + 22.972, + 12.24, + 36.535, + 30.736, + 25.621, + 42.802, + 16.562, + 30.276, + 24.14, + 40.538, + 33.165, + 12.757, + 46.611, + 17.725, + 27.016, + 28.892, + 27.844, + 21.695, + 26.322, + 50.155, + 44.122, + 18.186, + 154.671, + 94.654, + 29.39, + 81.364, + 77.602, + 28.367, + 98.878, + 24.091, + 54.687, + 13.548, + 19.717, + 21.647, + 43.596, + 26.15, + 13.828, + 25.821, + 44.066, + 27.75, + 27.444, + 33.093, + 20.312, + 30.904, + 42.939, + 26.553, + 11.024, + 27.973, + 43.604, + 30.436, + 19.524, + 13.292, + 38.557, + 29.667, + 26.591, + 49.135, + 12.544, + 18.005, + 25.094, + 45.949, + 25.923, + 13.242, + 21.202, + 38.318, + 22.532, + 27.418, + 33.494, + 26.91, + 19.999, + 26.195, + 44.901, + 7.002, + 26.927, + 25.609, + 49.979, + 30.5, + 14.087, + 43.723, + 34.453, + 57.264, + 29.156, + 19.935, + 46.654, + 21.689, + 74.342, + 32.567, + 72.36, + 32.157, + 77.628, + 94.435, + 25.887, + 71.746, + 116.001, + 98.134, + 91.441, + 22.346, + 81.675, + 95.191, + 26.155, + 100.302, + 61.511, + 15.184, + 24.854, + 25.589, + 42.799, + 33.708, + 11.537, + 25.224, + 38.508, + 28.01, + 28.702, + 31.376, + 24.392, + 24.189, + 46.286, + 25.228, + 16.151, + 30.645, + 39.734, + 33.565, + 22.964, + 11.408, + 45.06, + 26.09, + 50.344, + 33.094, + 15.551, + 28.165, + 53.477, + 23.374, + 51.064, + 20.741, + 20.595, + 33.635, + 47.206, + 30.239, + 17.237, + 41.179, + 27.947, + 29.244, + 44.674, + 12.289, + 28.007, + 38.474, + 25.304, + 28.758, + 10.631, + 48.391, + 28.295, + 28.62, + 47.716, + 15.174, + 32.105, + 46.928, + 34.524, + 28.703, + 35.71, + 25.012, + 35.439, + 51.453, + 33.572, + 12.9, + 45.931, + 27.645, + 60.019, + 27.375, + 80.794, + 19.41, + 87.112, + 91.464, + 23.771, + 70.385, + 98.656, + 47.653, + 80.523, + 34.563, + 39.083, + 25.94, + 25.553, + 58.372, + 25.992, + 11.57, + 21.874, + 41.184, + 28.789, + 28.391, + 35.737, + 26.532, + 26.79, + 49.426, + 28.365, + 12.08, + 28.569, + 47.742, + 28.282, + 55.792, + 9.924, + 29.644, + 28.384, + 46.129, + 23.685, + 16.104, + 47.413, + 26.724, + 31.634, + 50.178, + 11.723, + 26.408, + 47.565, + 29.672, + 24.078, + 14.838, + 102.221, + 45.274, + 26.937, + 21.516, + 29.958, + 32.299, + 26.6, + 44.568, + 22.378, + 15.406, + 29.997, + 45.765, + 27.618, + 28.659, + 29.556, + 25.008, + 22.387, + 21.212, + 38.199, + 15.666, + 27.255, + 23.521, + 40.01, + 27.571, + 11.717, + 45.195, + 31.391, + 82.248, + 107.227, + 12.718, + 92.844, + 90.607, + 24.589, + 13.729, + 14.705, + 7.182, + 15.612, + 15.461, + 58.745, + 10.735, + 6.333, + 17.706, + 82.854, + 9.331, + 8.699, + 64.103, + 16.205, + 14.545, + 9.189, + 8.267, + 55.603, + 15.7, + 70.139, + 40.685, + 10.56, + 67.127, + 17.859, + 15.255, + 13.765, + 63.362, + 7.057, + 13.318, + 13.357, + 13.028, + 24.971, + 11.098, + 14.023, + 12.796, + 12.268, + 12.736, + 9.767, + 18.517, + 14.824, + 15.531, + 13.366, + 6.221, + 14.066, + 17.343, + 11.561, + 20.224, + 6.903, + 10.086, + 14.184, + 17.499, + 14.048, + 7.125, + 12.115, + 16.318, + 13.349, + 15.259, + 9.424, + 14.229, + 15.624, + 14.638, + 15.244, + 7.168, + 12.799, + 17.445, + 19.945, + 17.808, + 7.943, + 15.464, + 16.451, + 14.242, + 14.216, + 6.433, + 8.643, + 12.784, + 9.538, + 9.818, + 52.086, + 25.571, + 16.356, + 18.896, + 16.729, + 6.769, + 15.994, + 17.417, + 20.002, + 18.988, + 10.737, + 15.83, + 10.154, + 16.337, + 16.214, + 6.717, + 15.339, + 12.36, + 15.167, + 14.943, + 7.456, + 23.385, + 12.646, + 16.792, + 14.058, + 7.433, + 13.905, + 13.838, + 12.366, + 13.644, + 7.116, + 13.452, + 15.455, + 14.311, + 14.049, + 6.769, + 12.706, + 14.809, + 14.002, + 15.195, + 6.479, + 11.497, + 15.562, + 13.304, + 18.106, + 5.453, + 11.896, + 11.996, + 11.92, + 13.658, + 6.172, + 24.72, + 20.347, + 31.481, + 17.624, + 20.033, + 22.732, + 15.831, + 17.366, + 17.562, + 6.226, + 11.545, + 11.328, + 19.809, + 15.737, + 7.248, + 15.791, + 13.195, + 16.724, + 15.31, + 27.241, + 27.096, + 28.389, + 30.448, + 52.367, + 26.103, + 58.777, + 25.732, + 65.412, + 22.921, + 78.038, + 26.49, + 75.311, + 86.561, + 36.69, + 98.364, + 61.682, + 13.741, + 30.219, + 26.016, + 47.979, + 24.815, + 12.737, + 24.311, + 43.371, + 28.304, + 26.039, + 32.758, + 29.958, + 24.79, + 48.065, + 20.666, + 12.256, + 22.854, + 47.796, + 24.548, + 22.506, + 12.171, + 50.324, + 21.52, + 22.094, + 39.87, + 13.363, + 25.419, + 22.184, + 45.165, + 26.476, + 11.875, + 45.92, + 30.424, + 22.596, + 20.543, + 36.353, + 22.735, + 27.057, + 44.717, + 22.348, + 14.2, + 29.985, + 47.878, + 24.983, + 22.811, + 34.491, + 23.716, + 92.432, + 90.984, + 25.952, + 78.974, + 25.608, + 82.079, + 96.87, + 86.066, + 13.237, + 26.161, + 28.703, + 45.72, + 24.659, + 7.164, + 27.261, + 50.224, + 23.957, + 23.608, + 29.714, + 21.396, + 24.441, + 24.574, + 45.369, + 11.386, + 22.864, + 45.351, + 26.237, + 23.299, + 10.02, + 43.309, + 30.036, + 25.96, + 41.804, + 10.601, + 24.78, + 20.378, + 23.854, + 35.738, + 10.129, + 19.531, + 27.664, + 56.668, + 24.24, + 12.263, + 34.808, + 26.053, + 31.457, + 48.117, + 13.465, + 37.138, + 52.459, + 31.278, + 26.706, + 34.247, + 32.73, + 34.178, + 55.858, + 28.762, + 13.511, + 51.206, + 27.345, + 27.089, + 36.772, + 10.512, + 27.218, + 24.391, + 46.933, + 21.366, + 14.29, + 24.814, + 42.845, + 23.588, + 18.33, + 8.971, + 50.899, + 36.379, + 160.669, + 81.336, + 15.922, + 81.049, + 27.804, + 87.385, + 87.853, + 16.109, + 33.628, + 58.834, + 28.03, + 23.529, + 13.301, + 45.719, + 27.655, + 25.736, + 41.593, + 12.266, + 20.529, + 27.814, + 45.76, + 25.652, + 14.256, + 37.944, + 31.833, + 26.84, + 46.66, + 14.659, + 27.153, + 51.631, + 26.718, + 26.319, + 12.681, + 53.197, + 27.603, + 24.997, + 43.04, + 10.892, + 24.221, + 29.035, + 36.628, + 23.27, + 10.351, + 25.202, + 44.574, + 27.254, + 24.665, + 31.33, + 21.752, + 26.125, + 25.377, + 44.517, + 13.634, + 35.123, + 51.0, + 34.537, + 48.619, + 11.472, + 31.742, + 53.019, + 27.059, + 72.504, + 12.147, + 22.202, + 88.467, + 85.352, + 27.902, + 77.844, + 87.302, + 117.446, + 93.663, + 27.573, + 69.078, + 102.703, + 26.584, + 95.05, + 98.714, + 10.526, + 70.842, + 28.926, + 22.746, + 40.296, + 15.515, + 29.377, + 25.203, + 39.671, + 25.756, + 12.764, + 22.616, + 49.717, + 26.898, + 21.511, + 29.617, + 26.745, + 27.349, + 41.563, + 26.566, + 11.969, + 26.981, + 40.337, + 21.536, + 21.434, + 14.873, + 45.279, + 33.28, + 26.699, + 40.128, + 11.808, + 27.983, + 17.036, + 47.654, + 25.781, + 12.648, + 27.16, + 45.727, + 36.77, + 44.245, + 12.387, + 26.673, + 27.635, + 51.002, + 27.726, + 11.175, + 44.077, + 31.718, + 27.095, + 40.492, + 13.24, + 29.405, + 30.284, + 47.152, + 32.397, + 13.925, + 40.05, + 23.485, + 30.298, + 45.179, + 10.896, + 27.487, + 25.226, + 44.741, + 26.221, + 10.45, + 42.725, + 30.611, + 30.553, + 62.164, + 14.054, + 88.462, + 87.015, + 97.369, + 27.122, + 74.878, + 86.923, + 50.137, + 79.856, + 29.301, + 7.848, + 45.83, + 31.196, + 33.334, + 55.167, + 10.122, + 27.774, + 40.747, + 29.583, + 22.011, + 11.323, + 49.52, + 28.654, + 50.018, + 27.525, + 15.251, + 30.727, + 41.774, + 25.011, + 31.007, + 32.05, + 30.265, + 25.254, + 20.766, + 41.773, + 12.293, + 25.723, + 45.28, + 27.085, + 29.462, + 10.397, + 52.484, + 31.919, + 24.852, + 35.929, + 8.929, + 26.665, + 100.414, + 24.417, + 35.621, + 10.316, + 24.762, + 23.114, + 35.394, + 22.696, + 10.398, + 21.695, + 44.676, + 28.587, + 23.284, + 8.042, + 50.983, + 26.462, + 29.755, + 41.662, + 9.648, + 24.012, + 27.62, + 45.414, + 21.712, + 9.441, + 25.35, + 36.855, + 27.866, + 27.267, + 32.248, + 27.423, + 73.609, + 115.173, + 92.046, + 19.396, + 87.606, + 14.613, + 19.087, + 7.808, + 17.534, + 22.456, + 13.155, + 60.387, + 10.114, + 20.523, + 11.576, + 9.575, + 54.44, + 6.292, + 86.553, + 14.677, + 13.087, + 77.016, + 6.49, + 15.678, + 86.109, + 83.097, + 18.121, + 7.096, + 16.823, + 65.807, + 17.02, + 14.975, + 6.708, + 12.754, + 27.109, + 12.068, + 14.11, + 8.345, + 14.72, + 13.357, + 13.419, + 16.268, + 8.821, + 16.23, + 17.312, + 16.143, + 14.862, + 7.26, + 14.247, + 17.592, + 19.109, + 16.742, + 9.203, + 21.034, + 16.202, + 14.703, + 16.384, + 8.721, + 14.671, + 14.437, + 14.632, + 15.69, + 7.449, + 14.241, + 16.509, + 16.941, + 15.735, + 8.636, + 14.346, + 14.361, + 18.088, + 15.293, + 8.558, + 12.211, + 17.628, + 77.734, + 13.459, + 6.805, + 16.506, + 19.409, + 19.021, + 15.454, + 9.326, + 17.623, + 21.387, + 20.244, + 16.475, + 7.54, + 13.014, + 13.345, + 18.919, + 14.29, + 6.541, + 11.676, + 15.22, + 16.548, + 20.6, + 8.308, + 17.111, + 12.33, + 13.144, + 13.124, + 5.756, + 16.05, + 14.19, + 15.185, + 14.631, + 5.921, + 13.728, + 14.764, + 15.192, + 14.216, + 8.91, + 12.612, + 12.797, + 16.466, + 19.164, + 6.641, + 18.526, + 12.74, + 28.169, + 22.771, + 6.811, + 19.52, + 20.582, + 14.926, + 11.61, + 20.73, + 18.247, + 31.276, + 33.718, + 16.189, + 8.333, + 32.41, + 14.307, + 17.253, + 20.88, + 7.81, + 12.47, + 14.852, + 15.422, + 20.96, + 30.034, + 25.917, + 30.682, + 47.543, + 32.858, + 57.053, + 97.843, + 13.639, + 6.631, + 31.343, + 70.748, + 86.101, + 86.902, + 15.369, + 82.93, + 44.036, + 66.359, + 25.879, + 15.352, + 23.147, + 45.117, + 27.541, + 24.051, + 33.957, + 31.117, + 23.347, + 42.29, + 27.756, + 9.932, + 25.29, + 47.695, + 25.798, + 27.374, + 32.374, + 23.132, + 20.962, + 22.238, + 42.686, + 10.977, + 26.969, + 26.022, + 50.226, + 25.847, + 14.468, + 42.259, + 24.805, + 23.593, + 22.751, + 29.664, + 25.668, + 23.905, + 46.335, + 22.515, + 13.532, + 25.739, + 41.682, + 27.994, + 25.205, + 12.097, + 51.376, + 22.03, + 29.929, + 45.463, + 14.607, + 90.636, + 93.043, + 95.336, + 26.316, + 75.31, + 97.723, + 31.738, + 80.991, + 22.997, + 11.872, + 45.546, + 21.695, + 24.164, + 23.874, + 31.377, + 22.108, + 26.875, + 22.249, + 47.198, + 10.261, + 24.401, + 41.595, + 24.932, + 22.21, + 13.113, + 40.563, + 27.216, + 27.144, + 43.742, + 5.212, + 26.903, + 29.941, + 46.459, + 24.678, + 13.372, + 27.597, + 49.166, + 23.895, + 22.389, + 33.593, + 29.188, + 26.688, + 42.313, + 24.261, + 9.378, + 28.199, + 50.643, + 34.371, + 25.899, + 35.618, + 29.697, + 34.177, + 49.388, + 20.112, + 12.744, + 25.758, + 56.248, + 24.47, + 48.596, + 10.302, + 24.502, + 26.197, + 37.275, + 27.08, + 10.539, + 30.882, + 42.851, + 22.048, + 30.347, + 29.049, + 26.391, + 26.473, + 45.053, + 37.069, + 14.62, + 160.947, + 91.414, + 29.861, + 84.853, + 15.599, + 93.536, + 96.368, + 23.303, + 21.935, + 41.311, + 26.806, + 26.126, + 45.658, + 19.82, + 5.864, + 25.526, + 20.083, + 37.909, + 21.981, + 15.453, + 42.892, + 26.947, + 32.493, + 41.242, + 10.799, + 25.541, + 23.621, + 47.802, + 30.3, + 15.206, + 45.557, + 27.465, + 28.817, + 52.156, + 12.684, + 25.875, + 23.507, + 46.37, + 30.353, + 12.098, + 21.762, + 34.249, + 27.183, + 27.429, + 31.037, + 28.285, + 25.24, + 24.237, + 46.165, + 11.74, + 24.332, + 38.372, + 38.764, + 30.252, + 32.328, + 22.802, + 30.022, + 59.873, + 28.932, + 13.484, + 46.608, + 24.103, + 75.119, + 30.093, + 71.783, + 32.56, + 83.719, + 92.32, + 95.211, + 19.234, + 173.664, + 26.434, + 94.199, + 92.977, + 14.457, + 87.297, + 88.968, + 36.598, + 66.139, + 11.154, + 21.009, + 27.21, + 39.692, + 30.158, + 12.405, + 29.37, + 38.081, + 27.131, + 28.949, + 36.684, + 20.787, + 23.85, + 46.135, + 30.172, + 12.716, + 28.607, + 49.629, + 36.089, + 26.635, + 23.478, + 18.932, + 34.401, + 54.317, + 26.622, + 12.307, + 23.525, + 50.077, + 31.646, + 43.263, + 7.701, + 29.756, + 26.635, + 40.932, + 26.663, + 14.829, + 57.536, + 22.546, + 19.323, + 21.382, + 32.943, + 26.416, + 33.19, + 43.028, + 23.125, + 12.475, + 24.042, + 53.383, + 28.64, + 29.719, + 34.383, + 27.522, + 29.981, + 53.631, + 20.561, + 12.748, + 48.201, + 29.716, + 34.924, + 40.674, + 16.172, + 22.586, + 27.979, + 47.682, + 29.699, + 13.421, + 67.837, + 90.983, + 82.654, + 28.846, + 69.974, + 29.909, + 96.568, + 103.948, + 79.525, + 7.673, + 36.021, + 53.108, + 25.882, + 35.096, + 34.445, + 30.757, + 31.036, + 40.192, + 23.926, + 9.601, + 27.508, + 44.778, + 31.513, + 30.709, + 34.713, + 31.217, + 31.224, + 45.243, + 29.196, + 11.394, + 60.014, + 27.44, + 25.333, + 32.846, + 11.396, + 29.653, + 28.813, + 46.398, + 19.482, + 9.916, + 48.443, + 31.162, + 21.459, + 25.486, + 23.176, + 21.552, + 20.538, + 23.987, + 100.133, + 7.863, + 37.925, + 25.2, + 27.643, + 35.709, + 12.533, + 25.714, + 18.863, + 47.801, + 21.944, + 13.665, + 30.749, + 43.286, + 25.127, + 26.415, + 32.54, + 24.16, + 22.157, + 26.543, + 45.202, + 12.85, + 21.603, + 22.234, + 36.266, + 21.384, + 12.111, + 27.089, + 52.442, + 80.368, + 109.444, + 7.766, + 85.538, + 27.892, + 80.55, + 18.208, + 12.411, + 15.808, + 17.548, + 13.231, + 14.085, + 8.015, + 12.779, + 12.199, + 41.232, + 15.894, + 66.401, + 12.854, + 92.245, + 15.174, + 70.131, + 5.324, + 10.81, + 14.169, + 15.113, + 82.592, + 4.708, + 8.714, + 81.563, + 12.277, + 15.181, + 8.181, + 8.768, + 7.462, + 56.045, + 12.286, + 5.212, + 11.691, + 14.619, + 12.517, + 23.44, + 7.491, + 10.054, + 13.953, + 11.247, + 13.086, + 7.087, + 15.342, + 13.772, + 15.239, + 15.43, + 7.608, + 15.769, + 12.088, + 15.116, + 15.98, + 10.468, + 22.04, + 11.134, + 11.042, + 12.744, + 9.227, + 16.121, + 14.914, + 10.866, + 16.603, + 6.664, + 16.143, + 12.795, + 15.368, + 13.305, + 6.963, + 16.686, + 22.567, + 15.168, + 14.913, + 9.351, + 13.712, + 18.041, + 13.498, + 13.2, + 9.289, + 18.951, + 86.156, + 23.195, + 16.769, + 6.889, + 18.249, + 19.732, + 22.419, + 20.573, + 7.749, + 12.032, + 15.884, + 15.474, + 15.077, + 10.623, + 16.291, + 12.776, + 13.135, + 14.017, + 8.495, + 11.189, + 22.311, + 16.179, + 14.486, + 8.031, + 15.83, + 14.526, + 12.225, + 13.868, + 8.285, + 16.489, + 16.347, + 10.435, + 12.847, + 7.93, + 13.74, + 15.149, + 14.014, + 16.484, + 8.001, + 16.253, + 16.205, + 14.352, + 17.129, + 6.002, + 33.595, + 20.335, + 25.008, + 20.056, + 8.216, + 26.402, + 16.79, + 19.052, + 26.239, + 17.126, + 17.179, + 14.779, + 34.945, + 15.581, + 9.192, + 15.912, + 15.566, + 12.731, + 15.312, + 7.883, + 15.369, + 26.636, + 24.297, + 11.41, + 27.574, + 29.171, + 38.437, + 27.807, + 62.117, + 95.092, + 17.03, + 26.311, + 52.585, + 30.13, + 96.589, + 95.499, + 89.868, + 13.85, + 79.008, + 26.015, + 25.346, + 37.982, + 9.446, + 19.227, + 19.788, + 24.605, + 42.195, + 12.034, + 25.108, + 52.214, + 34.417, + 23.38, + 7.761, + 41.208, + 24.658, + 29.292, + 39.049, + 9.031, + 26.367, + 24.69, + 44.325, + 26.972, + 9.975, + 24.919, + 49.485, + 26.733, + 27.146, + 26.885, + 25.576, + 27.273, + 45.312, + 23.867, + 9.976, + 19.167, + 45.899, + 31.16, + 19.177, + 8.818, + 47.288, + 23.799, + 29.083, + 44.051, + 9.631, + 24.661, + 25.942, + 56.415, + 88.876, + 14.077, + 89.023, + 90.944, + 37.032, + 76.12, + 16.56, + 93.443, + 84.046, + 32.189, + 26.78, + 12.149, + 40.417, + 27.591, + 27.337, + 45.298, + 10.113, + 19.731, + 25.058, + 48.967, + 23.979, + 11.694, + 41.268, + 25.241, + 29.351, + 42.309, + 13.292, + 22.532, + 22.376, + 43.811, + 28.223, + 17.627, + 24.103, + 45.869, + 27.119, + 22.307, + 10.807, + 40.528, + 27.132, + 27.753, + 51.084, + 9.528, + 23.187, + 44.362, + 27.657, + 26.84, + 13.575, + 48.23, + 37.087, + 55.735, + 26.986, + 18.322, + 49.723, + 33.341, + 32.686, + 39.381, + 13.647, + 24.682, + 52.894, + 22.843, + 20.61, + 12.267, + 20.464, + 33.841, + 25.727, + 23.11, + 10.149, + 48.972, + 25.887, + 32.631, + 42.733, + 10.24, + 24.009, + 49.718, + 49.498, + 156.111, + 11.996, + 88.864, + 26.137, + 83.015, + 88.019, + 13.216, + 86.478, + 24.199, + 24.44, + 45.844, + 12.86, + 23.523, + 26.173, + 40.31, + 26.826, + 12.944, + 26.777, + 42.025, + 29.365, + 25.655, + 33.672, + 22.562, + 30.413, + 45.912, + 22.225, + 10.977, + 22.954, + 50.006, + 24.824, + 24.291, + 10.784, + 39.216, + 25.791, + 27.204, + 49.401, + 10.546, + 23.818, + 26.674, + 46.21, + 23.654, + 14.773, + 21.718, + 37.754, + 25.308, + 24.85, + 7.802, + 49.003, + 22.202, + 28.008, + 49.905, + 13.798, + 20.827, + 43.422, + 29.396, + 26.273, + 18.216, + 45.605, + 28.789, + 62.264, + 33.713, + 14.784, + 42.975, + 21.586, + 77.261, + 27.796, + 74.676, + 24.171, + 90.024, + 92.019, + 29.698, + 74.7, + 108.096, + 94.741, + 93.161, + 28.924, + 76.64, + 92.587, + 27.028, + 102.925, + 63.547, + 11.846, + 27.434, + 21.534, + 44.161, + 33.226, + 12.705, + 24.658, + 40.967, + 25.138, + 24.584, + 32.442, + 26.338, + 23.987, + 28.133, + 39.843, + 13.73, + 26.59, + 45.805, + 26.151, + 30.356, + 18.429, + 43.031, + 25.943, + 49.121, + 40.255, + 14.715, + 44.992, + 31.699, + 27.722, + 50.453, + 15.394, + 24.959, + 31.804, + 39.467, + 26.606, + 22.299, + 45.965, + 29.048, + 30.15, + 48.35, + 13.867, + 24.017, + 45.328, + 26.955, + 28.032, + 14.146, + 45.992, + 23.959, + 32.418, + 52.144, + 16.197, + 23.842, + 48.445, + 31.533, + 26.101, + 29.9, + 30.743, + 28.518, + 45.363, + 29.192, + 15.506, + 46.19, + 26.612, + 33.854, + 57.518, + 12.596, + 90.5, + 87.069, + 99.909, + 24.639, + 70.888, + 94.079, + 46.584, + 80.626, + 25.505, + 14.059, + 55.265, + 24.279, + 55.849, + 25.184, + 16.567, + 27.247, + 43.589, + 27.684, + 30.034, + 32.305, + 29.996, + 22.544, + 53.286, + 31.319, + 14.497, + 25.87, + 42.132, + 27.531, + 51.528, + 15.548, + 28.835, + 30.289, + 45.371, + 30.247, + 10.345, + 39.996, + 32.113, + 35.06, + 45.438, + 13.233, + 28.767, + 47.154, + 30.881, + 25.153, + 11.782, + 103.701, + 44.574, + 23.126, + 27.163, + 26.458, + 32.408, + 29.076, + 44.381, + 29.171, + 13.72, + 21.4, + 51.433, + 21.321, + 25.407, + 13.041, + 43.241, + 23.741, + 22.598, + 42.519, + 11.16, + 27.791, + 27.282, + 38.822, + 24.258, + 12.067, + 24.079, + 52.192, + 86.002, + 102.971, + 14.107, + 90.738, + 92.422 + ], + "action_events": [ + { + "start_seconds": 2.0022432440018747, + "end_seconds": 3.3975913769972976, + "result": { + "target": "48fedaf342e1:4f9b3f90", + "stopped_container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "before": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:07.641189576Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:05.538455296Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:05.540733435Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "after": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": false, + "started_at": "2026-10-16T19:26:07.641189576Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:05.538455296Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:05.540733435Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ] + } + } + ], + "first_successful_bucket_after_stop_seconds": 4, + "recovery_upper_bound_seconds": 2.998, + "verification": { + "confirmed_messages": 1228, + "stored_messages": 1228, + "status": "PASS" + }, + "attachment_original_verified": true, + "monitor": [ + { + "seconds": 3.774, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "99.09%", + "Container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "ID": "9425d69912fc", + "MemPerc": "22.09%", + "MemUsage": "113.1MiB / 512MiB", + "Name": "mtusi2026lab4-api-1", + "NetIO": "2.99MB / 3MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "84.74%", + "Container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "ID": "48fedaf342e1", + "MemPerc": "18.21%", + "MemUsage": "93.22MiB / 512MiB", + "Name": "mtusi2026lab4-api-2", + "NetIO": "1.55MB / 1.63MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.26%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.04%", + "MemUsage": "10.29MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "111kB / 166kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "34.18%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "8.32%", + "MemUsage": "42.59MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "1.08MB / 1.69MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.18%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "15.21%", + "MemUsage": "19.47MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "3.74MB / 3.76MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.13%", + "MemUsage": "48.98MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "45.7kB / 34.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.43%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "88.44%", + "MemUsage": "452.8MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "781kB / 951kB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 5 + } + ], + "pools": [ + { + "instance": "9425d69912fc", + "stats": { + "connections_num": 5, + "connections_ms": 58, + "requests_num": 1613, + "usage_ms": 4743, + "requests_queued": 9, + "requests_wait_ms": 23, + "pool_min": 1, + "pool_max": 8, + "pool_size": 5, + "pool_available": 1, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 10.463, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "97.75%", + "Container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "ID": "9425d69912fc", + "MemPerc": "22.98%", + "MemUsage": "117.6MiB / 512MiB", + "Name": "mtusi2026lab4-api-1", + "NetIO": "7.1MB / 7.32MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.04%", + "MemUsage": "10.29MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "218kB / 340kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.79%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "7.01%", + "MemUsage": "35.88MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "2.9MB / 6.44MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.18%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "15.79%", + "MemUsage": "20.21MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "7.41MB / 7.47MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.13%", + "MemUsage": "48.98MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "45.9kB / 34.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.40%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "88.44%", + "MemUsage": "452.8MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "790kB / 963kB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "active", + "wait_event_type": "IO", + "wait_event": "WALSync", + "count": 1 + }, + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 4 + } + ], + "pools": [ + { + "instance": "9425d69912fc", + "stats": { + "connections_num": 5, + "connections_ms": 58, + "requests_num": 3641, + "usage_ms": 21158, + "requests_queued": 9, + "requests_wait_ms": 23, + "pool_min": 1, + "pool_max": 8, + "pool_size": 5, + "pool_available": 2, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 16.793, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "99.91%", + "Container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "ID": "9425d69912fc", + "MemPerc": "23.57%", + "MemUsage": "120.7MiB / 512MiB", + "Name": "mtusi2026lab4-api-1", + "NetIO": "10.8MB / 11.2MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.04%", + "MemUsage": "10.29MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "325kB / 515kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.76%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "7.10%", + "MemUsage": "36.36MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "4.26MB / 10.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.26%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "15.53%", + "MemUsage": "19.88MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "11.2MB / 11.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "75.2kB / 57.1kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "88.45%", + "MemUsage": "452.8MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "800kB / 980kB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 5 + } + ], + "pools": [ + { + "instance": "9425d69912fc", + "stats": { + "connections_num": 5, + "connections_ms": 58, + "requests_num": 5267, + "usage_ms": 34892, + "requests_queued": 9, + "requests_wait_ms": 23, + "pool_min": 1, + "pool_max": 8, + "pool_size": 5, + "pool_available": 5, + "requests_waiting": 0 + } + } + ] + } + ], + "target": "48fedaf342e1:4f9b3f90" + }, + { + "duration_requested": 16, + "elapsed_seconds": 16.016, + "concurrency": 4, + "profile": "closed-loop: 60% POST text128, 20% first50 history, 20% profile", + "retry_writes": true, + "successful_requests": 1995, + "failed_attempts": 56, + "uncertain_writes_resolved_after_window": 0, + "successful_rps": 124.562, + "error_ratio": 0.027303754266211604, + "successful_latency_ms": { + "p50": 25.392, + "p95": 82.536, + "p99": 95.979 + }, + "instances": { + "48fedaf342e1:eba13b85": 1725, + "9425d69912fc:665365c7": 270 + }, + "errors": { + "POST /api/v1/chats/e46f3bfe-6838-4437-8d51-58ad5709eb68/messages: expected [201], received 503": 32, + "GET /api/v1/users/me: expected [200], received 503": 13, + "GET /api/v1/chats/e46f3bfe-6838-4437-8d51-58ad5709eb68/messages: expected [200], received 503": 11 + }, + "operations": { + "write": 1243, + "history": 401, + "profile": 407 + }, + "successful_per_second": { + "0": 203, + "1": 219, + "2": 170, + "3": 33, + "4": 134, + "5": 91, + "6": 131, + "7": 140, + "8": 95, + "9": 98, + "10": 77, + "11": 109, + "12": 139, + "13": 93, + "14": 138, + "15": 121, + "16": 4 + }, + "failure_seconds": [ + 2.417, + 2.419, + 2.441, + 2.471, + 2.51, + 2.522, + 2.555, + 2.599, + 2.632, + 2.661, + 2.663, + 2.705, + 2.735, + 2.748, + 3.302, + 3.827, + 2.414, + 2.439, + 2.469, + 2.481, + 2.552, + 2.608, + 2.61, + 2.635, + 2.636, + 2.728, + 3.278, + 3.811, + 2.448, + 2.459, + 2.498, + 2.54, + 2.566, + 2.567, + 2.634, + 2.657, + 2.659, + 2.702, + 2.732, + 2.743, + 3.269, + 3.773, + 2.409, + 2.43, + 2.456, + 2.495, + 2.525, + 2.526, + 2.528, + 2.561, + 2.604, + 2.706, + 2.708, + 2.738, + 3.263, + 3.801 + ], + "latency_samples_ms": [ + 25.197, + 13.526, + 20.529, + 13.809, + 6.127, + 17.902, + 26.467, + 17.38, + 11.962, + 7.693, + 17.954, + 25.73, + 24.179, + 17.994, + 8.109, + 13.971, + 11.809, + 7.824, + 8.433, + 4.413, + 22.996, + 13.783, + 11.751, + 14.39, + 5.146, + 8.355, + 13.983, + 13.51, + 13.641, + 5.136, + 22.783, + 15.137, + 11.91, + 8.724, + 4.093, + 32.414, + 12.052, + 78.73, + 10.958, + 9.62, + 73.589, + 10.004, + 10.614, + 74.757, + 8.057, + 11.549, + 76.653, + 9.853, + 29.275, + 13.004, + 16.458, + 8.332, + 7.605, + 46.322, + 9.343, + 14.653, + 13.364, + 11.786, + 14.079, + 6.939, + 14.2, + 11.198, + 11.973, + 16.932, + 7.332, + 13.217, + 9.62, + 7.657, + 23.403, + 10.281, + 14.095, + 11.341, + 68.446, + 19.06, + 6.242, + 78.015, + 18.029, + 9.875, + 10.39, + 49.652, + 15.823, + 13.594, + 74.833, + 14.302, + 10.794, + 70.585, + 8.807, + 11.902, + 12.046, + 7.31, + 15.404, + 13.843, + 12.572, + 10.618, + 8.303, + 14.121, + 12.398, + 11.024, + 12.893, + 8.055, + 13.757, + 12.193, + 10.557, + 13.707, + 8.15, + 13.774, + 10.173, + 12.817, + 11.869, + 5.829, + 12.241, + 19.494, + 14.166, + 12.723, + 7.649, + 10.349, + 18.687, + 12.206, + 15.481, + 7.172, + 14.142, + 16.916, + 13.753, + 15.916, + 7.581, + 12.559, + 16.071, + 21.063, + 18.213, + 7.66, + 23.765, + 16.357, + 15.397, + 15.19, + 7.574, + 13.952, + 18.333, + 16.306, + 14.329, + 5.385, + 13.185, + 20.567, + 28.65, + 37.319, + 10.073, + 31.991, + 42.643, + 29.561, + 27.272, + 39.927, + 28.842, + 11.206, + 20.743, + 7.745, + 22.611, + 12.406, + 9.141, + 45.173, + 22.211, + 25.234, + 23.927, + 31.231, + 37.415, + 11.367, + 26.837, + 46.163, + 30.34, + 25.104, + 15.139, + 42.01, + 25.009, + 28.389, + 40.417, + 11.595, + 27.78, + 78.394, + 27.427, + 48.866, + 16.294, + 26.08, + 22.599, + 45.495, + 25.402, + 8.957, + 23.047, + 37.368, + 25.206, + 26.957, + 14.433, + 45.608, + 20.491, + 19.654, + 47.083, + 12.396, + 21.954, + 25.248, + 37.851, + 28.908, + 14.793, + 28.299, + 39.468, + 17.802, + 23.396, + 13.402, + 47.83, + 22.109, + 24.866, + 38.511, + 16.146, + 33.522, + 68.549, + 91.686, + 25.958, + 78.407, + 86.925, + 30.157, + 86.077, + 90.795, + 23.3, + 31.537, + 57.271, + 28.144, + 21.216, + 28.608, + 29.421, + 24.681, + 44.5, + 23.573, + 14.59, + 36.25, + 45.949, + 29.574, + 30.098, + 27.749, + 23.108, + 24.238, + 24.611, + 47.572, + 11.665, + 30.505, + 51.059, + 36.133, + 45.44, + 16.244, + 29.046, + 23.075, + 45.86, + 25.335, + 12.566, + 45.532, + 28.166, + 34.11, + 46.683, + 16.077, + 27.728, + 21.634, + 44.97, + 22.88, + 16.455, + 48.175, + 26.424, + 26.837, + 44.248, + 11.982, + 30.119, + 25.392, + 40.439, + 22.012, + 13.97, + 28.404, + 37.496, + 27.815, + 20.297, + 14.951, + 42.973, + 26.377, + 23.585, + 45.512, + 12.996, + 35.01, + 47.368, + 25.202, + 20.571, + 9.47, + 43.682, + 27.918, + 71.191, + 25.538, + 79.312, + 24.874, + 91.746, + 88.562, + 89.104, + 12.716, + 84.53, + 33.213, + 30.093, + 48.35, + 12.148, + 21.357, + 28.83, + 38.62, + 23.303, + 13.339, + 47.995, + 28.72, + 25.987, + 23.939, + 26.027, + 19.185, + 30.021, + 48.441, + 21.505, + 9.876, + 26.786, + 51.299, + 26.722, + 30.205, + 32.36, + 29.106, + 39.128, + 48.178, + 29.743, + 18.875, + 44.558, + 35.79, + 50.778, + 25.106, + 9.98, + 29.806, + 45.499, + 34.464, + 82.538, + 77.234, + 27.752, + 103.195, + 100.013, + 83.847, + 15.175, + 95.005, + 25.53, + 64.413, + 28.17, + 9.827, + 48.836, + 31.149, + 28.688, + 43.737, + 14.223, + 32.359, + 61.936, + 92.087, + 91.728, + 15.998, + 90.883, + 99.69, + 104.469, + 88.566, + 7.794, + 25.128, + 26.197, + 38.7, + 30.029, + 74.229, + 26.841, + 25.61, + 51.183, + 28.526, + 15.106, + 29.268, + 42.986, + 29.859, + 23.746, + 40.395, + 26.548, + 24.689, + 38.113, + 27.453, + 11.532, + 31.526, + 34.679, + 28.646, + 28.069, + 8.158, + 49.323, + 28.915, + 49.923, + 26.496, + 13.857, + 51.719, + 30.777, + 27.596, + 43.137, + 13.993, + 33.547, + 23.478, + 35.964, + 24.09, + 11.161, + 24.218, + 43.213, + 23.004, + 27.167, + 10.657, + 43.336, + 19.995, + 21.782, + 45.734, + 11.331, + 28.115, + 22.923, + 40.039, + 24.929, + 11.748, + 27.186, + 44.354, + 21.671, + 18.275, + 9.524, + 45.437, + 27.091, + 26.152, + 15.607, + 27.254, + 31.893, + 38.009, + 56.791, + 87.036, + 16.439, + 88.45, + 91.512, + 92.848, + 27.551, + 76.231, + 32.761, + 73.871, + 29.718, + 21.521, + 8.703, + 50.708, + 26.798, + 22.108, + 38.94, + 11.343, + 22.685, + 56.089, + 29.409, + 21.703, + 10.801, + 41.22, + 32.981, + 20.377, + 40.212, + 11.094, + 23.53, + 39.559, + 37.279, + 20.39, + 10.646, + 29.38, + 37.478, + 27.288, + 25.594, + 36.583, + 25.712, + 27.914, + 46.071, + 26.776, + 12.409, + 23.6, + 36.764, + 26.817, + 30.551, + 9.343, + 48.119, + 27.381, + 26.128, + 42.786, + 11.213, + 25.497, + 24.252, + 47.639, + 22.158, + 7.846, + 27.113, + 44.917, + 31.518, + 18.304, + 26.52, + 31.034, + 31.297, + 53.751, + 23.21, + 10.339, + 25.691, + 46.939, + 28.144, + 25.149, + 28.922, + 36.16, + 27.265, + 46.225, + 21.679, + 13.54, + 62.924, + 25.578, + 89.991, + 89.444, + 15.276, + 10.096, + 16.111, + 5.256, + 20.291, + 16.474, + 24.228, + 14.391, + 8.528, + 15.428, + 18.483, + 16.07, + 11.955, + 5.616, + 27.96, + 15.333, + 19.94, + 10.979, + 6.001, + 30.55, + 15.852, + 13.3, + 13.198, + 5.974, + 25.694, + 8.591, + 14.906, + 15.229, + 5.107, + 8.749, + 18.972, + 13.317, + 50.213, + 7.287, + 13.716, + 9.704, + 7.108, + 58.321, + 6.701, + 13.491, + 13.524, + 8.558, + 60.708, + 5.811, + 9.361, + 8.64, + 8.114, + 77.297, + 5.259, + 8.086, + 8.03, + 66.262, + 28.957, + 7.889, + 81.845, + 14.643, + 12.005, + 13.648, + 9.013, + 11.949, + 13.558, + 15.129, + 14.489, + 6.542, + 14.874, + 36.159, + 9.067, + 23.397, + 7.705, + 73.836, + 23.948, + 11.282, + 8.355, + 59.096, + 19.849, + 13.342, + 71.556, + 17.488, + 7.308, + 8.412, + 7.228, + 56.091, + 15.024, + 11.013, + 10.996, + 8.136, + 56.065, + 13.041, + 5.624, + 12.681, + 10.622, + 11.73, + 12.315, + 17.509, + 18.329, + 20.098, + 12.592, + 15.048, + 6.868, + 16.182, + 13.567, + 12.364, + 13.356, + 6.748, + 14.452, + 11.152, + 10.865, + 12.748, + 5.974, + 19.649, + 14.157, + 16.259, + 16.332, + 6.293, + 18.852, + 13.443, + 14.563, + 14.214, + 7.705, + 13.362, + 13.778, + 15.15, + 16.703, + 8.943, + 13.074, + 25.082, + 21.12, + 19.506, + 9.5, + 14.025, + 18.187, + 12.545, + 16.324, + 5.897, + 9.913, + 15.885, + 18.575, + 12.949, + 23.196, + 28.861, + 11.146, + 44.763, + 24.698, + 12.491, + 41.622, + 22.759, + 20.466, + 46.598, + 22.925, + 35.357, + 12.027, + 29.977, + 17.69, + 36.575, + 40.932, + 27.144, + 12.247, + 32.075, + 44.911, + 27.36, + 22.203, + 29.298, + 28.541, + 31.669, + 42.616, + 28.824, + 13.661, + 52.308, + 33.598, + 74.177, + 28.757, + 14.246, + 52.203, + 28.132, + 28.245, + 46.846, + 14.354, + 32.665, + 22.29, + 39.005, + 24.688, + 13.111, + 50.912, + 24.196, + 27.682, + 43.573, + 8.943, + 27.837, + 25.868, + 37.317, + 22.926, + 12.124, + 23.148, + 39.051, + 26.988, + 23.623, + 13.161, + 46.408, + 24.761, + 23.283, + 41.952, + 12.429, + 31.549, + 68.738, + 26.646, + 90.924, + 75.474, + 31.43, + 93.451, + 84.292, + 26.542, + 87.495, + 36.212, + 55.704, + 27.749, + 28.133, + 31.253, + 21.215, + 26.361, + 25.414, + 40.554, + 8.645, + 25.031, + 33.097, + 36.752, + 22.122, + 13.163, + 39.806, + 28.513, + 24.962, + 21.154, + 32.694, + 29.518, + 30.819, + 55.913, + 29.355, + 15.255, + 42.354, + 35.652, + 23.3, + 42.154, + 10.461, + 22.891, + 25.636, + 47.651, + 35.836, + 13.489, + 45.583, + 31.588, + 32.813, + 37.397, + 9.232, + 26.792, + 52.83, + 27.805, + 26.569, + 12.034, + 44.896, + 27.485, + 25.64, + 37.524, + 7.643, + 27.629, + 21.428, + 16.823, + 31.679, + 12.537, + 18.157, + 30.255, + 46.754, + 21.277, + 10.41, + 21.083, + 47.014, + 31.034, + 25.956, + 29.182, + 23.401, + 29.322, + 44.849, + 25.119, + 9.025, + 65.443, + 28.318, + 93.242, + 82.376, + 12.534, + 20.098, + 82.471, + 94.365, + 26.02, + 82.536, + 29.689, + 53.983, + 29.635, + 21.919, + 9.371, + 37.389, + 23.663, + 32.446, + 45.791, + 12.801, + 25.832, + 23.809, + 50.04, + 30.576, + 14.085, + 47.441, + 27.211, + 24.716, + 40.395, + 11.338, + 31.018, + 25.775, + 45.249, + 25.871, + 10.443, + 57.306, + 38.928, + 59.766, + 32.26, + 16.002, + 48.679, + 42.277, + 53.28, + 33.541, + 63.806, + 30.111, + 87.123, + 88.808, + 35.51, + 87.161, + 91.671, + 95.979, + 31.089, + 19.226, + 41.333, + 28.913, + 26.262, + 41.619, + 26.091, + 9.416, + 28.568, + 46.664, + 35.35, + 65.155, + 14.1, + 95.392, + 93.531, + 106.674, + 80.82, + 16.556, + 109.908, + 73.88, + 31.121, + 21.893, + 12.019, + 45.497, + 99.557, + 31.365, + 54.01, + 15.301, + 25.652, + 26.021, + 43.307, + 25.26, + 12.234, + 56.115, + 39.768, + 25.343, + 45.333, + 12.73, + 34.113, + 39.533, + 31.05, + 24.509, + 14.343, + 48.366, + 36.106, + 49.253, + 31.532, + 15.635, + 48.698, + 29.427, + 30.837, + 42.189, + 6.298, + 33.083, + 25.342, + 40.622, + 28.01, + 12.878, + 26.825, + 44.597, + 24.941, + 25.214, + 32.912, + 25.884, + 24.064, + 46.431, + 22.355, + 12.615, + 25.257, + 43.273, + 22.674, + 25.582, + 10.578, + 43.69, + 23.394, + 26.007, + 47.051, + 12.878, + 26.328, + 23.725, + 47.825, + 25.289, + 9.46, + 51.026, + 30.022, + 86.451, + 90.026, + 15.98, + 89.496, + 85.948, + 34.875, + 94.792, + 14.832, + 73.02, + 24.824, + 31.743, + 40.8, + 9.841, + 26.104, + 23.539, + 41.088, + 25.591, + 9.275, + 48.007, + 31.491, + 28.77, + 47.957, + 16.94, + 29.887, + 24.987, + 40.587, + 25.494, + 16.548, + 50.334, + 26.024, + 32.957, + 43.834, + 13.526, + 33.396, + 50.885, + 31.232, + 22.801, + 11.775, + 43.738, + 38.293, + 44.55, + 27.139, + 16.906, + 26.051, + 47.661, + 22.714, + 21.386, + 8.088, + 45.279, + 26.173, + 28.546, + 38.62, + 9.687, + 24.299, + 26.832, + 49.175, + 28.048, + 11.064, + 44.561, + 28.93, + 27.518, + 42.094, + 14.094, + 24.89, + 33.026, + 40.69, + 23.516, + 7.67, + 24.687, + 46.297, + 31.971, + 23.93, + 34.954, + 31.45, + 59.725, + 30.563, + 88.951, + 14.54, + 82.379, + 14.73, + 21.54, + 7.188, + 26.618, + 12.986, + 12.611, + 11.244, + 6.279, + 11.648, + 12.687, + 17.388, + 12.788, + 6.64, + 11.446, + 30.053, + 17.876, + 16.884, + 6.493, + 10.512, + 14.384, + 31.581, + 16.727, + 6.039, + 11.519, + 12.979, + 27.799, + 14.41, + 9.061, + 17.869, + 20.704, + 12.097, + 45.473, + 5.149, + 16.917, + 14.134, + 71.417, + 13.735, + 5.855, + 79.667, + 21.693, + 92.18, + 78.458, + 5.671, + 9.265, + 10.099, + 11.299, + 12.708, + 62.415, + 8.88, + 17.007, + 12.801, + 16.957, + 7.113, + 20.227, + 13.027, + 16.825, + 17.365, + 8.047, + 13.08, + 34.533, + 10.377, + 81.822, + 6.6, + 10.031, + 9.105, + 9.936, + 76.896, + 5.467, + 10.037, + 11.744, + 75.97, + 13.886, + 5.698, + 74.765, + 12.491, + 12.171, + 11.385, + 5.3, + 65.584, + 10.739, + 11.915, + 13.869, + 6.266, + 12.175, + 12.206, + 27.467, + 17.346, + 7.171, + 13.639, + 9.818, + 15.355, + 9.666, + 13.517, + 12.025, + 14.139, + 16.112, + 13.03, + 6.094, + 13.849, + 12.659, + 16.21, + 16.855, + 6.601, + 16.198, + 18.718, + 9.931, + 10.712, + 15.775, + 16.979, + 15.793, + 14.137, + 14.975, + 8.865, + 17.457, + 13.427, + 21.733, + 12.163, + 12.08, + 15.457, + 20.518, + 15.118, + 13.346, + 6.936, + 17.045, + 14.256, + 18.126, + 15.602, + 7.976, + 12.591, + 15.873, + 29.59, + 10.695, + 36.626, + 26.308, + 14.857, + 23.725, + 40.677, + 22.505, + 21.532, + 40.761, + 28.239, + 9.734, + 22.783, + 10.021, + 7.747, + 7.349, + 9.722, + 13.439, + 5.192, + 36.526, + 47.915, + 28.08, + 31.062, + 29.735, + 24.184, + 24.148, + 46.898, + 22.927, + 15.178, + 27.056, + 53.858, + 33.594, + 48.418, + 13.254, + 21.861, + 69.999, + 36.178, + 17.535, + 9.888, + 45.294, + 23.307, + 25.251, + 48.395, + 15.507, + 27.71, + 25.668, + 40.413, + 24.503, + 14.429, + 48.651, + 23.826, + 24.72, + 51.227, + 6.235, + 28.197, + 24.631, + 37.235, + 29.414, + 15.957, + 47.891, + 24.118, + 32.83, + 46.331, + 12.629, + 26.124, + 24.563, + 38.648, + 32.941, + 13.224, + 73.929, + 82.759, + 94.192, + 34.142, + 78.534, + 22.151, + 86.304, + 111.447, + 27.623, + 37.878, + 28.728, + 20.952, + 25.064, + 47.29, + 12.744, + 23.655, + 20.081, + 45.803, + 27.88, + 17.58, + 46.453, + 31.803, + 30.496, + 44.59, + 9.394, + 22.924, + 47.548, + 36.106, + 26.786, + 32.317, + 36.847, + 35.539, + 41.453, + 27.795, + 8.696, + 22.335, + 46.632, + 26.075, + 25.718, + 29.64, + 31.136, + 29.32, + 51.526, + 30.064, + 15.098, + 20.149, + 39.389, + 27.883, + 29.6, + 35.627, + 25.312, + 25.331, + 48.125, + 20.656, + 7.47, + 21.815, + 21.981, + 34.94, + 23.151, + 10.648, + 31.313, + 36.334, + 26.361, + 22.79, + 10.402, + 41.546, + 24.972, + 26.882, + 48.795, + 14.18, + 32.949, + 46.422, + 19.608, + 27.936, + 12.118, + 43.135, + 23.262, + 66.035, + 23.697, + 82.166, + 90.024, + 33.221, + 88.452, + 90.959, + 15.592, + 101.099, + 28.797, + 57.975, + 21.049, + 10.1, + 27.621, + 45.344, + 33.069, + 47.88, + 17.516, + 28.985, + 25.592, + 46.249, + 26.321, + 16.4, + 47.926, + 28.428, + 32.03, + 43.746, + 13.489, + 23.489, + 24.971, + 44.848, + 26.109, + 16.288, + 59.455, + 31.738, + 54.533, + 31.024, + 15.45, + 54.872, + 43.142, + 53.955, + 26.396, + 13.59, + 81.562, + 85.848, + 99.61, + 29.048, + 81.321, + 96.334, + 94.105, + 30.329, + 68.265, + 15.076, + 33.898, + 42.978, + 32.224, + 26.817, + 14.543, + 40.486, + 33.275, + 64.941, + 87.616, + 14.207, + 89.782, + 95.02, + 94.956, + 32.855, + 76.107, + 98.239, + 29.44, + 32.916, + 39.612, + 16.757, + 95.713, + 24.909, + 58.909, + 28.029, + 21.119, + 28.143, + 39.233, + 29.934, + 49.644, + 16.096, + 29.481, + 23.886, + 34.195, + 32.284, + 10.631, + 27.481, + 45.854, + 25.425, + 26.335, + 31.901, + 35.496, + 33.333, + 38.222, + 22.504, + 14.208, + 51.839, + 29.766, + 28.698, + 51.317, + 11.732, + 24.94, + 23.859, + 37.701, + 29.059, + 13.654, + 23.938, + 49.347, + 29.242, + 44.677, + 11.713, + 23.143, + 23.863, + 44.612, + 28.72, + 10.638, + 19.093, + 40.358, + 22.305, + 24.107, + 15.531, + 43.795, + 24.703, + 26.33, + 45.441, + 10.605, + 28.829, + 26.359, + 44.608, + 24.885, + 9.694, + 51.0, + 32.55, + 84.92, + 100.15, + 80.543, + 33.003, + 90.587, + 93.959, + 31.718, + 58.77, + 23.61, + 19.047, + 25.85, + 48.775, + 11.448, + 18.461, + 20.808, + 44.661, + 29.105, + 11.039, + 47.975, + 24.701, + 31.899, + 41.064, + 6.316, + 27.192, + 26.082, + 36.206, + 25.636, + 9.336, + 31.87, + 43.469, + 27.979, + 24.175, + 14.243, + 38.998, + 24.143, + 24.693, + 47.109, + 12.64, + 31.142, + 43.818, + 28.39, + 20.395, + 10.962, + 49.879, + 30.469, + 22.693, + 44.794, + 10.802, + 25.072, + 24.462, + 50.174, + 26.665, + 11.305, + 20.689, + 41.724, + 25.807, + 23.314, + 32.965, + 29.665, + 29.896, + 45.439, + 22.382, + 12.339, + 25.592, + 46.535, + 29.978, + 26.264, + 33.089, + 22.253, + 24.322, + 26.836, + 39.484, + 12.769, + 33.161, + 48.482, + 30.592, + 28.419, + 41.563, + 26.232, + 88.828, + 94.404, + 17.809, + 13.17, + 14.605, + 12.939, + 12.163, + 17.05, + 6.351, + 29.59, + 12.934, + 13.962, + 13.24, + 8.194, + 28.136, + 11.073, + 11.44, + 13.271, + 8.525, + 13.534, + 15.145, + 19.957, + 15.23, + 4.586, + 7.643, + 13.186, + 14.44, + 12.101, + 5.218, + 23.997, + 14.062, + 15.389, + 16.301, + 5.499, + 9.551, + 13.838, + 9.739, + 42.135, + 6.411, + 14.088, + 84.046, + 16.355, + 10.682, + 59.285, + 14.72, + 77.987, + 33.938, + 78.974, + 15.426, + 24.972, + 12.948, + 60.896, + 11.12, + 5.576, + 14.536, + 13.907, + 17.874, + 11.416, + 6.915, + 11.355, + 15.591, + 18.22, + 15.379, + 7.392, + 37.818, + 9.909, + 79.447, + 25.23, + 69.895, + 9.437, + 21.673, + 75.374, + 11.022, + 5.933, + 12.611, + 13.791, + 67.464, + 14.116, + 11.551, + 71.32, + 10.922, + 11.998, + 13.528, + 6.365, + 11.341, + 11.955, + 11.816, + 17.068, + 7.987, + 13.681, + 14.823, + 11.064, + 10.782, + 7.007, + 15.08, + 15.621, + 11.074, + 13.426, + 5.877, + 11.275, + 17.453, + 12.131, + 11.253, + 4.612, + 7.88, + 9.711, + 15.211, + 12.32, + 10.41, + 24.212, + 14.609, + 19.048, + 15.506, + 10.656, + 13.558, + 13.52, + 13.558, + 15.732, + 7.873, + 12.77, + 11.592, + 22.288, + 14.72, + 10.219, + 20.19, + 14.351, + 13.153, + 15.29, + 9.068, + 15.823, + 15.942, + 13.793, + 12.737, + 8.749, + 19.247, + 24.625, + 38.181, + 27.516, + 31.352, + 41.332, + 27.264, + 26.79, + 45.817, + 28.748, + 22.602, + 35.304, + 10.111, + 16.813, + 33.041, + 17.014, + 35.225, + 27.903, + 36.363, + 43.977, + 7.062, + 21.775, + 25.202, + 46.044, + 22.506, + 10.575, + 42.915, + 24.609, + 32.568, + 46.024, + 17.557, + 26.863, + 69.519, + 31.012, + 29.179, + 28.558, + 28.197, + 24.527, + 39.21, + 17.122, + 11.406, + 24.667, + 44.859, + 25.61, + 26.458, + 12.65, + 42.206, + 29.13, + 23.488, + 52.799, + 12.299, + 20.672, + 22.91, + 36.788, + 32.934, + 16.301, + 30.94, + 43.083, + 25.693, + 27.355, + 28.345, + 24.604, + 26.633, + 44.242, + 32.078, + 16.241, + 70.878, + 88.34, + 92.705, + 26.025, + 82.305, + 26.546, + 86.294, + 108.07, + 24.761, + 10.391, + 46.963, + 31.265, + 30.964, + 50.167, + 11.048, + 17.999, + 24.229, + 43.279, + 24.981, + 12.687, + 44.475, + 24.188, + 23.339, + 23.63, + 29.192, + 23.613, + 24.588, + 43.868, + 31.516, + 10.301, + 28.532, + 41.217, + 37.557, + 55.671, + 14.541, + 31.114, + 22.7, + 42.422, + 25.863, + 12.622, + 26.144, + 44.993, + 40.278, + 39.847, + 13.804, + 29.776, + 28.066, + 45.459, + 28.236, + 15.841, + 47.882, + 20.406, + 25.142, + 42.355, + 13.631, + 29.502, + 30.8, + 38.998, + 26.48, + 9.572, + 29.393, + 33.559, + 26.323, + 27.718, + 10.408, + 42.951, + 24.15, + 27.428, + 40.416, + 12.873, + 29.851, + 45.22, + 29.347, + 20.653, + 17.472, + 39.705, + 31.77, + 67.86, + 24.692, + 75.149, + 32.806, + 88.074, + 93.885, + 97.04, + 13.305, + 100.075, + 30.317, + 50.801, + 21.641, + 12.251, + 30.332, + 39.221, + 26.38, + 31.82, + 34.689, + 26.559, + 23.157, + 42.989, + 21.478, + 11.601, + 33.743, + 42.465, + 26.652, + 23.668, + 13.815, + 42.217, + 25.684, + 30.222, + 41.871, + 14.494, + 39.306, + 57.547, + 34.475, + 44.834, + 18.545, + 41.294, + 49.319, + 44.941, + 52.106, + 11.1, + 77.539, + 26.66, + 91.583, + 85.691, + 20.491, + 75.317, + 98.676, + 27.831, + 86.327, + 16.994, + 25.902, + 73.716, + 30.476, + 43.424, + 14.355, + 23.562, + 28.396, + 27.922, + 24.862, + 16.767, + 59.225, + 34.29, + 93.772, + 89.233, + 14.321, + 86.181, + 90.719, + 110.529, + 87.371, + 14.28, + 17.555, + 32.579, + 44.496, + 98.118, + 17.107, + 32.325, + 43.77, + 31.724, + 22.737, + 13.188, + 41.259, + 28.331, + 24.577, + 52.194, + 11.766, + 16.073, + 21.675, + 52.929, + 27.424, + 9.876, + 46.362, + 30.17, + 37.264, + 46.366, + 12.937, + 27.591, + 46.767, + 27.37, + 25.796, + 9.768, + 44.054, + 22.649, + 23.186, + 48.351, + 17.217, + 21.595, + 20.556, + 40.586, + 24.078, + 7.318, + 29.572, + 44.13, + 27.483, + 24.982, + 36.202, + 27.618, + 25.034, + 46.122, + 26.629, + 11.314, + 24.228, + 39.277, + 27.659, + 26.945, + 11.29, + 40.369, + 20.806, + 25.837, + 42.055, + 16.838, + 28.754, + 22.815, + 50.535, + 22.769, + 10.661, + 47.034, + 35.795, + 84.321, + 95.143, + 16.33, + 90.905, + 91.644, + 90.755, + 33.584, + 13.292, + 73.706, + 23.632, + 25.231, + 40.067, + 13.812, + 25.773, + 23.929, + 42.145, + 28.189, + 14.568, + 44.144, + 27.841, + 24.448, + 51.874, + 12.269, + 30.747, + 45.326, + 29.286, + 31.875, + 12.014, + 42.684, + 22.647, + 28.297, + 36.058, + 14.691, + 30.065, + 24.851, + 45.627, + 21.087, + 9.929, + 23.285, + 41.372, + 20.675, + 23.825, + 12.657, + 44.022, + 21.452, + 29.354, + 46.492, + 11.386, + 23.277, + 23.779, + 44.275, + 24.851, + 11.564, + 24.767, + 46.321, + 27.61, + 24.757, + 33.649, + 23.184, + 31.597, + 50.921, + 20.637, + 14.082, + 22.963, + 48.391, + 25.521, + 21.002, + 15.099, + 41.057, + 28.113, + 26.638, + 41.637, + 14.14, + 27.108, + 44.247, + 22.515, + 20.255, + 12.441, + 60.539, + 18.447, + 85.926, + 90.614 + ], + "action_events": [ + { + "start_seconds": 2.002438706011162, + "end_seconds": 3.294201979006175, + "result": { + "target": "9425d69912fc:665365c7", + "stopped_container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "before": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:34.060806615Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:34.060879403Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "after": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": false, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:34.060806615Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:34.060879403Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ] + } + } + ], + "first_successful_bucket_after_stop_seconds": 4, + "recovery_upper_bound_seconds": 2.998, + "verification": { + "confirmed_messages": 1212, + "stored_messages": 1212, + "status": "PASS" + }, + "attachment_original_verified": true, + "monitor": [ + { + "seconds": 3.514, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "94.48%", + "Container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "ID": "9425d69912fc", + "MemPerc": "27.29%", + "MemUsage": "139.7MiB / 512MiB", + "Name": "mtusi2026lab4-api-1", + "NetIO": "13.8MB / 14.3MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "64.96%", + "Container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "ID": "48fedaf342e1", + "MemPerc": "18.05%", + "MemUsage": "92.42MiB / 512MiB", + "Name": "mtusi2026lab4-api-2", + "NetIO": "1.46MB / 1.55MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.62%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.22%", + "MemUsage": "10.53MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "401kB / 640kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "30.12%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "9.21%", + "MemUsage": "47.18MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "5.28MB / 12.9MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.28%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "15.68%", + "MemUsage": "20.07MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "13.2MB / 13.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "75.7kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.59%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "92.46%", + "MemUsage": "473.4MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "981kB / 1.19MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 4 + } + ], + "pools": [ + { + "instance": "48fedaf342e1", + "stats": { + "connections_num": 4, + "connections_ms": 40, + "requests_num": 857, + "usage_ms": 2810, + "requests_queued": 10, + "requests_wait_ms": 7, + "pool_min": 1, + "pool_max": 8, + "pool_size": 4, + "pool_available": 4, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 10.225, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "101.42%", + "Container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "ID": "48fedaf342e1", + "MemPerc": "19.08%", + "MemUsage": "97.68MiB / 512MiB", + "Name": "mtusi2026lab4-api-2", + "NetIO": "5.23MB / 5.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.04%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "500kB / 801kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.40%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "7.69%", + "MemUsage": "39.38MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "6.36MB / 15.6MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.74%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "16.12%", + "MemUsage": "20.63MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "17.7MB / 17.9MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "76.4kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.30%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "92.46%", + "MemUsage": "473.4MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "991kB / 1.2MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 2 + }, + { + "state": "idle in transaction", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 4 + } + ], + "pools": [ + { + "instance": "48fedaf342e1", + "stats": { + "connections_num": 6, + "connections_ms": 70, + "requests_num": 2765, + "usage_ms": 18063, + "requests_queued": 16, + "requests_wait_ms": 16, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 3, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 16.526, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "99.79%", + "Container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "ID": "48fedaf342e1", + "MemPerc": "19.58%", + "MemUsage": "100.2MiB / 512MiB", + "Name": "mtusi2026lab4-api-2", + "NetIO": "9.69MB / 10.2MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.05%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "608kB / 976kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.71%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "7.79%", + "MemUsage": "39.88MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "7.64MB / 19MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.25%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "16.41%", + "MemUsage": "21MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "21.6MB / 21.8MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "76.4kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "92.46%", + "MemUsage": "473.4MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "999kB / 1.21MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 6 + } + ], + "pools": [ + { + "instance": "48fedaf342e1", + "stats": { + "connections_num": 6, + "connections_ms": 70, + "requests_num": 4516, + "usage_ms": 32204, + "requests_queued": 16, + "requests_wait_ms": 16, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 6, + "requests_waiting": 0 + } + } + ] + } + ], + "target": "9425d69912fc:665365c7" + } +] diff --git a/evidence/idempotency.json b/evidence/idempotency.json new file mode 100644 index 0000000..e21e58d --- /dev/null +++ b/evidence/idempotency.json @@ -0,0 +1,9 @@ +{ + "status": "PASS", + "concurrent_requests": 32, + "observed_instances": 2, + "messages_after_race": 1, + "conflict": 409, + "invalid_key": 400, + "logout": 401 +} \ No newline at end of file diff --git a/evidence/last_stop.json b/evidence/last_stop.json new file mode 100644 index 0000000..17433f3 --- /dev/null +++ b/evidence/last_stop.json @@ -0,0 +1,238 @@ +{ + "target": "9425d69912fc:665365c7", + "stopped_container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "before": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:34.060806615Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:34.060879403Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "after": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": false, + "started_at": "2026-10-16T19:24:22.143921145Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:34.060806615Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:34.060879403Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ] +} diff --git a/evidence/local_availability.txt b/evidence/local_availability.txt new file mode 100644 index 0000000..2555c12 --- /dev/null +++ b/evidence/local_availability.txt @@ -0,0 +1,2 @@ +BASE_URL="http://localhost:8080" python3 student-tests/availability.py +PASS: both API failures and rolling replacement, confirmations preserved diff --git a/evidence/local_check.txt b/evidence/local_check.txt new file mode 100644 index 0000000..9a55330 --- /dev/null +++ b/evidence/local_check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/local_dependencies.txt b/evidence/local_dependencies.txt new file mode 100644 index 0000000..7d0b63a --- /dev/null +++ b/evidence/local_dependencies.txt @@ -0,0 +1,2 @@ +BASE_URL="http://localhost:8080" python3 student-tests/dependencies.py +PASS: Redis fallback, PostgreSQL/S3 unready, old session and original preserved diff --git a/evidence/local_environment.txt b/evidence/local_environment.txt new file mode 100644 index 0000000..a8152bb --- /dev/null +++ b/evidence/local_environment.txt @@ -0,0 +1,31 @@ +CPU=16 RAM_BYTES=8187711488 +Client: Docker Engine - Community + Version: 28.0.1 + API version: 1.45 (downgraded from 1.48) + Go version: go1.23.6 + Git commit: 068a01e + Built: Wed Feb 26 10:41:08 2025 + OS/Arch: linux/amd64 + Context: default + +Server: Docker Desktop + Engine: + Version: 26.1.1 + API version: 1.45 (minimum version 1.24) + Go version: go1.21.9 + Git commit: ac2de55 + Built: Tue Apr 30 11:48:28 2024 + OS/Arch: linux/amd64 + Experimental: false + containerd: + Version: 1.6.31 + GitCommit: e377cd56a71523140ca6ae87e30244719194a521 + runc: + Version: 1.1.12 + GitCommit: v1.1.12-0-g51d5e94 + docker-init: + Version: 0.19.0 + GitCommit: de40ad0 +Docker Compose version v2.27.0-desktop.2 +Python 3.10.12 +Linux LAPTOP-66HKAM8B 5.15.146.1-microsoft-standard-WSL2 #1 SMP Thu Jan 11 04:09:03 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux diff --git a/evidence/local_experiment.txt b/evidence/local_experiment.txt new file mode 100644 index 0000000..f0b8486 --- /dev/null +++ b/evidence/local_experiment.txt @@ -0,0 +1,11 @@ +BASE_URL="http://localhost:8080" python3 scripts/experiment.py +PASS: private bucket, limited credentials, closed console +PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection +Upload memory: 256x256, repeat 1/3 +Upload memory: 256x256, repeat 2/3 +Upload memory: 256x256, repeat 3/3 +Upload memory: 1024x1024, repeat 1/3 +Upload memory: 1024x1024, repeat 2/3 +Upload memory: 1024x1024, repeat 3/3 +Reconciler cost: datasets 100, 500, 1000, three scans each +Experiment completed; results saved in evidence/ diff --git a/evidence/local_expiry.txt b/evidence/local_expiry.txt new file mode 100644 index 0000000..2212020 --- /dev/null +++ b/evidence/local_expiry.txt @@ -0,0 +1,20 @@ +bash scripts/test_expiry.sh + Container mtusi2026lab4-api-4 Recreate + Container mtusi2026lab4-api-3 Recreate + Container mtusi2026lab4-api-4 Recreated + Container mtusi2026lab4-api-3 Recreated + Container mtusi2026lab4-api-3 Starting + Container mtusi2026lab4-api-3 Started + Container mtusi2026lab4-api-4 Starting + Container mtusi2026lab4-api-4 Started +API ready: 5e906f839a98:4bde0bc0, 8bb802a890f0:0923a18a +PASS: short TTL expiry + Container mtusi2026lab4-api-3 Recreate + Container mtusi2026lab4-api-4 Recreate + Container mtusi2026lab4-api-4 Recreated + Container mtusi2026lab4-api-3 Recreated + Container mtusi2026lab4-api-4 Starting + Container mtusi2026lab4-api-4 Started + Container mtusi2026lab4-api-3 Starting + Container mtusi2026lab4-api-3 Started +API ready: 355cebdc2aff:2f7b7cb5, 90f387d120f8:d12f4b2b diff --git a/evidence/local_failover.txt b/evidence/local_failover.txt new file mode 100644 index 0000000..0fd9145 --- /dev/null +++ b/evidence/local_failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 3, "first_recovery_seconds": 1.524} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local_file_failover.txt b/evidence/local_file_failover.txt new file mode 100644 index 0000000..c9482da --- /dev/null +++ b/evidence/local_file_failover.txt @@ -0,0 +1,2 @@ +BASE_URL="http://localhost:8080" python3 student-tests/file_failover.py +PASS: upload API stopped, original downloaded by other API with existing token diff --git a/evidence/local_files.txt b/evidence/local_files.txt new file mode 100644 index 0000000..894272a --- /dev/null +++ b/evidence/local_files.txt @@ -0,0 +1,5 @@ +python3 student-tests/attachments.py --base-url "http://localhost:8080" +PASS: PNG/JPEG, bytes/pixels limits, filename/MIME, ACL, attachment message replay +BASE_URL="http://localhost:8080" python3 scripts/experiment.py --mode privacy +PASS: private bucket, limited credentials, closed console +Experiment completed; results saved in evidence/ diff --git a/evidence/local_final_ready.txt b/evidence/local_final_ready.txt new file mode 100644 index 0000000..40224be --- /dev/null +++ b/evidence/local_final_ready.txt @@ -0,0 +1,2 @@ +BASE_URL="http://localhost:8080" python3 scripts/stack.py wait --count 2 +API ready: 7d0a6457eb21:3ee5bf95, d66f39ddc1de:9d084897 diff --git a/evidence/local_final_smoke.txt b/evidence/local_final_smoke.txt new file mode 100644 index 0000000..1e7b9ff --- /dev/null +++ b/evidence/local_final_smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local_persistence.txt b/evidence/local_persistence.txt new file mode 100644 index 0000000..793b001 --- /dev/null +++ b/evidence/local_persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local_ready.txt b/evidence/local_ready.txt new file mode 100644 index 0000000..e279b5c --- /dev/null +++ b/evidence/local_ready.txt @@ -0,0 +1,2 @@ +BASE_URL="http://localhost:8080" python3 scripts/stack.py wait --count 2 +API ready: 3ce028ba0a27:df4472a0, fd8770db9dfa:6ce14f15 diff --git a/evidence/local_services.txt b/evidence/local_services.txt new file mode 100644 index 0000000..ffdbdfe --- /dev/null +++ b/evidence/local_services.txt @@ -0,0 +1,10 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-1 mtusi2026lab4-api "sh -c 'exec uvicorn…" api 47 seconds ago Up 33 seconds (healthy) 8080/tcp +mtusi2026lab4-api-2 mtusi2026lab4-api "sh -c 'exec uvicorn…" api 47 seconds ago Up 33 seconds (healthy) 8080/tcp +mtusi2026lab4-cache-1 redis:7.4.3-bookworm "docker-entrypoint.s…" cache 6 minutes ago Up 6 minutes (healthy) 6379/tcp +mtusi2026lab4-db-1 postgres:16.9-bookworm "docker-entrypoint.s…" db 6 minutes ago Up 6 minutes (healthy) 5432/tcp +mtusi2026lab4-lb-1 haproxy:3.0.11-bookworm "docker-entrypoint.s…" lb 46 seconds ago Up 30 seconds 127.0.0.1:8080->8080/tcp +mtusi2026lab4-migrate-1 mtusi2026lab4-migrate "python -m src.migra…" migrate 47 seconds ago Exited (0) 34 seconds ago +mtusi2026lab4-reconciler-1 mtusi2026lab4-reconciler "python -m src.recon…" reconciler 47 seconds ago Up 33 seconds 8080/tcp +mtusi2026lab4-s3-1 mtusi2026lab4-minio:2025-04-22 "/bin/sh /entrypoint…" s3 6 minutes ago Up 6 minutes (healthy) +mtusi2026lab4-s3-bootstrap-1 mtusi2026lab4-mc:2025-04-16 "/bin/sh /bootstrap.…" s3-bootstrap 6 minutes ago Exited (0) 34 seconds ago diff --git a/evidence/local_smoke.txt b/evidence/local_smoke.txt new file mode 100644 index 0000000..1e7b9ff --- /dev/null +++ b/evidence/local_smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local_smoke_after_failures.txt b/evidence/local_smoke_after_failures.txt new file mode 100644 index 0000000..1e7b9ff --- /dev/null +++ b/evidence/local_smoke_after_failures.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local_start.txt b/evidence/local_start.txt new file mode 100644 index 0000000..8313c39 --- /dev/null +++ b/evidence/local_start.txt @@ -0,0 +1,260 @@ +bash scripts/start.sh +#0 building with "default" instance using docker driver + +#1 [s3 internal] load build definition from s3.Dockerfile +#1 transferring dockerfile: 922B done +#1 DONE 0.0s + +#2 [migrate internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.04kB done +#2 DONE 0.0s + +#3 [s3] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0 +#3 ... + +#4 [migrate internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#4 DONE 1.0s + +#3 [s3] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0 +#3 DONE 1.0s + +#5 [s3] docker-image://docker.io/docker/dockerfile:1.7.0@sha256:dbbd5e059e8a07ff7ea6233b213b36aa516b4c53c645f1817a4dd18b83cbea56 +#5 CACHED + +#6 [migrate internal] load .dockerignore +#6 transferring context: 205B done +#6 DONE 0.0s + +#7 [migrate 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#7 DONE 0.0s + +#8 [migrate internal] load build context +#8 transferring context: 2.44kB done +#8 DONE 0.0s + +#9 [migrate 2/12] WORKDIR /app +#9 CACHED + +#10 [migrate 3/12] COPY requirements.txt . +#10 CACHED + +#11 [migrate 4/12] RUN pip install --no-cache-dir --requirement requirements.txt && groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app +#11 CACHED + +#12 [migrate 5/12] COPY --chown=app:app src ./src +#12 CACHED + +#13 [migrate 6/12] COPY --chown=app:app migrations ./migrations +#13 CACHED + +#14 [migrate 7/12] COPY --chown=app:app scripts ./scripts +#14 DONE 0.1s + +#15 [migrate 8/12] COPY --chown=app:app student-tests ./student-tests +#15 DONE 0.1s + +#16 [migrate 9/12] COPY --chown=app:app tests ./tests +#16 DONE 0.1s + +#4 [s3 internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#4 ... + +#17 [migrate 10/12] COPY --chown=app:app contracts ./contracts +#17 DONE 0.1s + +#18 [migrate 11/12] COPY --chown=app:app lab.json ./lab.json +#18 DONE 0.0s + +#7 [s3 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#7 DONE 0.0s + +#4 [s3 internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#4 DONE 1.2s + +#19 [s3 internal] load .dockerignore +#19 transferring context: 205B done +#19 DONE 0.0s + +#20 [s3 internal] load build context +#20 transferring context: 241B done +#20 DONE 0.0s + +#21 [migrate 12/12] RUN mkdir -p evidence && chown app:app evidence +#21 ... + +#22 [s3 2/3] RUN python - <<'PY' +#22 CACHED + +#23 [s3 3/3] COPY infra/minio-entrypoint.sh /entrypoint.sh +#23 CACHED + +#24 [s3] exporting to image +#24 exporting layers done +#24 writing image sha256:6737b5a1fceb3f5d6035e6ea1ea9c32f3dffbfdb7912203727f68d41bfe41ba4 done +#24 naming to docker.io/library/mtusi2026lab4-minio:2025-04-22 done +#24 DONE 0.0s + +#21 [migrate 12/12] RUN mkdir -p evidence && chown app:app evidence +#21 DONE 0.4s + +#25 [s3-bootstrap internal] load build definition from s3.Dockerfile +#25 transferring dockerfile: 922B done +#25 DONE 0.0s + +#26 [migrate] exporting to image +#26 exporting layers 0.1s done +#26 writing image sha256:6203bb8001386f7e6b955bbcd5e3770c21a56231807036e5aeea33c25510bc97 done +#26 naming to docker.io/library/mtusi2026lab4-migrate done +#26 DONE 0.1s + +#3 [s3-bootstrap] resolve image config for docker-image://docker.io/docker/dockerfile:1.7.0 +#3 DONE 1.3s + +#5 [s3-bootstrap] docker-image://docker.io/docker/dockerfile:1.7.0@sha256:dbbd5e059e8a07ff7ea6233b213b36aa516b4c53c645f1817a4dd18b83cbea56 +#5 CACHED + +#4 [s3-bootstrap internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#4 DONE 1.5s + +#27 [s3-bootstrap internal] load .dockerignore +#27 transferring context: 205B done +#27 DONE 0.0s + +#7 [s3-bootstrap 1/3] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#7 DONE 0.0s + +#28 [s3-bootstrap internal] load build context +#28 transferring context: 241B done +#28 DONE 0.0s + +#29 [s3-bootstrap 2/3] RUN python - <<'PY' +#29 CACHED + +#30 [s3-bootstrap 3/3] COPY infra/minio-entrypoint.sh /entrypoint.sh +#30 CACHED + +#31 [s3-bootstrap] exporting to image +#31 exporting layers done +#31 writing image sha256:0f35e9da12772a90417ce81511aab2c03dcc88a5134dbb734e13e12a2aa4afa4 done +#31 naming to docker.io/library/mtusi2026lab4-mc:2025-04-16 done +#31 DONE 0.0s + +#32 [reconciler internal] load build definition from Dockerfile +#32 transferring dockerfile: 1.04kB done +#32 DONE 0.0s + +#33 [api internal] load build definition from Dockerfile +#33 transferring dockerfile: 1.04kB done +#33 DONE 0.0s + +#4 [api internal] load metadata for docker.io/library/python:3.12.10-slim-bookworm +#4 DONE 1.8s + +#34 [api internal] load .dockerignore +#34 transferring context: 205B done +#34 DONE 0.0s + +#35 [reconciler internal] load .dockerignore +#35 transferring context: 205B done +#35 DONE 0.0s + +#7 [reconciler 1/12] FROM docker.io/library/python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db +#7 DONE 0.0s + +#36 [api internal] load build context +#36 transferring context: 427.92kB 0.0s done +#36 DONE 0.0s + +#37 [reconciler internal] load build context +#37 transferring context: 427.92kB 0.0s done +#37 DONE 0.0s + +#9 [api 2/12] WORKDIR /app +#9 CACHED + +#38 [api 6/12] COPY --chown=app:app migrations ./migrations +#38 CACHED + +#39 [api 9/12] COPY --chown=app:app tests ./tests +#39 CACHED + +#40 [api 3/12] COPY requirements.txt . +#40 CACHED + +#41 [api 11/12] COPY --chown=app:app lab.json ./lab.json +#41 CACHED + +#42 [api 8/12] COPY --chown=app:app student-tests ./student-tests +#42 CACHED + +#43 [api 10/12] COPY --chown=app:app contracts ./contracts +#43 CACHED + +#44 [api 5/12] COPY --chown=app:app src ./src +#44 CACHED + +#45 [api 7/12] COPY --chown=app:app scripts ./scripts +#45 CACHED + +#46 [api 4/12] RUN pip install --no-cache-dir --requirement requirements.txt && groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --home-dir /nonexistent --shell /usr/sbin/nologin app +#46 CACHED + +#47 [reconciler 12/12] RUN mkdir -p evidence && chown app:app evidence +#47 CACHED + +#48 [api] exporting to image +#48 exporting layers done +#48 writing image sha256:ee25a4e40dec7eb92c2d6ca1b5c0cbc473cf40883f6e6c24bc59346bbed17911 done +#48 naming to docker.io/library/mtusi2026lab4-api 0.0s done +#48 DONE 0.0s + +#49 [reconciler] exporting to image +#49 exporting layers done +#49 writing image sha256:fe4ad675e910216faecbc2b15a9850bd8f1dc78b7f78b987e099614ed137bb62 done +#49 naming to docker.io/library/mtusi2026lab4-reconciler done +#49 DONE 0.0s + Container mtusi2026lab4-s3-1 Running + Container mtusi2026lab4-cache-1 Running + Container mtusi2026lab4-db-1 Running + Container mtusi2026lab4-s3-bootstrap-1 Created + Container mtusi2026lab4-migrate-1 Recreate + Container mtusi2026lab4-migrate-1 Recreated + Container mtusi2026lab4-api-1 Recreate + Container mtusi2026lab4-reconciler-1 Recreate + Container mtusi2026lab4-api-2 Recreate + Container mtusi2026lab4-api-2 Recreated + Container mtusi2026lab4-api-1 Recreated + Container mtusi2026lab4-lb-1 Recreate + Container mtusi2026lab4-lb-1 Recreated + Container mtusi2026lab4-reconciler-1 Recreated + Container mtusi2026lab4-s3-1 Waiting + Container mtusi2026lab4-db-1 Waiting + Container mtusi2026lab4-s3-1 Healthy + Container mtusi2026lab4-s3-bootstrap-1 Starting + Container mtusi2026lab4-db-1 Healthy + Container mtusi2026lab4-migrate-1 Starting + Container mtusi2026lab4-s3-bootstrap-1 Started + Container mtusi2026lab4-migrate-1 Started + Container mtusi2026lab4-migrate-1 Waiting + Container mtusi2026lab4-s3-bootstrap-1 Waiting + Container mtusi2026lab4-s3-bootstrap-1 Waiting + Container mtusi2026lab4-migrate-1 Waiting + Container mtusi2026lab4-cache-1 Waiting + Container mtusi2026lab4-cache-1 Healthy + Container mtusi2026lab4-migrate-1 Exited + Container mtusi2026lab4-migrate-1 Exited + Container mtusi2026lab4-s3-bootstrap-1 Exited + Container mtusi2026lab4-reconciler-1 Starting + Container mtusi2026lab4-s3-bootstrap-1 Exited + Container mtusi2026lab4-api-1 Starting + Container mtusi2026lab4-reconciler-1 Started + Container mtusi2026lab4-api-1 Started + Container mtusi2026lab4-api-2 Starting + Container mtusi2026lab4-api-2 Started + Container mtusi2026lab4-api-1 Waiting + Container mtusi2026lab4-api-2 Waiting + Container mtusi2026lab4-api-1 Healthy + Container mtusi2026lab4-api-2 Healthy + Container mtusi2026lab4-lb-1 Starting + Container mtusi2026lab4-lb-1 Started +API ready: 3ce028ba0a27:df4472a0, fd8770db9dfa:6ce14f15 diff --git a/evidence/local_student_checks.txt b/evidence/local_student_checks.txt new file mode 100644 index 0000000..adb4fde --- /dev/null +++ b/evidence/local_student_checks.txt @@ -0,0 +1,10 @@ +python3 student-tests/test_api.py --base-url "http://localhost:8080" +PASS: malformed JSON/media type, duplicate username, blank message, outsider access, pagination +PASS: 10 clients created exactly 100 unique messages +python3 student-tests/checks.py --base-url "http://localhost:8080" +PASS: registration race, invalid input, cursor scope/types on two API, read-after-write, logout +python3 student-tests/idempotency.py --base-url "http://localhost:8080" +PASS: shared idempotency, scope, conflict and auth on replay +docker exec "$(docker compose ps --status running -q api | head -n 1)" python student-tests/database.py +PASS: rollback of chat + members; migration reapplied twice +PASS: persisted key expiry, atomic message/result rollback diff --git a/evidence/local_validate.txt b/evidence/local_validate.txt new file mode 100644 index 0000000..683be25 --- /dev/null +++ b/evidence/local_validate.txt @@ -0,0 +1,2 @@ +python3 scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/private_bucket.json b/evidence/private_bucket.json new file mode 100644 index 0000000..f66cd27 --- /dev/null +++ b/evidence/private_bucket.json @@ -0,0 +1,8 @@ +{ + "status": "PASS", + "anonymous_get": 403, + "bucket_list": "filtered", + "create_bucket": "AccessDenied", + "console": "not reachable", + "attachment_id": "0cecaeeb-42a0-4f63-9281-51ac7df57414" +} diff --git a/evidence/reconcile_cost.json b/evidence/reconcile_cost.json new file mode 100644 index 0000000..5f7684b --- /dev/null +++ b/evidence/reconcile_cost.json @@ -0,0 +1,194 @@ +{ + "status": "PASS", + "scans": [ + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 0.692223, + "counters": { + "objects_scanned": 100, + "metadata_scanned": 100, + "list_requests": 1, + "head_requests": 100, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 70963200, + "dataset_added": 100, + "repetition": 1 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 0.696161, + "counters": { + "objects_scanned": 100, + "metadata_scanned": 100, + "list_requests": 1, + "head_requests": 100, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 71294976, + "dataset_added": 100, + "repetition": 2 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 0.808335, + "counters": { + "objects_scanned": 100, + "metadata_scanned": 100, + "list_requests": 1, + "head_requests": 100, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 71565312, + "dataset_added": 100, + "repetition": 3 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 3.238328, + "counters": { + "objects_scanned": 500, + "metadata_scanned": 500, + "list_requests": 1, + "head_requests": 500, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 72646656, + "dataset_added": 500, + "repetition": 1 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 3.476147, + "counters": { + "objects_scanned": 500, + "metadata_scanned": 500, + "list_requests": 1, + "head_requests": 500, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 72916992, + "dataset_added": 500, + "repetition": 2 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 3.516724, + "counters": { + "objects_scanned": 500, + "metadata_scanned": 500, + "list_requests": 1, + "head_requests": 500, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 73187328, + "dataset_added": 500, + "repetition": 3 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 6.510459, + "counters": { + "objects_scanned": 1000, + "metadata_scanned": 1000, + "list_requests": 1, + "head_requests": 1000, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 74539008, + "dataset_added": 1000, + "repetition": 1 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 6.586175, + "counters": { + "objects_scanned": 1000, + "metadata_scanned": 1000, + "list_requests": 1, + "head_requests": 1000, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 75079680, + "dataset_added": 1000, + "repetition": 2 + }, + { + "mode": "dry-run", + "grace_seconds": 600, + "chat_scope": "2713e220-b475-4a65-afd9-15484252e4ed", + "elapsed_seconds": 6.482201, + "counters": { + "objects_scanned": 1000, + "metadata_scanned": 1000, + "list_requests": 1, + "head_requests": 1000, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [], + "process_peak_rss_bytes": 75350016, + "dataset_added": 1000, + "repetition": 3 + } + ] +} diff --git a/evidence/reconcile_cost.png b/evidence/reconcile_cost.png new file mode 100644 index 0000000000000000000000000000000000000000..acb69ec5abb5bbee63509889f08b9f35f324e731 GIT binary patch literal 36285 zcmdSBWmuJK*ETxE2Ej%|!2%@&MM{t|5CQ3K5$Wy@MMO!RfJk?DcL+-mkY+MLTAE2n zOYd>vdcXJmp1qIn*WN$&I*#XXEuVYdam5(td5&{jA0)+vNQo(lF&GS~@Uth<7|g-9 z7z~l(u_N#|+~4T>;14b!U2+7_1jrY5Y+T+D2@ zuj*M@nOSn*xnumlE?_pb(7i+baZnF#a@_2hk|hRn;SBoU0o|w$c$6=L@RNtKcHy%F zwyHtad-aR!ni7K5OmAY{eu;=SXOuL3O7QKw$Jmz;5;CBzByT%WereFS;9k?a&=AM4 z8|sq5C6<|ZwR>6^fHI;_U+lw2@Hnr`JqP`Gm9HfeLjAB*|0n^ zJ)I&F#E>GBB)+-5+!_=VG(IK$nlY5iGV!SoMN9~*E_dlU3?2C~nCG;m@g^1i{{DWe zskUoOirE4OkDQRq(`mN)_0MskU`3-KW;NwcpFRtM6u&@MYaB^}gbS6nCv_|sEl~`!oJ2<2o_2wFMC4W+Pn5H!zD8=b) zU8*quvFhY}$0f7OcA;C#pEuC}m)g+Txm;=>{Onm%A}bh0lg zD`UKRRRBI+c71+ewO#0nl7Ep=?>(j4fq7&jTmomIRq zjb%yjTdBCGN<~UlF3yn~VqqL1405T8MdsGl?7O?W-K|XI-u9CbwoA^lU0t%BWwxe- zLaa|!RTaHjY5wZ+<|<*b*R(fR`{vcF)up~Lj*F|S1up7|&oAa#j5qRZ%?*^{OPjN* zb`2VX7ACtaD$LlwUog{Pv4MlfNM;gaC5_qPEP9XlneP&b8I~i zWovkLFD53&o_fK;l5=Ub;&+L71UF7O%HQ7~o>DQk$UL5wRe4WcUA?=0<@fKq4UtdecQ9!^gNAY%d&b7zy#vIFcceeVRe0R3Dec(RLJ%bgF zzASayyITv1P8&*MY1Z9kn5{QS|Q zH`dJXsn6jPcozRT%ds^(R5e6-#1NlpET5sQ9ifu*(&K}V4;%LRix<&cX2WjHl^5Q{ z-_vgVGSM8S^4LUP{w%7sQDNrUO>w>!)2EpIf~IXHyEEQZ7h`72MJ zJ$ttKR+Q6Ab(an8u$@fKQ-j~}>ot=zFtnj;`Y+E;X=!OKUtW-Q*;!REd~Y~dUZPZD zknJQC94!B?J*h(2 z8rs{{%^QMw%^bmRHnWe!W@a2nMo<^0+jt>p|XH}Yu31;}m z{$Mg+I5s*u+U$1VVEXbvnQd!c^M=L@JY!dsTC2a33_?Kh{Bj3<~+2un#x-DF~7YTd2cUh=jMBRQDLQs=b0G1+a# z1Mwp*bHm}_h%f9wr%YH&H)(0dQwbH4FqkRw8KE$c`Ffi7?%m5Ys46hTEBo&?+=?12 zYu>1)xO~~DC6d>Wir-bfRIA<(FU$D)6eg2&;o2dLqFl1%*Ub?E0fiRlwO(Cpd~!0K zSor;RovoD>E9b4bB5Zsx!~m@S*0B3Y5iE!4pwaSR<(;Z+X{>CLI2G^4Sb&0D9LrOP z9>Fxc@Oi0{F^?jRQ)N?PZ$;WY!mbS14yi5WsN0PkhR`*6aiJr@n}>&oMp&iLC@18B zh9y2z4F@lrmb&3EFoj=#)XKB$v%S5YndX-u8f*Xo0vo?LTCY;Yp|G^GJ|xP$zGJu2 zF0?$LGxq%xHXi1PNwv6*oBGtLtHL58BGTgGE!H?vOz}NVp$m{TxS7x2q;~rK(0Qrh z*3!~a?!}>E^SISSYN`h|<*8c#)YqrMR>8*O(iB9o^TnfnX_@BmNN$(yVwx1^t+n9( z%*UQ&4ht3QRW1>6){8o)O}VXCv%g;d4^No};96z_C;FI&^LBLD;Pv zvyn@c&FQ^Wb#-K6!L?y$pueT!iH^3mQoSG51D&01WrIp}&dt>d$r6k4%rZa12A}Yr zQx@~OGq%M!3*OYYvS?&v1SQ_pf6hwiHHWfyKS+U>l{fAy#KA0O%@348?pBo%!(dcY z?-66Z*S^e2e)-MIVSBNjU90W{gmO|UZduXKw{v>uYeY9-eKK0kS4;=*=yfLgHk*5$ zb;LWyX$+B+l$NU7Qu%H2?GQBfq@A3~`F8(k4Oq7FLV}c%mc|wg2obt7tT$KmJ-d!z zXb;2qm?;_m%x^zOk3)9iG=Fc2ZQ_PYf3J0 zJ+O(Xt`>ltBU)NH>6l@%LQrY5Ry8D@teOwr;#0dHHMC9-3D=*0x43d^{=wW(l?F^l zN493|s~p9Ai*XTO3YK4?21J;i>!)CbrV8-t0cmMzg{DI$#@kH7Mg11ut4D6+>9$r* z)EjrF_p&djWGJ&AK5|3^OMXY)sbP+sNts5Nn3$L$dp)A0q$I_Bv<{oOu%Vs?3y3^$ zc_^vY^Q6kq1v~bZsv)!Kj)X=?47OokzEIHKyeY!H?vzEt!=t*l(dudiVO|v5?5jtk zG?ZqyyQGV$9!CI1;T$z}rsI5xP)AAGqe#~)5s=$fFDyfp)!*CQ33d)AF>Sjp?9a3G zo<_f~(0CJqp{-nj;%#z6rYNVS534)I7r5N7~PYLC1w48hFte_^VdiFub~>81@_lB+Gzk zA0MA4zO{|AT6L$qQ0W8_VJzf1dpUL@x4au@BpaLVRNeOe77N>O+WSsrwqMn9%L9lj zSV&t4Da}C6l@}55vgV_w7v|qt2>>Y8!ZVT%Yp}cR_Yc zb0(F-=kW44fM6M0>?Zw+wgc9|G}FDhQ8>>lE_xNy%_Fs*>@TV@JCKC1EjuwIHDx_~ z%Rg9WI+OfeHm7feg*AtBSw6kux}8(GIm0O2F;_HXB4ZfAZA~N0sE{f9{Q2{TuJoAb zu^f%2#~1-AtCG5v3xxJK&rh#loL+Wxc4mzB6`69LK7G2o-rU@r<@M{=b$C3U`({JF z=@4J5R$^jesCtD%uaQEkEWJ#rl|Ete0;eix-4_UJy+)%1f}v$OK$$$)xjb8HJ?YL6 z8-w?=&r@s&o4#~pNqjs_XRfv=VHB3j*;A)pK;%x>;&oWb?wxeF)OpErM`I|>Ia9%~ zF_6Blv$J#a3t-!ubadZ~OG=dLA;i7PYXzJy275ST*~G*o@p;I-x|cbM^$^RjlZum< zenjp4US4K}Ipfe`QqI2%c`&DU;m_h|BEZIO<2UZ^VtRVX$4N;6EvA=@WF5c+vV}up zQ!P+Xw-@Xv33CgtqLJaoi_Sw%?glIo3lWat{(Z$HiKx#kPuUH+B;lJ@r<0 zudf$ED+%F=0l7ibX-IW7B^T2061YBw>kv&-@E@gZdWj`UKZ?Vx>9 zwV&+`jiEOykY5>Y-hAqQgj5=619~O|4F)zgSzgDrF1&_o2Hd}Q7R?fqdby&8)1n7p z!W*uxt`V?C&NIli0lrB5o~(-tJZ@Y?|}u&(i|JN7s!v2FZAmQLVt% zFaWFzjG>@63kF>oAlrP!y1HDZs#2181d6l}F*A6n4q*1ODQe3y_nG8H^LM8ZwG2N8 zarXa>KLf+!Na$J^WRs%-lFMZ%7KZL-Dq(ju*CS9CYak1alumzx_H8mdg^-+Zy=lG*N$VE}%YRgV*k zSXkuJ(BHq`^EIeiDd)(DUi!oPl!mAL(>SXo)c@N($>e@u}}xc-uIv;6qL6V6pG zRq}Pz^2$1yMBC+E`3@g9`!C1l?ne4CXSMw$?e{ZQ(l1xAIEK0O+X*OfO}q(VRW4yw zI!pd-dj9D(hYA&Aph?{6Ej5p>9)bZI4(!OY=vrvv&o^0E6g3G5(qfzQ-}xC}^;Cx1 zMD^FBYY6P2%Lczx%;jqSeneQK#$U=Sx?mx=CiM3XR(^@`gnHkjnfg&zIB(L?)5d*d zus8v17XC?Mb#|^1T)0@Cn|?WN`qCNVlFAKL$ zu7FR*_#H2Xe}9b)3P=lFNhkcox8}WKH=t8$MY;=y`71@7y~;_$m2Jg2VeyL0dOc;{ zWVbYn+xtIrJ7d~BujOE??!$2_ynMdi%(fu8wfWj%1=-UA`)@4y(wN^)Id0;cp1mrz zx!7Bfm}IFUk=4VNlUPd0we<~idkkX#7>r2x+fjYWva9lW99IHMlFOcW{v2n|mx+Hw z5~*Gy=ZHr z##-Oiy4kr+bFE$H;{$3;rDam@)fePd)aNrj)T=7~$}Q&{Q!e&yzQn0Jm%fyU>^^mR ze1|PxCeS2JPMqBPn@&s9fZX1+oPzs2$ZS$Y5) zKRi5K!%*_<-!HMTu=JfU;)>59d0DWa7}~6=Z)(fiS@@2+m_7fA`h?QQzoX<8G-OLl z7${-+iM4Cg>2Dek+zC_YJeyG~_p@){A@2Mg&|l!6Z#CWj$WXH1dCF*ev(KljRIzJv zf`QUbuou$3;#?9Hy>U8VeOY0eJ$d;TwHk6K?cU%e;*h> z#(Ht{#iF>FT9HO?Ng>lUN(GXxY-T9~#R4DqRK|<{UKYp-<7o7a+PTugUz*j@Xd;&3 z4H0O+2MR-(-uR=UHq(j*iOs6x7}JV2mPfplvXw94I^Msl z(X+9cJ#=UNp|9Ghq@8+Wm1lvA=6D#oWK$XCXXlRmd(quGz44Y;zM&W5jE_q6{je`9 zn`!@yV~duv4qA%!c%u1tY_rR%PKI|S+p}ys<5|MY6)t$sSI|B=XquK{k)U@?s^{+) z%<7n0q}JVjj|;ClIUZ70VIqiL3)j@~s7C$o6~qJWfmRh&c1e|6H?B#Ld{9X{I-LLm zaq7(QmtWx&e}^%mWFd7YaTeJHFCQLUkQi;X+%|_NeCHJwg23B(pue`2%bV@uTA51tBCWt8JEZ z22;B?W0uS;%_TjGPwhXpnv}IpPe~O!7BrjTL0HcX+8}shB+lj=|~~E}a$Uj$>KYVz(uX@K_Ga zD0INPog<$mxHXy;{2h$UhEJW-q1k##iKfpaTLhVCl)H{~i$=uB7K!K|mGd-|`gKqf zkpEL(%7orLT|yL-I2#qw77bk6Q=iW!49!gUHW(Q~W^s7F4BDfVD{L8b9ui?J73c zIw}ilBIcZP9R-swZ>p4~)_F%72pjlXIsXD;d>m6+13vGcy5mVw*Q?xhTO)CtLrym! zW3Da?4YiNMFJS=c{+y*`=X~lqn`eA&WZs~?D)jA4hzSs{`S*w`mWY5?rg(b5V6 zu=-e3R5Z~Nk>){ifq|D-4G<#&p`f3QP2Gdhbi1yFZ$DSTc}Je4@Ja(KKJFoHMcXxL zId`Yak{5|(KAd<9I5U_AxTQEkE9pvkMiYBG3$6qJ?rlIBCf_f?8@XK{ZA-}u%{EJN z>G=}v&HXG{?uioV+_@C)ApL7n(m~$^>_Czg?f`Y+x`Kj&Gw}SGfwH2Gj*b<;*akno zu*~4IqCj;_I6_LPy0{2J=Q7n$g=0}oO-*2Uxa!C8+Sj0~jZd8!9s@+c)_wJ|FS{Op zQfD6fxWD`6d~0pv_d>BZ1x;Hsz1`PJ6Abi)mBd%KOr%uZ0Z&+1TQddLGk#M;p9`iBwXtvMlS z%5|a=1QC|}(xua>^*(pADI)fyle$= zJ&iCzjvE`5x*m5RGx~`O-n4Z3rtsuXiC3>)eedjCD=e<}k1F$+_s~6q<{zuGQ#pzj z)1_Y$JwA@brOBq9mP|B6&b*gpsl48txp*uMByZ4A4VNdH;qywkZD#dAJ(F%<{)*}n zbi%*<;-@O?S(Ni@*Szj~3zp_R)aD(!_UPK{{Eu$3FYlb9=C(?vLHL>5YU-o)HtdKF zP~m8WC3qI!J~lWA_aLtQ8lb!S>tqO({5e?(iPWQ826~gvti3V?An{0lAbU&hEeNH9HjoJ~tD!QK z_1SX?--&nq=*ei0>}LhKM}A22lckc;lK#Dn zsN-H9`}Df=rd_W|!YjBeS*k_i@dLrs$8$^h9|c|`$pnKT3j}}&u#!!)U7R)nT_xEl z3cx5?p|%mOiunsQ$zJ%h_h)#rJGi8;-Rvv;NH;#7nAF!TS*4@7WRc`?i1_6GGqdw> zWdZC4>gfO*^DnMjS++lJu3eEOP%_Mo&VTteEW4j&+#-zrcknnfK=;I*yG!<42du}P z{aoty=(8D*7=Rdq?g@8$Y|qX+$gP@Cp?~dl?1%kDqw1OQA&JdE z-A8`@dErG{>l_5JDZ45Y8-tes%~j+6;vD?dUB%Qo=J%awNv6h zDtqK#{AG3?D}8Te1RD`rYEqSJ3AlOIOS?DscCrZlCq80zHh>h<*T5v^#U#nlSYTTbg^;s3Ts#We-`AGU7fd#Sn z^eSRA%pb!H*|rwk5r*0FM=N4H>y)pyIq=P{yx%BL1tX<+Uqm# z_XjMgfRjs3i~JvhmLboj9|P(|vFcM4F*EQ|$M;_fxKAiHbN#|Xm#r(!nEo|$!_}*# zgfj`7*ABq$$-c1QBp4DrBNDP`I6d20^a*1uPr={r8dlrTuDB zZ|3I?LLO9A$sSLhMHjuZ@{uia2pW3?>*nJA%|z1sRpXSx&a&G)3?~b}Z^(LKj1Z&S zhilsrdo(^W@hh76K>hxpM*qX|e?xKx0!=fNZKfc8tFgN zR?=0&N=k!7_YuAnYtKX6fo})*H^?^;2YJ6rrAt{SgXcRM@^6Qgv|YVII!%$%N05*A zIAnJI?`x6&KcFpy$Q?~mV0R2t{vF$R$cZ!)$q3c<+@R;MtfbC#NS7<7HdYd=%>RhV zytrq8g6}J19~cTtt(AJdKJqj3ndCcZ!pV{P3Xi|KpsOzIznEZ4W*ohPsd_rO5&uh8 zlbmO~0BeWIKBZEQjlx%A-S@xp`sg!}cYJCQ$-OE8R#Hcg7x>Yo#YEgVul9lbjKIVF zeO5gpo}fML+GK0kvEUf^$vEr~L4iI4tqZ05Of+t9{;sfE0Hk`2GtQyKMn&Qg1;*E; zn0gdYTngqs{|X=n5#5RXW#6M_BcAE9%PKW`;oEduid9(2RSjDdPxv@JzYhTN6wusX z7u6;{EXu_@@1M5%<67c$jV88v23D$>&Re0S;?T+^n3UJ{+P7c;QL=wtNvULz$Kltc?l#(T7|(uWyd zO^(fY`W|J|lnVC?Z^;f})M%7(|0$Gg8^>S~=W$4KtY=PR9CW&si7-=xt+hFhQK*MX zLMaE8sSh7MJYBW7lT{%CIjou#7~^1f?aX~m5A|JpZ!wCMb&JlnmQTtY@fq`mszG;; zos+V8K(+q$=#?p`b>llF_>HOh%51Zc>ZYx&odOywx5e1|BPXfT0Y;@k`Ff3k6xJYK zZy@ro3^K>B&pJOc5$hT1F`dFOLi~Ht?72B6Gjw2AHqT~GdBCbujF@i3VQxNh8{mY%>GPI<@~I=s)z_U$yKkN-Rg53>P1T$J$gydQSAP32 z`b&1-#hM#Je#sBrk8o&_Two4_VnL++#6@B0xTn5dAoUs#miMY_?+@SQ{{{*_fT#V$ zGbCMjAV{D4rb^4}pxMAI`7uiii@=DT)`7&9aIQH1y=@s}i14iEcYTPY)$U9+!>nBR zMzf3cGOVlZ9cr5nV&(TNfn~B0GsOWGvl3N|c27NM76KS1^wLxYov>Ht;m&2&gv9khM+qo`OG6D~6^a%O-@Xa%%h)EJiDJa0q)csEL)Lh; zrRC)&R&}U~5%H42XqpHACvZ{xDcV}(FPZf&sKk&?@=*`dDYA?Aok@A+M~(X<#O_iU|L?Bv>|=#xAg6T#uPwxSYuL8p1gHfn@vfTOXDuR4HN76?0#e- z2v6+GS?=}|+}+Gs5c1^WE;G?!bS3p({|3`TY0S>JUYrlHvlruV&`vC7FGv zE@?qoE-)a#vX;Gk?WdvF`}fG|Bb)P5kd}sKs{V>=mx^gc0x0VSHE)ko-ZvXQf8`c8 zH}}vY6z>^t-+m6OtaL|3FqQ@Fq8gM>U0V^A!PpCvekzia{h`AM}AQk-ctA(|>)2Tzl6dvTqgnTG8mp57> z9^gPtfTf^t_U3cR7twR$$H0rw7X8qj&2h~rL#?bZyAewR?<55e&*L!aiMA$mr$G1# z9@v!@-rf$W1v&8z^J%jGW~bgo06Dl+$nhZ181OFa%DWj(Mv zI?ZroN2Q=oOdHuZg6L(QLe;!+{W$tJRK2)<{rU2oQ+b6v=iPY9U5C=yW2Qs>Q3Es^#nXOf0g z0R{!IgrTwuC8vo~ZM{l`LxCSPZ>q^aX>WK?f01d_=Gt5uR6VR02GyV%UowK8(xV~` zAVD=(Mg~8pV3Ro~HX0_ouZfnX7+gFmcQm>% zHH6;N_G&I?c?uQ1O{iUU0kgiurw((xG8Q0#l8$nfO9h$>P^H-Q+7R>Z4v^pzE<*v% z=#|doOrQ1Q=8FbqD$(ZrkhIIg86XjN#kpvMOi5JD4)HB89$84#%4~38aEp;klb;^` zYKbpZNLQ4CvYIR?e;F0WO=Y1l#%BKOA0m4cafBNFALL!uIfGQw&eSC<#!`qp=hPr< zVPy*pfwUQhaY2&f7<}veKG+K` z;u?v6+U(svGXNxEK=pQM5s-wkjKoGzLy4+=EuUTOuX3#-qvDQ-0=KA1XQ~|I19shx6?5TJfoeJ0D*Z_?lyQbsP2qyT#ZuFc|E4^-YY8shVGc z0E`{h6Db|kD3=-9`SdZ#<4NA{eLmqJM|4@{vfG(vlhQNR@8Y?D<65GDj-U@1U$O@eZyAO(DwIO8q6Ubn`+l>FgV z>(3vDHyox)`V3|&>*W3=Aonl99;>bv0OwyH!3<_e@Eyz3re9b#0~4Q@WJx_gHtdT3 zn~Ibi;K6L=_DMv5VPy&mpK@Sv>K~X61`Q~@HgV_H%6vaXMxSH9PJi>aSXcIxRK)p| zTqR(CRQLAIIlP6cs!(NR<;+l36|6tatXcqQ_y3uXXLVQ#^DVXfH3Q<2K!%C@%__Th18$ zdq!ARNeioLlmgOSV!w7{%Z+NJIZ@0%S41wiMDo9b7X`Vs>4l{hS89`ZTl z$i$a+r#yr8D2N>((gXxCXH5q){Rp%|q}S9jy~ll(ndY$k>JkY>8Pb8i;?#aMeYAmp z4c5WHv2+x zK}}7>rl!^#=9#N8-KVwL^DX0J8PO_U)GiYEv({%y09;1N?^1q*?D7}gT~y(Rm(Bw8 zI}K)wLVX$75HBQ>I&#^}H<-nCoulsMg0#_}MIVZXvR z($i0;DoZ>O6htN*n`B_d!{M{xsyo}dv2}O6I%B+*pP?!rD8RVMiBsso=I~(mH*d~e zy5#t6?$@th=O`%jJiTB!#djSPU*j5FmHihd9u&@B<@QCD{46v^ zG-GT&gzgL-F=AiAgLZJ^ATpAl)I z;(qs7^pm_kiDCr}XE|2gmM-8k8Q{jAf$3I&S_}<6n5AKwA^fF{jgH2F8{^wP6uouU z@xEZ(Wc`1U!!AN1gG1$GF**9ju>4wJAhN*`hpcTYS*|;xV5BSq_12ix)JF=utnAh^ zvdDq=_g0_mmeT9)_;j|qQh$e1l$tCT>D=X%Wa0z}jhOrI)v{g}BSVgED>brUgJY%( zQWvdo5QCg_(`P1JlEjsbKff<>7&K>k^ z%>B;;cf8gY7uz47bp2z;R8V_VNr*oOum##r86J*xjjx@x)+mA^qNug^+&aYOZ3o`O6}% zg|q2n!L@;l74(M016&OXSrUj0I;>1-W_7@wSc+^~Sx{9|Y)j%N$oM76u6cKPdvB^9 z;G+h83gdMb0yXH}X)ZgPODikoV;qp(4cek#?`{(AI0ivQ$}5M;Q=EW=e9Ev_tn)8F z%^N5Rw%WdzOu(3;?J5`r(XTyD2xcjb(L@1ocZO149ohK<7`I5HfmJqBNk;2rG5&{`VrrWQk9)xA&%3w1 zj6w{8YKIRUN(L(tHXh1kQ;;O?)lH$8LuK2qxj)sXU-`6~s*oA}0AwvDUJ$}KHTAKiSXA;9qIvy;P$AJ|P;$J(r0yvRSbVc4ZUQx&)fTwF2dtV?K_C>pX2LOhq5Iu| zsr~~Z5yQV;Lu!{ln!+7W{M5@sodlF@dM_-GKppIyX-Uab%fibYUk>MZ@&E7430Mv0 zD4^bOPSeO4T$(Pad+-@@^2Ea=i8*{MqK)9Blgb0`|o~1X&v+8aPHXp)_wP`u@nEyv(?$Jy$X} zrrI80Aaor3ta9BuUj#1yI4aoa-(vCq*IX+)cp!g22xRkL=oDJ=wl!@=43GRTUc)eMHo7E7AN8ku0rKwKo;~z zjAU2O9CO^`cPBovJO#0(4NzrJK!6r&)vJaxyuYgt!j9{N8_&uCOej8}doPzAhF{rV zJ(b?@mvPMt*~6}4^RC{BqPz8MFG8+r0@|DIpw|a*cKb55LnhQdPOCs(M!r zU<5g@l0@EB`Q!C~ zg@eFck~e2Cqal{o*y)SD`SSjrgVc!Bpo>~iw@&ZxS4BNBxS$IlDtR$z`N+>c?@k6M zj?4E#kGDl-X-D$+s})UA=mzOMyr&{{a};I4V<<#LjlrlUG#|eO6)zw81wI|t`H7e9 z&c10gy?qAB;ru{%qVMObTzxJoDg!3CH*emIfvb~>-~88GWT41UDN2QUI`T1sN!SF1 zplaG#F>xHy_4CFhBTM=Sa$VqdNF|>QYmI~$UIT;!ocG8QzXEDXGPsEMH!`B%;ETqx zOwNm}0irN~I`GQ%LzwD^*w$AD+l@?{F3yi2FE-b>qsToYcN~eAZbaHeg=fJAE@fv| z%zi?o12l-ZhzGW*VB!tscdd+coadswbql#M4!dka^FoShi3Q-kRCw%bem+>GYEVkq zEn;}c%RdyJ6!pR?L3(p$q%L$*?<{1+z^kw?i(U6z*o}JHbY_D!Tt0n!*&Z((aPG{R zSjY={gXQ+24{S4G?^h77GbiE0!XL`G-b&=d-mSC9I_|8y~}HAM@Z<1CsRAlvN%JV$KlN$ z2m%URB~SBqMY?(p|7!^Oj{j>2Vb5eZ$yuf*c?gru3LA|0JcAy%Q@dL^0hgV;a0`=Z`?)Rz-MCnk+b z)%p3Ar$m_hMSvQnz#*nwv|ij?T$U!dXK*HUi*S^o4Igp$7)HFtgntgqrKuXOJ95bD z9B=6wVFcg-iIM{o-#^{>S#F=VFL!cHAl(v@7>ea!L`?vXw8B+?5+F%=52k&dwtC~r zjXl9QmZc$#%6@tu?<^D@#oW>=66i@+Vr~Ay@>r|RM+#OnVrhu)F*EO#-_Zo>wzR&k zFxe7e`o|8;^oSRO?jbqMnS1~fU3~xlNZc@fI(WiEXx~T#kw(qJ1vD}U+>);^6WmYo zwE?wY*YBVK)qkK(3sTtRgG$GB6AurMMSRzpv91&uKjs7Ce(dkxV6l-IS>0DUiY}dG zyzv09lPPbn!Q031#P_kNXk)HD)&KCTf;>JpGXs8Awd+GyvtgZ9Y|f;CbW%X=Fm+7` z2(R0~t5=gVo?=35`U4LmL|O|&NRI#Rl3{verSd9TmH5w16WEd|;N$_irVPDXn}0_A zNyZkzSn_${&jzdK*RNj>TU05E=|B_SzEWwOQd3BMCpfY zYjpYtaIuwQk5`}fQ4%jLBXH;TEBeWiM_SJ-M?qDSDKNmHZU*3by(N$oiR=Ee^ma+M zdl<@)$5+hve;W_Kh0PrMA5@Uz6Hq~!Km!Ga@h`t1ikty$$L;#Otn%lZ%tHwcRSQkP zCtRMny|eb~$Qa@5et8_o7^*LxlbTlI3e9zhLU1$*n1U4V-{RXO6yJc#VTQTSrwA0k z{6-8NC29Jdi5^S8$jQlV8rClmVI*Zy!D*8JEW3+TIGI}4quB7Oq$36Xk_%yJs@4!x zFw6X7;^Xh8i#6(jt zN@vl5P8P)SG$$vg6zt!tKyeL$?D#R0fJ#v0GE_o-=F%uoSi9oVsjn)JSD~uFFiIE! zc!i;r`$YT}204I+ir+;QegqVv>9X=k490_L&CCnxgYB_YLdJDWM;;7~7hms+7J&8lBC=Tmk-$xeU^V z894I2e0;iAraSFj4`HI8{4}4+hq82BLXI%UC^I?Oz2(@Fo!>GXwtE0r4YLHvoUIO( z=iWZV2z;;{eiB;8Q!E^(c$*0g7D82{3IDO;k6m+3=&i3ZvGE_JnsmLKw2Y(L9xT7$KyVbN# zM_UW-(`E}7%AZ^!@W|0r3XH%fU3}qLz+JLg>J_LHDGeIuQl%5TR8^O%nr)!0GOGmQ znF6xQod8p*>`7)c%=bI|wt6f9G96wc7vy9vOW zxy)Hy+%JORAZB>Uhh@{aNfV$c)W*O&A>vp@m@XV~_}zWiz63N_jAS|7f$z_-yDIlV z44sYqg7`j|avBb7E=~&brZ$lhlgKAi02?$Stku2%oWptSb1TRWLm9Wv<~PF1IIrSz zCK_bRe>itTHYPt9%3YY~7d@-ICHEC)I33CR4P%_3uzd~-M ztJ6IpC!Dii1t(^DBU=#b*E$?fnaTg6G8nqcHYeY1tuNqVj0-F$#SxlR?fT7p@HoY$Y~DaK?*CiC;9b8 zYCI73TAS^SgQgjE6bPVZsjqqxQ+!jH{^F*~jtj`Is({DGC#Mq=aqeINKNYmPp5M#C ziPvlS{20n4@{9EyMiWF%{V0;p^sxm(oWLYOndH9tuP63)c6!jr24z*G63_?(^ZZD| zyZdJpvE+9EAOPI_1j;VJ@;Ndnj#J09)Bn=RG5et*ATXiXbmHLfZzl{GG=YUa&E_Uy z!2Ur#`(l1`Au>6|3;s1Zl_N%PMh(mcEP+L#?1qWZ;VNu@=X6l~WR$BqDnJc75ST#g zXjM-*fbog}xpj4G=_GvFHn#88IEC1jR#hQQq%8d%+l)cyA3B4$Z?TRh$J?v>3{suw z(e@4Z2MPA+(|6Fm@?xjIiO^570KH6di9h&ljng!$_%xvxQPA|qpiS>xyBP1*VYsT8uQE=-1k-WA3^2;4C!=O({wz>g_Z#pG+G9Pex1mF^5qi$@_%WDCDluU^ z41p@XQY`JLfnu;SIKw5g1rLCxQPM*YfS4s`WR$+Tj4}sB9AG;&s48OPQ9THS7eBv3 z1tsv==xuJk<$*2^N?Z2ybp3;&XO0}e{BeaLf?^G-M1Q`mbY0j0x~P#d2%SW!&=^Dv zHD)mI-hyZf71_T1S+Obz_EMkK6FIVT=m4hJ4MqVZgaWO4GAQ#@b8|p{C+}L17V(J_ zve2G~04kU{l>ycNeT%8*PYc!7VK>F+_BkGd!vW4;yr|DA4;^UEP=i1SW=`%1Ci>9t zZl4nhjM?FtJ^K|_Y zbw77QNFs^Qs~?t0Ad3U@&T0u){HatTO!S}Jz5-S#hqfVGCnO0Ci%ZZ)MM}-12&J|5 zx5v*Sp+EY4Qv>^^j&XV24}ZBv>&lbAq4S1;u%J{O1Os2*3t~O=VUbd@p%WGEl%PH| zfQ!-Kpx$4*r`OkC2RLZg$r}svXtSzkS?FK9XxxxAz5Oi|N_wz-FUWs@P@)3E52{Nk zSdFua)qn*7tJXy#u3#_t*L@?dfb>w;q}pbY4cSM1=1jttNlNOEa8>9HDs$=o;;Qk; zR>6DTER8DdC??ttWCSZ~Yq_US>&_Q7eg7^Av6G4c1_UDN&gS~HiVcDd z7%eMJR!YItb6A}^{>g#U} z8G}I^{WmUMsT0wDuaDqaa4_zc&*%>^VpXq5&l;Rj5(474Jdl-Ml?3YXV~L^+O@A%CFSVVW;2UXPXogl)v5p;`K-?s8?rK4(qGpq>ZQmYier+glso}yTrg~1HKV$)OY zaVXdRt0`1}MT*LAC6xls-Zg&@>~wo+BEW-vq0Y4enlGrST_BGk)(=Yt zP9OPXHNSHm+f`_+g0d58p@ZFpIf{&bMJ@v|8XQ;hFWvzs(T!5UG|LF$ZagY@-iL9c zO?l)&3|*z4y0bL01ii`WuU&hHMN|~n58<NIL}nn|LD&_%&q+KYWj+MmfpL3FZyJ%0kaOyD+xiXgN>v% zRK1d}FW%|_yShG5B9x6GRt-;CU@(Vg!E*=IqD}?FHTw0~$TyE42u-7c-Av7R3`lWd z1M~MvUwkFv-Tqw*U_YQ4)pD|ckR}TU+(b8ubltUg3_66Ndo}4mOf9>W)5LCHU`$cI zoY)9;wM(nUlpx9g`-jg4-;X77y>sv$fnz*kQTPSBy}{3Xbb?C)l53zAl#3$Nque`O zawlCjJR>)30{qn5UQ{|LX*h4`o%O9sIv_R*0Uhbn$VUX*4xub-BPc|7@9jd>;U_fq z!@&Yj34VfvYPuH}d$~3jVk2hkq$D1K8@i%xWY3YfJ38qq?5)=WgOBZi_wsOJmCEgM zh=_=+K+q7uLOM!By`6x9kvt^n<}wtNKEhgL&??A#SVbrh7seZVp7Mzb;8L$MWrrU@ zmjYrk{QS^RjFc}_l?Nsoo_t%SFdd7HU*i*m zPAHZjdpaC|0gQnFg9B};kPx$b=fP70ZQ=U^bLk`W0?b|6g1X%o9^S)S>?EDQa+cOD zY+ralsRnJz1yGkOmKqsB&Iv;44(6$0U_cFY=6n+^pLXbUL2(u-Ja~P>U$A7fyg4-b zWwJ%|E1Y%J9Ebrooj|y0z26dk{uxFg34$u}L80acNFy12w_yu_&&c7|hg;~whA9)` zb@h$-cn|OFRV5AYtegY*(*vPrWpxz?RGt)FAz^`@CTPQS8 zAP>oOb8nyV&u7o1TaN^-Ec6+sgjeljEeyWaWguZ>H&i^_nZdZIA{JMh-YbqLp$NsCObtcSFwW5iYwMx&!9*RAiZF zFkYXb@w0+gw@gR9OC}q5pE^Nar3jz#OyVK%)Awhnj_E?{5NL7@O-*YzkAm8>Us4mg z;w*cV?DEGjl@Gi3Fx7G;suEw1nv!;sQ0+foc=Xp45wQ$tZOd;T3ZvQuV#fdn^S^SjInxCI^$yxmU+sup>Gkl+Z zT8XmguDFc6Mwd)mrx?VA8R!YypW8#0jRJo_4`AfrkY_jS{?SQt8WaAS<*8*6bvHDV zPJh?$i+ls^b{&l9@@o3~f>$r|I^?4vf$^E^$$1QwO_>CdAU5NlY$c@@WmcFLT5y)D zhZTr5-I<|R1S9JW$1W-F(EZ;lfl_KRRH+6#Gg3ZXF$M?)Ju55g(A;fuZ}9&uN0d=7 zPqyNJee=$U@_hGh2Fzs0q?l>2H#IfoRBsDGKRf#_$cGo6MVOUC5acR7bibrV)jg!Q zWYb{hnwWBMvd^q*=E+sFf@IeK5r=GZTKm!uQTpP3QGdIKkfAF)Jp9pvPf#r`)@i0J zFdb5dW~(3?IKLqkg1HUX8l(pr;ad`sT~K&JKG9UzHOR@f(#V+AlW+uM7Eih*G0pG2 zM!#!U+H4olCQaF08TzQv%pMB&E#F`IuqX`9))fHZ0VIfQSu5}%a0*l#wP&*eI3G+Z zR^@j#CZkZr!=z;VGr=KzS(mH`&JX!G{`GZB)Wzdu6IibMPakG|MJsH%kF2>TP>)r; zdq^g0KuX27D=z7((W2$4{m!EE-`Ewul+*&6;sm1?2!VQVM$H=c7%hrDAhx0~!44RR zK=u~Yit!0bF_kKfxQV(cE7;{nZG9x^e**9wUV(VS%foZ-{CVxHTDA6#f1&~dtvr|x zMBDWOeNqp}k%Ml){EJx%3HgwX6s*$%ZJuj_bL~4&Eh9*%pA3kqoHi_kr2+4_Gahhz zV}A37$|(6i`b#qv_723AJzUazu_!_ozxS>nTWNW)pa(#2b1Ke%)}kDgLp50NGA1UO z&2}(A!1NC8C*rlWZr=5NtzZXCbT)<7O5f+l2K+W9mD>XxCu~4R^w!YJfUR=_!3CaK zn|HsV=!!fM;7YxIA$Xg08Wcaf$Mr}J-CSSDf0;2@Hcm{E+|jXD0$iFlXZRPCI?;406<8+s@(DdnZR18K#4dkqP7 z=GU+5fTKxH1`kEd`LKZK3J4#5jLwxoCscgCqmc>mnhuIcV6z}5A&~(^eFa`G1>ypy z)rt9g%1h_Im5F>UQJtgw@x7&87rH^+Y0CmX?X(7+qRwP?NHs2w)T3g)vtBc;@D}U) zKbG+s^Wb(keWN+Eh$9P*3mSCUGA^S!iJ8?#P1e>E zk#As`l!7W9eXP&rcfNW^iD1}T6f6mqlKjESJU^uWKU0Gx@haQ#s%n8j5NezRiogKv zh#Lc)a0UcAQEiQx6yxOs-ZbP94^UW;X*ujnv&}oe7B3UO$+4e9i_PKjbrify-)|tH zo&C^+Exvu(_TpiTE|gNS5x3Y}dUdGyB1~BE87{P>RTZBbgHUS=JUTu{MQal@bVX1l zyE>TN^l1TI=W}*@CTX{_`Fo5yhLv6nRpN{XJD&I0qB8q zc?_T~UmbHV&vF$SVjNr-TOhqhfkJevuT2KdSU5-auPz0hYybwZD-trB;o+A{N+Y)f zCjsB0_eMVN)v5Mc;IVGCNO8()=}%P?YVGkUNG&kg0XlVnQ$er>zybRPDlJ$z zGK^;WpATn&;1&Ae)0Jc_Cw}Ar2pc{HE#@mNrVmz@qY8Nsy90%H>Py{J@P0NZ0e*Di z2)G5H0qftx4+)c#ca9GLI)s+J7gYY<-T`5k2JT`4pF@#zC_|zR8l$mKTKIR*g`2ejFQ2jzzD}VV8O>JbWw{9j;({k^iWSKfR1hmM%JBToT%cS z9X5AVf-vu|Kp0{z?0Nf@J4jT2LZ*3D$Dey?{nyEtfY)7S*TRI*fpKsEksZUmp!?$9w5F^6m5P2uPzWIlpJ2t!j9vRrUm%D2hqU^KEt4!RLpL&MtO2Gdh$xv19RfmY<& zh2)I+RmOIx@LC$uJ4v`V`F3P#$}9%7g&9vduJu81hDIx@tIWA#}6k0AQXjB zRfh+{!HP{TWSDQgevgw@8-18IEusH#aY`g%W$piJ?#sieZrgTOs6;)fM6CznDN#;iL5Xx8zNhCsMB_tskLS&!!@;={w_qV@eAK$T${m1rC z!&-jBJzV!SoacGPx;60%Jm+&;g0C`n9blxi@FJx}kT5mf<5R3$$osAC+8`ohz_$D- z#@%@mDR$>|;w^eVrV0^O92L@}y>2n95dl>$tQs~${+}*|f#P_md`v2A`c%w%cqv)7 z3E^Ge*QD=^=;J!VK_WNdCyMMyx_5ds<-*3+)rt3nPV z?P;8?@daY65)4J3D1LH6<^RUkbor^~r*mH!FGengBe0)Wqa2)NaP# zdH|`7HFN|(&*SI&){%S%A&R8QFYS(HOTKX$dv?^xL?0x}$4^!-Ud9|yH|Q^=qG~d8 z`NcRcLu43HL<2MFxuf}C@y()=`L-5wD`Ljl)@ck5?;iU4AmtQoEEPYI;<37VVG3u3 zrM0!FtZXYFInM4x+hX){EYY`!i3=p`sYD9_+TDV!3&`L=dk|F>HK!-Ej08O(XNfbD`|rkI{Na0&;4kJmb{d)aOx|p0 zT$m9Q!^P-I3dg+j zkDeDKHY`P^lc1sJfB!N7G)OSE=lfqf|^id+b`!r{sSeJ`P=tF3*#qvQK=CjYf0OaBL+ z+`O}n)(c*<)g_)PTPN+cqsc;j{#aJ*6442RaNRYzJ^pLOWLmyn30pDyuA>-JZ)c*v z?LB#1MJherNq6^rAT*KzWIBLcqHV*y;6-E#aBw#PJV0UAwUXtzTd=IRQD)u8lvQk# ztZ9?5ikuDP63^dcJhnq}i|`ZHu3c*xit4!zJT7ULjAOaV5QTidT464M2uPvGQek4$ zzwnFI zOh6=sxqwIKii_w35L!D=W@*GS8BOfBlW$1cfLtntmj~7SU4Td<R>;RC=o%azxj6pGMheCw0?M(x#Y&RU(;oZT6EeUouN52w9J?ZQbPokU%ma@ahzm% z=f|yf>n}~%_2x}=6JhC=UHv)5>Yb+V29=Zt#qzYba2AA{0WbfEs_N@?1sa-~03|oC zE1>VWOHh&q*7o|}BUod`HY3Xqt9K;%ZW+z(6Y_0{Jtl-b z-T7>{l+S^#xWLSg;~E}aO2pLp+np*X5_@9Ta@cQvGU}GNsb{r=)!W;N`1QvaJ1lv{ zHBo?=_#6myiqLZJ-z#v)`jyts|`v#y|&KJ+A~Wt-G8x%80s8A*^unF1(2EK{@06}Jc44HFw|&) zLC_&WRprdvg!b&|h!b2QWWL@}A!$vSz24{Q+?~C#$)aEL9P{*rpSSNrLY5M_hoJi5 zQKhW-d-KQNS(K2cuWndNe~xl6dfJJ={pOA?&s>!!Z3l09kjxlGFsc&-dlMmO>)i0| zP90+=2U2}bVVr{sZ6=7aQqOkIDee|g_`CKdTeY2jr6u3#BXKvtSWH`4khYmG`~_%u zTZxy)lB4tqLb4He9(xQ_e*L{_#m?S<(v5^vSD4C2>>}lRBF=?UpWmP-S%yCA_nA4! zA{32M*2&%_^8*FkI~@cBkbvU@kern`4wUbY>bG$?fYL>HB0$2wZjULVr>hegQ4KjB ziva8UZb!+D`Y+6!x{s-Z?WgA^C!hbg$P>kBC~lqx4Fo{2IKn0*ntO!VMqqw0`*n15 zhd%?j+GUob7oYwy`Fn?720JFYX?&P}LVJAsv!b?iK@mKFxVBB%rOClY6*#I52)-vk zT@;d&Ga_@2owp8oOQJiBg=%n;d!yo-taDXs@^}hNK)`I@ywW{HJoDfMV>=Kih$*Hl z7n$gOMMqChAe5)!Wn;_=pw!HVyn7h;?xSHFq;+vXO4)u80`k*j<^_e78TI3D*_k`P zMhJ7-Z+L6)*NBD&7zyTs(h?S?Na*ay| za&!Um?nXv;){7Y?5TzL`?U3;BD2O|PX_rXA@YqDR;^WN%xL5l>k@$nu%;kr;&x2#S<}4JoT59sQmg+;qOVr184V0H(HH!8P`p+iuG#H_DE71)-)1&Qh;TX zq&oyyQNYwev6p1bJ5e_h0c`&yx6%CUq#o3Be})q74v)`71j$e$O>ORTfT zv8-|WNb35zGzBqmt_A_5B=<-a3_XU_{DzHvp3C0|Tn|a9DARSOE8vYCm z;1J6HXPpl!j=3Nd?Hb7-u<}W#<6%43V*OR9CYtRi^!e>ha-^ixnc>NUSR5Rk{g08+ zk*C&~9^JtH`q;`I+pWQ}Ni9|mjRAjKJ5=rlXR)P7C3OdFQ-;LuUxjG%l(jWmKVL?x z-@pE+}8-%ZBlyWPG;CTj@oAVMLX@hTJ5 z-{|FHCf}7=LuP8W%Vb~p&U7)S)dtOeOa6r@B6>HA87stxh3nr~>`q&qSb})!kyCdm zNMXxCuxGx&U+=TTX|9q`u=-ivJ+A$sL_ZP1A~wbG+3&gUEVG^6>7UjU^bbmG=|zto zp0GUcSK_mUwrJFI)^ajR?%44mS!MM+gqUCrZ{X}Y`skriz~_D#zrBXU_;n-$WgHiH zp{?b^%dNCyUb_JzM7II|{ykEpMPP|2eow-6u6cQ(rqh31yh$$$$|W22k=0Lu^b$RE ziF_Y%NLs1N{j#cm80RU{JzN>ep?9xKjEOZLprdbLX8Cqq`7d|&LS#ic$?J(6Ev$e3 z7_vY_>}Y5qe)Z)U4heIH0G7jiyKxp7j5ZRXsM@#L*NUDQPN>${g2PMSSii&7N0L$~4 zxWMdL$9dmqzY&gP3XjKpk;c}}&LHAXQtGZLGDdD6KxG0RCpQ_%_?T2+LX4R15ucm_ z7QT9B(zmyK;HOQX z;$Fs!pdCo{KRW*VRO;Gm?NhAwW~s%>XEW19PI=Q}3`&R`9iR3gZaKH3PIX-w&x{?s zn2w4J-(S<2+$2#LNOPnVUH0)ZghXphvYvIz{`!7>(?o}Zbl*_5f&-q=griKFuHrSJ zhK+iAGIenoDLZu@Q_)$cF8gQ)K4(>7*q^|@>*lTdzxu1bwkr_#?wLlUrwmgf?F}*yKmwa4RDpbRrlrUzj*xf}+JCEJW0M^!7Yk!jF=OpXZ%!-y zvbn@S^Ry{Tp^wJ8m85Nz6Fdz2pC87%PqyqUk+$~2_us;$ojqqC|4Fnq`xdj!MzLBH>$JECv#z!8~29WDAUXO~w zv1iji>_poFl!Fuu$_c!(-IvUEY3mxkm@&ek@GP)#3!z1W6e4(|tw`+OWZOBS*N04C zGf~;w=(zC#a5=Os>%Zwz_u&A#9|q-Wq^6b6y|`*a03LIuKO9@z@O4H4^;9RV7er5h z77-(XvW78W=m$oS8%fQRk{$*xMLeI`Mbe5o@GP(+O!yba^&a3n2Erh4{T(&jTvg-V zKEXP1c6W=Ov!cf07tTSt!{8(&C{e}GAFIO3$qy4tmGSzpxVAzB@f;i~haZxOwX5sK z@QFtLYzP|3YZ5M`qfLRfJ)Xw<%a;v4c~)knsP4%yHzuIEx43pfWPOvDisua1bA-j^ zA29wKo1^Q9v?83*)V7<8BR~yMWSlEQ$SP&~e((Hr%No;qU2RAP0BF6n(~EiWCOIsO zpC({rPmS*jo0GWC`87rDv(lGpM2Kua)JM=)wZo(ITaaZ+r>RsH??bs8*n)q`-4y@cHHAp7T0VeKTDDl#Bip%uSorD8CF-XhR@NLk$0pS<#s%bzY?#~ ziW~_2ytH;~N-+g#0$L*9(l@$&NLxl5On+wF<~yX{zV`MNNxQTRTMFe}>L#HwxBD~TZo0}&ZZn@h`Pm+ zqx1kxt|I{Y5wTXbeRWqonU?+jmL5kNJPH$C0LwvjW%p})kLRm^oO-HJul{B2!FNy6 z6RV7Ai4x#+@bnyjX4y@rrMJc0wUz7=NlNSbzA-)P1Z~q$mO_5&%I<#v6XGcS)h%EY zGTav5fb}xYZvuJ;nC2TY^!0=}^y-z8+JNJcz@!&#dd|LSx((O~Q+Degyx`tM+XfWY zB>WaU_s7VU%NDLQDHp^!BAi|@nnv?hDbN2>x92UMY&O;hRloC0&DO%cTgK4bcmO9* zbTnwco1DA`ico{8(!YPA!qFzI`%7lU;sJ1C3uGQkCmJ*W@@50l86Y=L*C}`I5}cOd zC|ti8&O1`qT|+aMl)A^SmhETd{!P`Bo?9!wRAnkHxqor&xB?Y}_8*{4K!OdXUYlri zkwq4VG0sopNQ;pFwrCuM=ol{=@cp|m1@bpDb-R--9v{1tC3f7^gDaO~^^{vnPk!Pz z77n4Sbt1AeC^Z$6I;ZJ*{sfl)By$V|kclbtz#AO3OiG6{-iMZje|vdzh1`t@3vQ2W z)e_FgmK$4BUcbB1w+su6RdDXI@4A13k=Oa?MWT{Cf&6ZpHSIH~ zU%z+j?b`S+PmkR9;OSNCn_Kqq_2uM$_I(P8g5^B$Zb-t=B zIUk%5p{ti-D)Cs93VrUTS|kXH`TKBNXiK_@5sB?5Qy6yL6DL<*8m`Ew)({ z-7Pm-8ievaK=JEE1Q`>pA@U&KVGvDCCoEBFXuZ5>3?+G4E;za1b815r2b9Sy4j&?#^DMSP@lBQe0-;uxmY)( z^n@~AtT2S87c{9g6&&{@q;XkD7M!up*(?^)h$uoRC#j4elm@Xk_STaT3q5k$X>o4q zf)ou8!Tvt#j`FkV?COo`*eqIQ(*}5BFHw1Immf;;RaN98MFmR2BYu|kdW$&B+dK+d zG6Xob+;P?7@k*gP>2g*|&=e2gtohqetYM-mCdK`J!=)qZ@Vow>D=XeF?P=3Ai@!b~ zf5E^!^oi8Y!8e@nxG2sW7|PAR8f{xA?=&1|vD5?z6nsBa(QsvK_3P4~%1fW}kpH0u z<9}w(AMZ9+$n*f16Pc&K1G3m{b+`T~oU${5;x!RnplIJ+!+14uZ^6aaos(^C%1^~?cKHlXkUIHzMGYGZMatJ8`7So6%9GB^`yX^mNuv}IZ78TXBv=nG>gxdV`r6?8( zssXLy>MZDd^k1wP635E6Z}_s39Kn=b!i1ghfBJI-jZg4%I7IRy`LpL9s$2ifKZVu+ zQz-?4ASrYJvKWbmFy1~s`enX8B{Tgoa;Vh`{)z=?9eHdq1qB5HM?uNT2zkC!;lBXY zOU1tcz>Ne(4qr*jES*hBh+iM_?0=!qRh}w1#ht_pj26*dCAWbJFmiNFStk#aOn<&Q z5!2Yptkw z3$hfU*OOG*$jAs2BcnC~X+odsYZ((my|=e2+yLpizTshVg%qv$z37@ybt>Fd-i}T1 zCF;JDsBNQmd7MBEk@*o@U$gF4Zp+Xxlq5cs`T3CxqgfUy#gY`+#KeT1XPo;<|Fu16 ziY7@770?40x)CkD8lmpoJ38tl5RJY@t_1P-8`+F{)g4`3T@L{IdVu4SblQ4aSa=6v z{1>k@xEk;AuMJl#%whQ*2(5!Owh)MBJAN+ldc1H5xMBWniyF66 zE}=v`Lw|=b0rG_8NgE5@>@2bkNk6b-IJQcwp2g)|rJ|O;vAKI*eC{ciS4kJt(Ha?+ zFXvQy0BxxU7<$s%LYaCRxNkKrt+MwI%@W{?ui2~>XHRI0xyYHtj3Lu`2l^IhV}|}O z@wFZ0mzZ%^R#r>MPNelBE3*@b71BBF?k<;dt9UH4XtZ?UQ0oUVE$14V5PkgcNav;M z4vi@Btv?2?e6~h?8lA78Y9-Fec+ zMl>L}!^6U)fx#j3^&Kx^6^|w>dT6G@UmeORQzsWkrJ5786n`}uu01{x&mIy6gQ1sU zB8&e)uQDq~?EcL_R^c(wc<5UELb4-T7ZJ6z^j?(6k|QjU*gXmAwEe~*&t`G;TOQp$ zEvUbC5N)1*5R1^lWIQLJJ}$EWya5^NTPS^4bJw6!{^wIKaB*cL1E%aGjv-Ggz=5hi zX+p+LT3$t1R)+J$TgPK<)T!(r~fEHdGJGB7l3msw&Z5UsNDyg;`Rs-Om6GQNCzutR7Jq!)fQ92N%XTqht5oFfMZ zhX}B|M6S>}9ughuRDg}0V#crl+TWTHp#Y+~1n4L>0@H*n6t-Y}rQN?9l3FuVs*1no zryXFuhYBS@wa9dyy@Aay?J;6P*alkilaZSb-A=%8w%Pmr{Gvv>OHrZYDARYo7%~iC zg^NN*1-)wWubDZ{o9^-*E4<|T=y*vX*hDUcB6*d%}nwOQq?j|iAA<8-1Quc?y zUk7nreS(@mjHdrs_uy5wZTWu!raYS)Co@cUZSyMaT3;0(!LUElrfhyHrgZuaMEI2sZ+}-rOYW4s?hj~CPB6>9S`!n;901CF6 z79J577pFl74Kf`&|BL4|M1xIIdI^=-04yv3Nuyt>+JR)E$)QZ}3`8#Bec$sHv7-x7 zv9ald&_x!;n_F8+S3UwAfppqY)ZLZCHxBox>ogDq0`~q-`Ued|A~kKee*GdWoF9=L!N#hT0tQnZyz(9d1~IY);CRIARoLnbiC7o z*`9^(0?4sbsIu~A-%z#xp<%wA46;_l$$%M!ko#P<_^V9?QIZmibM^Ph$-OqXN9l6so4eqM{mv zrZjVDwiqfPs#I7tzW?IUh0ujc2Pga!9E}7pNTid*)@#HKZ{p`iCFn7lVnKJcKY9U` z13@)>BbP(LuXS){h8aSyr%~%hCqi8%C8d3PxLG9$bQle49rj`>x2UOWX%$=pzCpvB zJLyDhS^H-c9>+_#NXIMcpP3;yMMeKfwkRxszSg}!*B~hX79IA<{j}aFV?VZQX2p)5 zxat;iy#BBaUXA5ZScq#GA0oE|p6gAnkMGc+XN&mVD0wrdtG(}3>G;%<`&bF^-~qq6 zu`2|fh-zbXQoiPRqC^35vQhfXE3$tk#ha+uv#a4W-l9@C}< zxUmjOziC5Ub@u_Orq<={tA2-rOmN<{vJfJ3z+~M*f522(*Zwc9e>|H%-Jc2|=4^Pz z$Ld{N>_E6Haz5{IWa-2HNjr7TEsN7x;(CN$DS;Ptx)`AI8&!9}5=(&-XV75{cB3+t zIBRH#);tF9ixcNG@t=rPT<$x#p@%S>p#tMjv^rK5&LwLOr$Fb+hOqK6@s`_N>@Xe8 zu!~^=dT_8@2?}x=DFxE$G{O?zlp}X|aIi#1Mh0njhCUdTavRy$tf9LR3X#25;E8J` zc1TKU0>>6AAnDk#YiJsV`xh(Futsyr=m+rTK;V^^& zm82fRiXCjrOC_%BB-{NZ^vk}q`T}&WOeKyis>2dce%TIh9Nr)*$m8;r;G8;k68Z|w zH-#*+GW8kCahj;dpH$FfJqf*-?9$CDDk?EJ{bOLfIRVPzuigX7;m-DfFJBB79vNdL zxWZA50U?qdt~!p~TAcFZ@^!@3;VSzxn|c~ps7Z9CH1yjXyFf?{4pZ5PFJ=@D-CdyX zBsg+}AC3aiux8N+F9QFZ>0rTjxd?J)=j>GL^N#=tZW!tAbKWZaI1Z~Zu*cLHZDe$V>K=kK<3i4yP%?gC3QPFl~k+<@Qn44KJo zMLT_G`q+qe7wI32ZhD6hIMsqAyv^$_#9HnEbx^~wbScNSZQK6l=5|1&WQ}q$)2whJ z`z%NTns5cut%uNU&jK7yyHc>Flzvbl9!&|CKL67a{~(Py zE19n}K8?#;MiJTtQFNh10>6M{$k#BLu6msHM9`X8Kg72UP_ny^YZ3GVT5P(VeJo=M zoFG8@o?jEMaG9!S8{93sHMb9)DZ|mV4mr|d9DXFuhGlI4+~P4BgMe|>DOUzg5l$1I zS#+Beg@*%t%hcs?(t0`t))jc`(oa=LAWgmCz!RMW1QAVbU=t=Gq+~b!MAhHL7#fi$U;^y_)wz1})F#{eJ9n(rF17A|W`DTx@u@y&0$gwT4JxJ>Z(sc-We=!NDc^j4-Tl z{G*5yLue)#RDi_m1>=k4O{XW@0saBwA{o32i1DctX-7~X1=U(xTRkyMtZUZrVSzV2 z(m4RE$a_H^860&a%z@Y2*VlI+N}_~Tg=|M&t-HbRyt`2AB)CSb2o2066`P3wzlkgi z-1nL0>nP|;SX2s$h^XVF0%7AlOdNW~4r(;}Y^}nagn{I-P0bUEH}ahcFdn*1+X8N7}{xO?$pF!;2 z_u2fnM>%e4^Z8Y9^DiFw#YYL46%^|vP^TV&`Dl=~1vod-AQTSr7wqxd@U-JxV+s4)bFWNeRNQ|6P10I9!;h@>zC5 zj}#y?gw{^l2u=um9T+m9ty?t-!%YTeAi4`Vf(WfabS+Q&-3xmz)t*-;u@VR`Yd(5b;dE>*y zRaD8tpQJR`%Tb#iy}ZiuV%G5R zgk$Hu=GWa^=bs5RHT2_`VY-c(Ep3`rfRQl9uL^l>8g*yDNXFDp~4Z3<`R1 z6FZiPB75#f&uxe0%a9ItC(f5UVB%bLNFk)t0>OTCXF zlXGg$#MoGrBCr+5g}md(tbLT%SagTJ;YivN8yjnt$(ac(*b?61;o%$KsAlIwkB{XGlTz!mKNh;r%g$NA>pBG*Jw%t@CJE-XV1b~ zo12^08>O$O)VH_W=`NtP?+I-6%6IPW?jLilS4(YJb&W^i8Y0DOZ434FEby)&S?=G^ zp!>9_h;!(Jt@Pfx9|!RFLeyKL;}b`f0?dx^l+}1WALOBXAl>NxVtVnfuj`imdhzf?23My z8tG)Fd@S|ZJyN#dYw_&x!U>IWeVTE zecQfgPw2&q4ClPOv}QMA`iX)KFV-%mX(zaC%Z%HD7sfsQ%$+ZW+qJoG)v8q#e{@6J zB_}8H4gHR@vGG=f5$|KI5#nu1$6sIIS!`~Zo}TVO?nA}EAWGt_zP|nv3MyuAujL|W zq$p}=)LiUePH}K>c=YJebCh+R$NM!AdnhV}x-GkK;X)<-)s^87{&d2PPi(Sv!S+%M zlU$=P_k%tqB?Y4!0HpXnrTUf@na6vBgZ|X8GX@7e;DF;t@f!{ev$C;yJst1j`tX7g zljY_5_pPn6whNcJQ@?$)?Y$Nyh*0LouU|4Fa^5(CaI%&5$=+}#}I>zus`j51qZ*4D1t5LRjL<>cgaLvh!8m&yZ~ zE#yYXxUX+N^h675-9O-Zkj))5Fg|Eu!4KBb1(>PuHY|_a`k_hZGiS<+&yPRw7QbAj zIPflB!D_n%L+j(6)^%LaNl3`fUW3G;ET&XjPwxU=7UvH=;>lM)m2S6Ez^qukRP*$d z3keCS@9N4pC-5`AaQsC4SaWA}))Z0G%|hia5K)xX)K+Xn8tzM9AK#umT-cV_hSP91dPYYT1YNgm zn$0}PE3oN~72JqRaFRbm_w6G*lt){_(9`-M#sA|+!`zd}$;tGfhFH}sdH(XH#fN9? zhxu0FIex9U2Ekd1KVIv+$#B(y`?@FLgH_&;L>e;U=FLk~>RPz1s}KaWMuxc+JRL3l zUEkIg0?!!h*j#;9OjMMf^1Z!aHBN<)LP+0M+$+HjfeG;R^sMjhcKR8kNFm<-7Wq1* z{^Pp3fe8r-!h^I;mAFGmsqv7B2^Z7=&}UsRx9q&6je|o4DA)`))h`k9q&=tWzCoO& zk5Wr_NekK|iBvf?&K7Xsu{@ne8R%j-w(gV>IFGwU#~TMeEW}COOVvs0;ezZ z;e!olJv@5h(&{`}?zr5<70sB#r>5L*{0s?U!hTadEVHe2ded1sZdh8@hfg=4`(R>D zj_#VEDkBXPqbQ2|_62+%dxLPtXph7M^1XDFFa7-@?({IZ+32Dt%xPWh+KsGk(i|2C za#uUOR;`K9pDru@9vBq#$E~TUDXGbJsXxjvZ!p?|;KL&O8#e0e1>COXxpN=C7`^mj z1QqTKG4X)l@+`MeG9sWYExoQxK|aA8hlAp~Dw4g#}8d2u^|!(7n`%2tN3p zn>JPcUWK|EgkyBGmm%{OdHM2Xwe;~LWmoDOye3Zcn_lp!OU%zds?*icA%-*no#GyO zc|lwp&2B`&@Mf{LwqC-EJWz<8vT;pw^HCRP=j|ERu;pS9T0A0h6D%v^yS+$0ILsMi zO&P(6zG`hf1~&uG)?hjRr+U}?{Cra5AcH@&!mAMh3W6K8fs^x~(!J8sBR>d|r#nd}@M6jOLXG$`ZW2%M4 z#3B)7s9M$C6-IuOdtAyI2Ly=G%Q#CV- zPnx%}6W(-Yq(k;R)G3r6HU=vvM}Yk=EF{G4b`1y2Pvn(@Q4NI`38{nY$e>ljwau8$ znD12J7yus$&KExp04xp$e}IBE3?W~0sv=|^R^lm`;~Z$x%0gcENkM@m%i4%0-QXXp zTQ_aGQw)FkBmyXeYPr)wyl9--X*$ihAD^s6mT<{-x6#!YnuH!2~L9f+a z{^M^FEcggN$Ap*RX5>e-LQxeH56|E}IUUJQ8Y?{??OPEy&0&+tPmb=&i?nYk9Sy;H iAV06w|3A6d0=>0Gv&O67c>=GXs2x14l(GLr;Qs(z+PK#M literal 0 HcmV?d00001 diff --git a/evidence/reconcile_faults.json b/evidence/reconcile_faults.json new file mode 100644 index 0000000..c41b140 --- /dev/null +++ b/evidence/reconcile_faults.json @@ -0,0 +1,221 @@ +{ + "s3_failure_compensation": { + "dry_run": { + "mode": "dry-run", + "grace_seconds": 20, + "chat_scope": "4849767c-8c80-4469-ba2f-7b439782abc4", + "elapsed_seconds": 0.035159, + "counters": { + "objects_scanned": 0, + "metadata_scanned": 1, + "list_requests": 1, + "head_requests": 1, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [ + { + "kind": "delete_stale_upload", + "attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c", + "object_key": "originals/4849767c-8c80-4469-ba2f-7b439782abc4/d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c", + "applied": false, + "reason": "lease_expired" + } + ], + "process_peak_rss_bytes": 70615040 + }, + "apply": { + "mode": "apply", + "grace_seconds": 20, + "chat_scope": "4849767c-8c80-4469-ba2f-7b439782abc4", + "elapsed_seconds": 0.02742, + "counters": { + "objects_scanned": 0, + "metadata_scanned": 1, + "list_requests": 1, + "head_requests": 1, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 0, + "protected_active": 0, + "protected_used": 0 + }, + "actions": [ + { + "kind": "delete_stale_upload", + "attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c", + "object_key": "originals/4849767c-8c80-4469-ba2f-7b439782abc4/d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c", + "applied": true, + "reason": "lease_expired" + } + ], + "process_peak_rss_bytes": 70946816 + }, + "metadata_removed": true + }, + "after_put_db_failure": { + "http_status": 503, + "metadata": "uploading", + "object": "present", + "immediate_cleanup": { + "mode": "apply", + "grace_seconds": 20, + "chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44", + "elapsed_seconds": 0.027833, + "counters": { + "objects_scanned": 2, + "metadata_scanned": 2, + "list_requests": 1, + "head_requests": 1, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 1, + "protected_active": 0, + "protected_used": 1 + }, + "actions": [], + "process_peak_rss_bytes": 71266304 + } + }, + "status": "PASS", + "dry_run": { + "mode": "dry-run", + "grace_seconds": 20, + "chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44", + "elapsed_seconds": 0.047514, + "counters": { + "objects_scanned": 7, + "metadata_scanned": 6, + "list_requests": 1, + "head_requests": 5, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 2, + "protected_active": 3, + "protected_used": 1 + }, + "actions": [ + { + "kind": "delete_orphan", + "attachment_id": "4baae95b-26db-42df-9dfa-263e7b8c10a1", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/4baae95b-26db-42df-9dfa-263e7b8c10a1", + "applied": false + }, + { + "kind": "delete_stale_upload", + "attachment_id": "633b27f4-a695-4381-af03-8d6f53d0b055", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/633b27f4-a695-4381-af03-8d6f53d0b055", + "applied": false, + "reason": "lease_expired" + }, + { + "kind": "mark_missing", + "attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "applied": false, + "reason": "object_missing" + } + ], + "process_peak_rss_bytes": 71266304 + }, + "apply": { + "mode": "apply", + "grace_seconds": 20, + "chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44", + "elapsed_seconds": 0.053742, + "counters": { + "objects_scanned": 7, + "metadata_scanned": 6, + "list_requests": 1, + "head_requests": 5, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 2, + "protected_active": 3, + "protected_used": 1 + }, + "actions": [ + { + "kind": "delete_orphan", + "attachment_id": "4baae95b-26db-42df-9dfa-263e7b8c10a1", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/4baae95b-26db-42df-9dfa-263e7b8c10a1", + "applied": true + }, + { + "kind": "delete_stale_upload", + "attachment_id": "633b27f4-a695-4381-af03-8d6f53d0b055", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/633b27f4-a695-4381-af03-8d6f53d0b055", + "applied": true, + "reason": "lease_expired" + }, + { + "kind": "mark_missing", + "attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "applied": true, + "reason": "object_missing" + } + ], + "process_peak_rss_bytes": 71536640 + }, + "restore": { + "mode": "apply", + "grace_seconds": 20, + "chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44", + "elapsed_seconds": 0.072341, + "counters": { + "objects_scanned": 6, + "metadata_scanned": 5, + "list_requests": 1, + "head_requests": 3, + "get_requests": 1, + "bytes_read": 111817, + "protected_fresh": 2, + "protected_active": 3, + "protected_used": 1 + }, + "actions": [ + { + "kind": "restore_ready", + "attachment_id": "eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "object_key": "originals/378cc960-c6a7-467c-aa8e-d11e9032df44/eea304f9-f2be-4eae-b46f-e3d3fcf8388e", + "applied": true, + "reason": "verified_original_restored" + } + ], + "process_peak_rss_bytes": 71536640 + }, + "repeat": { + "mode": "apply", + "grace_seconds": 20, + "chat_scope": "378cc960-c6a7-467c-aa8e-d11e9032df44", + "elapsed_seconds": 0.037725, + "counters": { + "objects_scanned": 6, + "metadata_scanned": 5, + "list_requests": 1, + "head_requests": 3, + "get_requests": 0, + "bytes_read": 0, + "protected_fresh": 2, + "protected_active": 3, + "protected_used": 2 + }, + "actions": [], + "process_peak_rss_bytes": 71536640 + }, + "used_and_fresh_preserved": true, + "active_lease_and_lock_preserved": true, + "missing_original_restored_by_operator": true, + "s3_write_failure": { + "status": "PASS", + "http_status": 503, + "metadata": "uploading", + "object": "absent", + "chat_id": "4849767c-8c80-4469-ba2f-7b439782abc4", + "attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c" + } +} diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 0000000..b394ac0 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,7828 @@ +{ + "duration_requested": 35, + "elapsed_seconds": 35.01, + "concurrency": 4, + "profile": "closed-loop: 60% POST text128, 20% first50 history, 20% profile", + "retry_writes": true, + "successful_requests": 7020, + "failed_attempts": 176, + "uncertain_writes_resolved_after_window": 0, + "successful_rps": 200.515, + "error_ratio": 0.024458032240133407, + "successful_latency_ms": { + "p50": 13.72, + "p95": 71.279, + "p99": 91.743 + }, + "instances": { + "48fedaf342e1:eba13b85": 1013, + "9425d69912fc:01a7b3b9": 697, + "b89ca7b5c7e8:e619829b": 2806, + "26996fc7b6ac:0934d46f": 2504 + }, + "errors": { + "POST /api/v1/chats/ba7bd3e7-4bb3-4868-98b9-8acafaa0950f/messages: expected [201], received 503": 103, + "GET /api/v1/users/me: expected [200], received 503": 40, + "GET /api/v1/chats/ba7bd3e7-4bb3-4868-98b9-8acafaa0950f/messages: expected [200], received 503": 33 + }, + "operations": { + "write": 4356, + "history": 1401, + "profile": 1439 + }, + "successful_per_second": { + "0": 187, + "1": 208, + "2": 228, + "3": 112, + "4": 205, + "5": 155, + "6": 142, + "7": 209, + "8": 135, + "9": 213, + "10": 136, + "11": 195, + "12": 49, + "13": 184, + "14": 214, + "15": 186, + "16": 165, + "17": 105, + "18": 259, + "19": 312, + "20": 192, + "21": 306, + "22": 237, + "23": 198, + "24": 198, + "25": 206, + "26": 325, + "27": 240, + "28": 250, + "29": 261, + "30": 142, + "31": 141, + "32": 225, + "33": 216, + "34": 280, + "35": 4 + }, + "failure_seconds": [ + 5.994, + 6.004, + 6.017, + 6.035, + 6.106, + 6.126, + 8.068, + 8.112, + 8.128, + 8.138, + 8.179, + 8.199, + 8.214, + 8.227, + 8.273, + 8.32, + 8.371, + 8.441, + 8.498, + 8.536, + 8.553, + 8.575, + 9.094, + 9.953, + 9.966, + 10.059, + 10.108, + 10.127, + 10.153, + 10.199, + 10.281, + 10.387, + 11.893, + 11.907, + 11.922, + 11.968, + 11.979, + 11.981, + 12.074, + 12.183, + 12.763, + 13.28, + 5.981, + 6.093, + 6.112, + 6.139, + 6.173, + 6.183, + 7.986, + 8.0, + 8.08, + 8.117, + 8.191, + 8.235, + 8.254, + 8.282, + 8.297, + 8.308, + 8.325, + 8.352, + 8.387, + 8.4, + 8.422, + 8.45, + 8.452, + 8.486, + 8.503, + 8.519, + 8.564, + 8.586, + 9.133, + 9.996, + 10.013, + 10.036, + 10.072, + 10.136, + 10.166, + 10.189, + 10.222, + 10.294, + 10.375, + 10.391, + 11.913, + 11.929, + 11.962, + 12.011, + 12.115, + 12.194, + 12.784, + 13.332, + 5.979, + 6.012, + 6.041, + 6.051, + 6.061, + 6.067, + 6.082, + 6.097, + 6.154, + 7.978, + 8.099, + 8.164, + 8.172, + 8.293, + 8.335, + 8.361, + 8.42, + 8.432, + 8.471, + 8.481, + 8.516, + 8.526, + 8.545, + 8.56, + 9.114, + 10.007, + 10.024, + 10.09, + 10.21, + 10.37, + 11.88, + 11.946, + 11.988, + 12.004, + 12.076, + 12.092, + 12.173, + 12.176, + 12.21, + 12.221, + 12.803, + 13.309, + 5.991, + 6.027, + 6.077, + 6.121, + 6.172, + 6.2, + 6.21, + 8.053, + 8.093, + 8.154, + 8.209, + 8.218, + 8.24, + 8.257, + 8.305, + 8.342, + 8.375, + 8.402, + 8.466, + 9.101, + 9.971, + 9.982, + 9.993, + 10.031, + 10.044, + 10.06, + 10.074, + 10.084, + 10.098, + 10.118, + 10.175, + 10.194, + 10.278, + 10.307, + 11.885, + 11.9, + 11.933, + 11.951, + 12.002, + 12.026, + 12.087, + 12.101, + 12.773, + 13.293 + ], + "latency_samples_ms": [ + 13.376, + 13.725, + 20.805, + 15.826, + 6.771, + 25.612, + 77.58, + 13.401, + 81.451, + 5.04, + 81.663, + 16.027, + 14.779, + 13.554, + 5.572, + 7.034, + 56.817, + 11.808, + 13.343, + 5.716, + 7.123, + 7.107, + 71.911, + 71.576, + 5.699, + 13.226, + 12.995, + 18.205, + 21.085, + 10.149, + 31.871, + 12.804, + 13.539, + 13.242, + 6.085, + 14.41, + 14.305, + 14.084, + 14.076, + 8.937, + 12.045, + 20.32, + 20.078, + 18.636, + 8.68, + 15.73, + 16.023, + 13.038, + 13.599, + 6.783, + 12.792, + 27.014, + 16.761, + 76.887, + 6.92, + 19.251, + 89.102, + 14.278, + 10.191, + 4.549, + 67.387, + 14.407, + 81.135, + 15.816, + 6.394, + 86.852, + 16.304, + 13.074, + 15.633, + 5.649, + 10.787, + 11.318, + 14.017, + 12.411, + 6.049, + 18.941, + 77.176, + 15.853, + 12.742, + 6.0, + 19.7, + 13.145, + 13.441, + 17.64, + 6.093, + 18.45, + 11.665, + 11.256, + 11.333, + 6.1, + 19.542, + 13.03, + 12.433, + 13.702, + 7.602, + 11.441, + 17.878, + 11.736, + 13.299, + 8.318, + 16.261, + 20.938, + 22.181, + 14.46, + 6.047, + 14.312, + 14.159, + 13.026, + 14.897, + 5.897, + 16.603, + 17.617, + 25.99, + 19.494, + 11.872, + 14.373, + 12.546, + 14.485, + 18.206, + 8.471, + 23.132, + 12.784, + 13.309, + 15.599, + 9.705, + 13.44, + 11.191, + 11.6, + 15.58, + 5.288, + 11.404, + 12.945, + 12.326, + 13.869, + 7.507, + 10.293, + 27.228, + 19.724, + 18.655, + 17.079, + 19.212, + 16.226, + 78.579, + 15.618, + 71.222, + 12.172, + 12.164, + 91.415, + 19.229, + 9.466, + 69.315, + 10.708, + 13.365, + 13.191, + 4.601, + 62.362, + 103.07, + 91.105, + 15.524, + 10.125, + 77.237, + 10.691, + 75.094, + 14.255, + 10.873, + 65.961, + 21.29, + 16.06, + 13.63, + 6.693, + 103.695, + 8.135, + 51.976, + 15.405, + 5.485, + 80.07, + 105.169, + 13.478, + 12.698, + 4.685, + 65.381, + 19.792, + 73.667, + 14.07, + 7.878, + 12.842, + 8.492, + 61.309, + 16.082, + 7.583, + 16.198, + 17.042, + 24.992, + 16.945, + 6.087, + 8.59, + 10.894, + 17.137, + 16.346, + 14.209, + 15.056, + 15.691, + 17.605, + 18.436, + 7.589, + 15.479, + 14.903, + 17.044, + 15.708, + 10.531, + 23.704, + 14.808, + 25.274, + 20.663, + 8.581, + 14.291, + 28.803, + 9.543, + 21.281, + 6.784, + 15.205, + 15.532, + 18.638, + 14.391, + 8.478, + 21.912, + 19.079, + 14.61, + 18.579, + 8.661, + 13.079, + 23.712, + 16.651, + 14.093, + 9.041, + 19.337, + 18.738, + 15.962, + 13.007, + 6.382, + 23.922, + 16.559, + 12.245, + 14.503, + 7.217, + 14.21, + 13.139, + 16.026, + 16.767, + 11.716, + 13.183, + 10.305, + 35.497, + 18.981, + 8.098, + 157.209, + 23.065, + 16.602, + 82.456, + 9.596, + 11.036, + 9.798, + 75.32, + 98.338, + 5.897, + 83.474, + 14.19, + 8.87, + 11.826, + 15.939, + 6.147, + 30.375, + 15.463, + 17.236, + 6.531, + 11.1, + 43.561, + 20.893, + 8.61, + 79.016, + 92.791, + 19.224, + 21.354, + 6.348, + 65.406, + 22.711, + 14.43, + 13.213, + 35.183, + 14.422, + 116.923, + 74.117, + 15.879, + 5.95, + 12.194, + 11.664, + 9.374, + 13.298, + 13.536, + 10.672, + 9.937, + 28.095, + 48.226, + 10.759, + 9.663, + 9.737, + 12.629, + 13.757, + 8.228, + 13.948, + 11.706, + 15.936, + 15.502, + 8.463, + 8.626, + 10.702, + 14.546, + 13.592, + 7.68, + 13.409, + 11.325, + 10.496, + 12.677, + 8.202, + 10.668, + 19.275, + 18.07, + 19.734, + 7.838, + 9.956, + 9.851, + 12.489, + 11.813, + 6.088, + 10.398, + 14.963, + 11.02, + 12.497, + 6.811, + 10.7, + 16.85, + 14.448, + 15.09, + 8.175, + 15.374, + 19.347, + 21.313, + 52.899, + 22.945, + 15.108, + 11.65, + 9.23, + 10.471, + 45.745, + 101.546, + 109.911, + 13.117, + 82.155, + 7.175, + 86.126, + 14.8, + 19.287, + 7.453, + 14.143, + 8.528, + 9.146, + 30.349, + 17.265, + 14.072, + 11.083, + 9.337, + 16.769, + 17.623, + 21.231, + 7.843, + 15.288, + 17.484, + 17.748, + 13.177, + 21.324, + 18.377, + 18.441, + 10.374, + 18.605, + 16.286, + 19.476, + 16.059, + 19.373, + 13.764, + 20.104, + 16.265, + 6.041, + 10.781, + 17.34, + 14.973, + 16.444, + 14.056, + 34.25, + 17.924, + 17.966, + 17.384, + 19.614, + 11.211, + 15.794, + 21.07, + 15.137, + 10.62, + 19.446, + 20.166, + 17.354, + 18.727, + 9.773, + 15.392, + 16.339, + 16.033, + 15.858, + 11.003, + 15.781, + 16.567, + 14.208, + 23.216, + 11.834, + 17.314, + 15.997, + 16.048, + 23.59, + 7.882, + 24.226, + 34.399, + 34.567, + 21.973, + 6.05, + 21.936, + 14.252, + 8.371, + 9.266, + 4.646, + 21.452, + 13.265, + 11.71, + 16.908, + 9.923, + 20.081, + 11.19, + 5.888, + 26.328, + 17.026, + 14.436, + 17.487, + 7.951, + 11.906, + 19.584, + 15.559, + 15.277, + 12.594, + 10.677, + 26.433, + 18.513, + 80.02, + 89.115, + 14.464, + 15.669, + 59.265, + 29.515, + 11.424, + 149.539, + 27.136, + 12.702, + 17.771, + 85.677, + 13.367, + 12.599, + 6.119, + 59.576, + 25.996, + 18.406, + 32.81, + 18.121, + 22.048, + 15.757, + 16.962, + 20.477, + 7.79, + 22.304, + 61.381, + 10.89, + 25.442, + 9.662, + 190.588, + 15.877, + 70.15, + 18.529, + 8.538, + 72.419, + 20.681, + 24.295, + 12.488, + 6.646, + 16.634, + 10.765, + 15.882, + 12.931, + 7.698, + 18.952, + 19.836, + 23.185, + 15.295, + 6.07, + 11.462, + 11.249, + 14.888, + 13.429, + 5.167, + 9.72, + 11.581, + 9.69, + 11.818, + 8.501, + 12.022, + 12.309, + 15.831, + 14.778, + 13.821, + 6.263, + 19.237, + 71.515, + 12.398, + 80.323, + 15.331, + 67.115, + 10.056, + 14.075, + 5.311, + 12.825, + 12.845, + 17.901, + 21.666, + 11.342, + 16.132, + 18.288, + 17.419, + 71.17, + 8.712, + 13.878, + 9.924, + 15.476, + 15.907, + 5.789, + 10.114, + 18.694, + 15.241, + 14.991, + 7.288, + 8.872, + 14.92, + 36.198, + 13.599, + 7.763, + 13.296, + 11.788, + 12.886, + 16.435, + 8.104, + 12.433, + 13.837, + 15.593, + 15.345, + 10.987, + 16.697, + 16.626, + 9.59, + 13.111, + 7.938, + 12.534, + 13.179, + 10.693, + 11.765, + 7.098, + 12.628, + 13.572, + 18.988, + 11.031, + 6.294, + 12.303, + 13.755, + 13.413, + 10.664, + 6.796, + 14.153, + 12.378, + 16.571, + 14.128, + 6.035, + 12.602, + 12.09, + 16.954, + 15.345, + 9.49, + 20.683, + 13.323, + 12.338, + 14.912, + 6.122, + 16.726, + 13.274, + 14.668, + 13.87, + 10.195, + 14.269, + 17.691, + 13.39, + 12.895, + 9.706, + 15.642, + 16.518, + 38.691, + 18.016, + 5.925, + 71.631, + 12.12, + 69.536, + 12.305, + 4.925, + 89.057, + 11.547, + 16.333, + 12.74, + 4.544, + 70.969, + 76.832, + 11.115, + 27.057, + 6.26, + 7.993, + 69.088, + 14.997, + 13.543, + 10.413, + 15.506, + 11.521, + 13.197, + 10.421, + 7.21, + 15.066, + 17.119, + 11.184, + 9.884, + 35.866, + 25.038, + 86.47, + 76.949, + 12.902, + 11.633, + 19.546, + 74.273, + 11.916, + 17.252, + 7.073, + 8.319, + 8.641, + 59.71, + 11.474, + 4.847, + 16.861, + 14.37, + 42.523, + 13.103, + 9.04, + 15.024, + 15.806, + 13.859, + 16.284, + 7.882, + 15.527, + 12.195, + 14.935, + 10.92, + 7.009, + 62.521, + 18.465, + 15.071, + 16.189, + 7.364, + 13.718, + 15.361, + 16.083, + 12.142, + 8.381, + 15.308, + 46.129, + 14.161, + 17.58, + 5.081, + 7.585, + 58.942, + 13.079, + 14.363, + 5.594, + 7.27, + 60.702, + 14.463, + 80.64, + 9.266, + 10.949, + 70.426, + 10.834, + 13.91, + 6.55, + 68.243, + 15.055, + 11.315, + 13.124, + 7.973, + 14.571, + 7.888, + 43.525, + 17.724, + 6.251, + 8.966, + 73.455, + 17.406, + 13.761, + 5.188, + 65.038, + 80.965, + 13.963, + 13.433, + 71.973, + 98.578, + 110.007, + 93.666, + 19.969, + 5.681, + 16.128, + 10.043, + 8.324, + 49.135, + 8.482, + 76.523, + 16.782, + 16.435, + 79.164, + 10.737, + 89.12, + 30.591, + 54.477, + 12.803, + 8.064, + 15.114, + 78.277, + 18.012, + 17.456, + 5.54, + 8.64, + 58.977, + 13.555, + 15.816, + 9.305, + 48.347, + 16.42, + 12.359, + 17.782, + 10.393, + 27.531, + 19.59, + 14.485, + 16.686, + 7.718, + 22.568, + 12.161, + 8.457, + 10.415, + 5.443, + 17.854, + 14.641, + 9.683, + 16.314, + 5.904, + 10.458, + 10.852, + 20.447, + 13.8, + 6.545, + 8.46, + 16.68, + 15.962, + 14.707, + 5.888, + 14.397, + 16.25, + 16.247, + 14.179, + 9.344, + 13.289, + 13.769, + 12.461, + 13.809, + 5.574, + 19.676, + 13.244, + 13.824, + 20.08, + 5.12, + 15.022, + 12.417, + 13.392, + 14.413, + 8.602, + 10.082, + 18.176, + 15.08, + 15.157, + 6.392, + 17.452, + 14.923, + 20.788, + 16.777, + 7.165, + 13.894, + 13.153, + 13.526, + 12.359, + 11.169, + 13.779, + 12.789, + 10.362, + 17.222, + 8.596, + 12.477, + 13.93, + 19.022, + 12.928, + 6.305, + 12.801, + 15.204, + 19.83, + 12.21, + 6.258, + 11.306, + 11.542, + 11.632, + 12.792, + 7.097, + 12.355, + 11.9, + 13.797, + 14.567, + 7.267, + 17.744, + 14.898, + 11.911, + 13.061, + 5.764, + 14.931, + 12.497, + 12.563, + 11.956, + 6.523, + 24.833, + 11.829, + 9.706, + 11.707, + 4.489, + 10.286, + 11.878, + 13.355, + 15.79, + 6.704, + 16.134, + 12.625, + 9.554, + 14.645, + 6.317, + 8.798, + 12.162, + 13.733, + 13.502, + 7.762, + 13.148, + 12.619, + 13.168, + 11.647, + 7.076, + 16.488, + 14.544, + 12.715, + 17.574, + 5.535, + 47.999, + 12.227, + 14.519, + 11.019, + 3.936, + 65.747, + 11.868, + 10.894, + 7.991, + 54.959, + 8.733, + 11.517, + 14.546, + 74.38, + 8.613, + 16.767, + 83.776, + 21.566, + 57.086, + 10.62, + 13.847, + 21.231, + 14.978, + 53.787, + 11.926, + 16.174, + 78.42, + 18.867, + 12.748, + 9.135, + 11.246, + 8.577, + 7.164, + 22.09, + 5.942, + 11.517, + 9.705, + 13.443, + 12.279, + 5.195, + 15.889, + 12.689, + 12.878, + 13.222, + 6.157, + 14.237, + 12.704, + 11.307, + 11.827, + 5.603, + 14.826, + 12.346, + 12.122, + 10.459, + 8.726, + 8.351, + 15.287, + 17.352, + 14.4, + 6.084, + 13.586, + 11.074, + 13.835, + 12.348, + 15.56, + 16.001, + 13.966, + 11.734, + 11.703, + 5.693, + 12.78, + 9.726, + 13.135, + 14.799, + 6.898, + 14.894, + 12.444, + 13.075, + 13.99, + 8.549, + 15.394, + 26.406, + 10.232, + 15.989, + 6.788, + 14.614, + 14.223, + 19.182, + 13.138, + 5.133, + 15.033, + 17.688, + 13.676, + 15.18, + 6.848, + 19.29, + 60.786, + 19.445, + 11.573, + 7.227, + 27.801, + 14.969, + 11.726, + 12.466, + 6.107, + 16.422, + 24.037, + 17.634, + 13.821, + 6.572, + 13.738, + 10.696, + 10.752, + 11.933, + 7.818, + 13.103, + 10.826, + 10.136, + 12.586, + 5.37, + 8.774, + 15.152, + 10.107, + 15.792, + 9.916, + 13.627, + 11.747, + 11.079, + 12.5, + 7.014, + 13.476, + 14.878, + 13.524, + 9.876, + 9.528, + 11.89, + 11.42, + 11.113, + 11.382, + 6.616, + 10.393, + 10.113, + 9.366, + 12.377, + 4.744, + 16.706, + 12.794, + 12.418, + 12.46, + 6.813, + 14.948, + 11.939, + 15.746, + 14.068, + 6.983, + 10.153, + 12.869, + 13.132, + 11.964, + 6.393, + 11.535, + 12.077, + 10.624, + 12.058, + 8.298, + 16.782, + 13.658, + 13.449, + 14.887, + 7.878, + 22.361, + 13.42, + 9.861, + 52.273, + 6.576, + 13.337, + 11.387, + 81.073, + 14.868, + 4.591, + 69.509, + 11.729, + 90.062, + 17.85, + 6.942, + 78.375, + 16.006, + 12.691, + 52.684, + 6.649, + 12.172, + 8.467, + 12.708, + 56.496, + 5.12, + 17.423, + 16.41, + 15.615, + 16.194, + 12.433, + 15.549, + 11.409, + 13.736, + 13.994, + 9.69, + 14.096, + 10.986, + 11.929, + 18.643, + 9.0, + 15.446, + 18.5, + 19.702, + 18.211, + 8.766, + 19.178, + 17.459, + 26.969, + 15.722, + 6.658, + 13.188, + 16.173, + 80.83, + 13.233, + 7.713, + 23.536, + 19.602, + 8.689, + 8.577, + 33.481, + 10.336, + 12.729, + 70.083, + 14.424, + 8.632, + 14.045, + 12.276, + 69.425, + 14.949, + 6.86, + 10.199, + 8.126, + 56.388, + 15.478, + 6.93, + 11.024, + 60.844, + 16.579, + 14.637, + 5.949, + 71.999, + 11.022, + 74.962, + 13.081, + 8.094, + 10.963, + 84.374, + 12.111, + 12.484, + 8.037, + 7.393, + 66.884, + 15.439, + 16.007, + 6.168, + 49.738, + 8.982, + 8.812, + 9.356, + 6.4, + 9.958, + 12.809, + 13.283, + 21.765, + 7.497, + 15.872, + 16.565, + 15.779, + 16.038, + 7.198, + 17.423, + 14.044, + 15.988, + 15.169, + 10.096, + 22.206, + 13.235, + 19.059, + 17.552, + 8.11, + 19.285, + 16.455, + 13.042, + 26.468, + 8.512, + 12.375, + 17.332, + 16.322, + 47.996, + 5.534, + 11.484, + 85.021, + 20.942, + 75.069, + 7.183, + 76.584, + 30.396, + 13.168, + 86.618, + 72.468, + 21.056, + 11.05, + 9.689, + 15.306, + 7.875, + 18.677, + 13.319, + 12.732, + 14.933, + 7.957, + 12.332, + 13.378, + 15.123, + 12.797, + 4.509, + 7.69, + 11.538, + 13.246, + 19.362, + 4.588, + 8.239, + 10.827, + 12.667, + 14.187, + 10.998, + 12.617, + 12.577, + 11.713, + 15.11, + 9.59, + 11.918, + 11.037, + 13.073, + 19.863, + 4.887, + 10.035, + 14.372, + 16.465, + 13.117, + 5.907, + 12.116, + 11.426, + 13.403, + 21.981, + 4.606, + 8.579, + 8.644, + 10.895, + 15.039, + 8.84, + 10.703, + 12.422, + 13.294, + 18.095, + 7.076, + 13.243, + 11.658, + 11.707, + 13.676, + 10.597, + 11.532, + 14.827, + 13.286, + 12.812, + 4.918, + 13.522, + 13.456, + 11.917, + 13.522, + 5.741, + 11.405, + 14.104, + 11.239, + 14.04, + 6.917, + 15.809, + 10.198, + 8.453, + 14.535, + 5.727, + 8.565, + 12.682, + 12.233, + 13.421, + 7.196, + 15.666, + 13.836, + 16.754, + 14.006, + 7.764, + 11.114, + 10.702, + 14.255, + 14.614, + 4.754, + 8.776, + 10.042, + 16.627, + 13.71, + 6.25, + 13.582, + 14.796, + 12.467, + 14.654, + 8.169, + 10.085, + 12.062, + 11.035, + 14.67, + 7.737, + 12.257, + 11.71, + 15.864, + 12.726, + 5.825, + 17.588, + 18.614, + 14.328, + 12.149, + 7.804, + 13.282, + 13.707, + 10.534, + 12.964, + 7.54, + 15.918, + 12.604, + 14.749, + 14.127, + 7.704, + 14.048, + 11.209, + 16.293, + 21.307, + 7.22, + 12.138, + 14.223, + 14.014, + 16.463, + 5.54, + 11.428, + 10.694, + 17.123, + 15.098, + 6.321, + 12.482, + 12.495, + 14.02, + 18.918, + 6.683, + 13.662, + 13.432, + 13.138, + 15.252, + 9.591, + 14.124, + 12.841, + 13.013, + 23.901, + 10.511, + 16.547, + 31.537, + 23.937, + 21.065, + 52.042, + 25.389, + 16.007, + 72.901, + 19.709, + 8.364, + 57.718, + 12.086, + 15.601, + 16.28, + 57.144, + 18.536, + 19.627, + 67.838, + 15.403, + 7.867, + 64.727, + 16.259, + 14.95, + 15.288, + 5.272, + 8.676, + 18.256, + 33.163, + 13.004, + 7.134, + 13.79, + 13.563, + 12.028, + 12.798, + 6.992, + 13.27, + 10.974, + 13.262, + 14.614, + 6.447, + 18.117, + 11.012, + 10.533, + 14.62, + 5.387, + 13.051, + 16.784, + 17.736, + 14.943, + 7.132, + 14.91, + 15.511, + 11.343, + 12.453, + 8.752, + 13.831, + 16.465, + 16.529, + 11.916, + 6.641, + 22.802, + 20.981, + 11.0, + 13.122, + 6.838, + 15.811, + 12.724, + 13.211, + 16.132, + 7.689, + 13.994, + 18.156, + 17.652, + 14.809, + 7.272, + 22.168, + 16.25, + 13.854, + 13.632, + 69.868, + 20.85, + 10.757, + 14.398, + 14.226, + 7.086, + 13.244, + 10.097, + 16.405, + 13.8, + 7.392, + 16.798, + 18.197, + 13.002, + 15.677, + 8.157, + 23.018, + 14.89, + 16.411, + 16.191, + 6.569, + 8.49, + 16.075, + 13.962, + 22.857, + 7.561, + 14.714, + 14.888, + 12.366, + 15.535, + 6.345, + 14.272, + 12.937, + 19.046, + 11.866, + 8.003, + 13.947, + 19.958, + 17.276, + 13.158, + 7.66, + 14.619, + 14.612, + 19.947, + 18.721, + 7.773, + 23.442, + 15.166, + 30.793, + 27.897, + 7.621, + 37.68, + 21.367, + 16.997, + 15.6, + 11.348, + 25.197, + 9.751, + 13.061, + 16.806, + 9.086, + 19.554, + 17.887, + 13.434, + 15.444, + 6.512, + 15.362, + 19.758, + 19.458, + 18.985, + 7.037, + 10.394, + 10.595, + 13.528, + 23.703, + 28.715, + 12.434, + 13.865, + 19.006, + 73.984, + 8.38, + 80.535, + 12.758, + 25.8, + 51.13, + 13.417, + 16.741, + 96.812, + 15.175, + 15.872, + 43.164, + 20.139, + 19.755, + 24.026, + 60.642, + 5.3, + 171.082, + 19.235, + 16.569, + 15.184, + 4.417, + 7.107, + 62.28, + 9.025, + 82.24, + 13.603, + 88.996, + 77.57, + 13.6, + 10.608, + 8.597, + 13.376, + 10.805, + 10.444, + 17.515, + 24.786, + 9.652, + 16.405, + 12.79, + 14.19, + 8.57, + 55.646, + 13.848, + 80.07, + 14.748, + 7.632, + 10.067, + 7.609, + 138.454, + 12.095, + 6.665, + 12.524, + 13.131, + 10.463, + 11.381, + 3.968, + 67.317, + 11.05, + 10.247, + 13.225, + 5.628, + 23.789, + 18.851, + 10.941, + 19.743, + 5.361, + 15.683, + 17.319, + 23.792, + 18.084, + 6.497, + 8.344, + 13.902, + 12.092, + 14.245, + 7.433, + 82.096, + 13.592, + 24.272, + 19.44, + 4.633, + 14.958, + 15.813, + 13.116, + 14.162, + 8.678, + 21.625, + 19.909, + 12.715, + 14.023, + 6.749, + 11.316, + 23.449, + 18.335, + 12.762, + 12.673, + 15.768, + 13.653, + 12.735, + 19.672, + 5.85, + 9.659, + 11.598, + 15.518, + 54.959, + 25.481, + 76.082, + 14.295, + 18.07, + 70.766, + 11.734, + 9.625, + 11.03, + 67.706, + 18.305, + 71.674, + 9.578, + 16.126, + 69.195, + 13.962, + 6.959, + 9.694, + 11.713, + 8.567, + 58.747, + 6.911, + 13.658, + 12.594, + 10.617, + 12.676, + 5.701, + 13.94, + 16.612, + 8.665, + 17.195, + 5.16, + 8.233, + 10.829, + 17.834, + 13.936, + 5.671, + 16.666, + 12.44, + 12.267, + 15.721, + 7.868, + 11.679, + 12.514, + 14.468, + 19.36, + 5.538, + 9.466, + 9.506, + 12.816, + 15.2, + 9.277, + 13.554, + 13.787, + 12.817, + 18.714, + 15.426, + 22.129, + 15.65, + 18.755, + 18.675, + 7.566, + 16.074, + 14.127, + 20.313, + 18.227, + 6.266, + 18.769, + 14.331, + 15.506, + 15.707, + 9.711, + 15.173, + 15.688, + 15.259, + 19.357, + 6.717, + 15.717, + 14.13, + 13.682, + 14.955, + 5.537, + 9.61, + 10.297, + 13.743, + 15.06, + 6.845, + 14.569, + 14.568, + 14.219, + 16.895, + 7.907, + 15.534, + 15.423, + 16.367, + 14.07, + 9.161, + 13.223, + 15.765, + 12.432, + 20.095, + 9.096, + 19.581, + 22.225, + 14.399, + 19.987, + 7.078, + 9.739, + 11.496, + 20.688, + 16.09, + 7.479, + 23.669, + 9.965, + 12.52, + 13.018, + 7.859, + 14.574, + 13.688, + 20.611, + 17.723, + 8.305, + 20.011, + 14.004, + 14.046, + 15.549, + 6.757, + 11.796, + 10.902, + 11.836, + 13.717, + 10.586, + 15.593, + 20.671, + 12.96, + 19.129, + 16.7, + 8.689, + 6.227, + 11.117, + 22.107, + 81.786, + 11.799, + 75.364, + 27.893, + 84.332, + 14.4, + 10.311, + 62.664, + 9.258, + 16.795, + 15.38, + 79.741, + 5.746, + 84.622, + 10.819, + 11.008, + 16.475, + 9.295, + 21.052, + 11.119, + 8.801, + 10.014, + 13.031, + 14.131, + 13.623, + 13.346, + 12.963, + 8.562, + 13.848, + 13.319, + 14.761, + 14.806, + 6.098, + 20.667, + 19.354, + 22.336, + 15.638, + 8.685, + 15.266, + 14.049, + 12.802, + 34.182, + 11.697, + 11.551, + 11.328, + 7.187, + 70.425, + 7.678, + 95.224, + 84.245, + 9.963, + 11.703, + 6.729, + 11.186, + 7.512, + 57.746, + 11.391, + 11.444, + 8.488, + 8.438, + 69.148, + 13.95, + 5.652, + 12.773, + 17.628, + 16.437, + 10.974, + 5.429, + 12.337, + 16.343, + 16.819, + 8.739, + 4.926, + 57.131, + 11.98, + 14.47, + 13.727, + 9.664, + 12.511, + 12.306, + 10.313, + 11.956, + 7.042, + 19.156, + 13.104, + 13.558, + 13.269, + 4.872, + 16.788, + 13.817, + 9.995, + 12.179, + 8.949, + 18.493, + 14.517, + 13.804, + 12.561, + 7.841, + 17.1, + 20.809, + 10.08, + 18.161, + 11.06, + 15.831, + 12.986, + 14.775, + 15.494, + 9.695, + 16.129, + 23.089, + 22.384, + 19.874, + 11.225, + 17.389, + 14.574, + 17.168, + 21.45, + 7.636, + 13.932, + 13.602, + 13.463, + 14.454, + 8.934, + 14.751, + 12.125, + 11.035, + 15.545, + 5.707, + 14.485, + 13.315, + 14.009, + 13.63, + 7.85, + 13.714, + 13.877, + 19.282, + 26.434, + 6.998, + 19.67, + 15.944, + 10.443, + 62.66, + 6.986, + 18.072, + 10.821, + 9.61, + 62.94, + 7.878, + 10.654, + 10.47, + 71.19, + 19.342, + 64.359, + 12.001, + 16.999, + 84.934, + 13.689, + 78.842, + 17.179, + 87.882, + 14.2, + 68.897, + 6.759, + 8.903, + 14.98, + 71.328, + 17.757, + 6.434, + 21.108, + 65.139, + 16.19, + 109.854, + 62.291, + 11.297, + 92.192, + 15.363, + 14.118, + 4.664, + 91.665, + 85.646, + 24.91, + 12.477, + 5.084, + 8.586, + 48.398, + 17.609, + 81.12, + 6.759, + 11.852, + 17.616, + 10.271, + 14.295, + 6.345, + 31.827, + 25.593, + 17.95, + 13.55, + 5.754, + 21.503, + 16.514, + 19.87, + 20.069, + 8.294, + 14.637, + 18.679, + 14.287, + 22.796, + 9.147, + 10.983, + 16.074, + 14.986, + 24.715, + 8.544, + 15.703, + 14.002, + 27.677, + 19.702, + 9.903, + 20.258, + 19.277, + 13.758, + 19.146, + 8.607, + 23.79, + 15.15, + 15.551, + 13.886, + 12.364, + 27.708, + 19.428, + 15.639, + 15.702, + 9.85, + 16.102, + 16.48, + 16.729, + 13.919, + 7.374, + 22.363, + 11.659, + 13.695, + 15.03, + 8.08, + 13.441, + 14.514, + 16.895, + 15.535, + 7.552, + 54.054, + 16.998, + 17.9, + 145.68, + 16.475, + 106.582, + 15.938, + 98.28, + 11.65, + 4.575, + 141.399, + 9.497, + 19.133, + 7.105, + 20.905, + 14.028, + 13.014, + 29.209, + 9.568, + 15.157, + 17.955, + 10.745, + 14.102, + 31.967, + 7.325, + 83.862, + 89.82, + 12.365, + 10.313, + 19.502, + 18.912, + 9.569, + 44.464, + 15.766, + 161.882, + 13.759, + 105.729, + 26.049, + 7.885, + 28.544, + 11.971, + 8.383, + 28.432, + 10.861, + 58.395, + 31.791, + 14.374, + 15.951, + 7.712, + 11.75, + 13.791, + 14.285, + 18.559, + 6.88, + 16.509, + 13.68, + 15.126, + 15.884, + 7.958, + 12.058, + 14.12, + 14.804, + 16.51, + 7.008, + 17.998, + 14.473, + 13.561, + 13.68, + 5.849, + 10.089, + 9.534, + 9.744, + 15.433, + 6.313, + 10.931, + 16.566, + 13.687, + 13.447, + 6.406, + 12.004, + 21.43, + 19.9, + 20.071, + 9.186, + 20.305, + 20.593, + 132.211, + 13.524, + 6.488, + 25.882, + 16.471, + 70.392, + 10.053, + 4.687, + 7.986, + 82.137, + 11.875, + 12.435, + 46.638, + 23.701, + 11.773, + 4.883, + 73.805, + 15.402, + 18.865, + 12.272, + 38.954, + 20.909, + 9.157, + 13.525, + 19.047, + 15.945, + 27.032, + 11.16, + 8.883, + 16.071, + 16.163, + 8.573, + 20.524, + 12.638, + 9.664, + 18.655, + 18.461, + 7.002, + 15.417, + 16.058, + 14.414, + 14.132, + 8.79, + 16.387, + 18.911, + 19.55, + 11.561, + 8.698, + 9.298, + 9.185, + 4.681, + 14.246, + 14.574, + 30.938, + 12.381, + 23.198, + 18.807, + 17.619, + 15.76, + 12.488, + 16.48, + 22.115, + 14.431, + 19.363, + 9.582, + 16.024, + 21.534, + 18.774, + 19.173, + 7.922, + 17.355, + 14.48, + 19.49, + 17.512, + 8.071, + 15.136, + 12.247, + 22.501, + 22.496, + 8.619, + 14.362, + 14.092, + 17.786, + 30.341, + 14.125, + 44.841, + 26.236, + 14.964, + 19.165, + 11.119, + 12.25, + 32.644, + 16.195, + 11.384, + 6.553, + 10.08, + 15.958, + 35.319, + 20.589, + 8.949, + 15.329, + 14.174, + 6.855, + 19.079, + 14.805, + 10.422, + 16.97, + 17.96, + 17.397, + 27.512, + 20.335, + 16.354, + 5.504, + 8.679, + 49.079, + 13.611, + 7.488, + 78.57, + 13.852, + 84.587, + 14.629, + 164.238, + 22.983, + 9.773, + 103.006, + 75.42, + 23.892, + 12.333, + 15.117, + 14.193, + 13.15, + 9.0, + 9.965, + 5.754, + 9.014, + 16.374, + 18.517, + 19.595, + 6.181, + 24.961, + 21.721, + 17.761, + 11.817, + 5.019, + 58.042, + 15.511, + 10.118, + 8.227, + 165.748, + 14.976, + 75.504, + 10.06, + 10.343, + 5.186, + 11.09, + 8.336, + 7.606, + 58.106, + 8.251, + 27.195, + 21.08, + 19.595, + 15.737, + 10.914, + 17.215, + 12.803, + 17.723, + 13.918, + 5.422, + 14.083, + 16.35, + 12.481, + 12.566, + 5.563, + 11.535, + 10.665, + 9.496, + 13.474, + 7.021, + 11.847, + 10.775, + 13.844, + 12.516, + 13.715, + 14.358, + 16.071, + 15.632, + 17.326, + 14.733, + 50.757, + 11.576, + 20.195, + 11.595, + 8.774, + 58.934, + 17.183, + 71.289, + 6.927, + 9.257, + 10.88, + 10.252, + 5.381, + 8.775, + 8.14, + 22.696, + 10.257, + 32.844, + 88.736, + 16.471, + 18.847, + 5.918, + 13.081, + 19.303, + 18.761, + 13.271, + 9.807, + 13.65, + 13.75, + 12.254, + 13.503, + 5.664, + 19.173, + 12.709, + 18.036, + 14.983, + 7.177, + 13.242, + 16.683, + 12.386, + 13.372, + 8.483, + 11.531, + 14.386, + 17.792, + 15.302, + 4.34, + 10.812, + 15.127, + 12.16, + 13.936, + 8.361, + 13.376, + 12.299, + 13.812, + 13.153, + 7.72, + 15.098, + 14.371, + 12.096, + 13.252, + 5.844, + 10.422, + 13.75, + 10.953, + 12.036, + 4.379, + 22.038, + 11.19, + 11.054, + 14.716, + 5.845, + 8.091, + 8.97, + 13.871, + 19.275, + 9.273, + 15.679, + 13.879, + 14.252, + 19.655, + 8.442, + 10.841, + 12.526, + 18.004, + 15.037, + 5.518, + 8.074, + 10.737, + 11.976, + 15.539, + 8.567, + 14.249, + 13.828, + 14.454, + 27.444, + 6.057, + 85.588, + 13.059, + 12.494, + 12.805, + 55.905, + 21.141, + 84.583, + 17.151, + 12.771, + 66.925, + 17.006, + 74.104, + 29.484, + 69.896, + 6.508, + 8.497, + 8.439, + 12.203, + 16.235, + 7.611, + 11.712, + 13.138, + 21.352, + 13.98, + 6.801, + 13.599, + 78.66, + 11.996, + 11.432, + 57.811, + 12.001, + 10.88, + 10.718, + 78.085, + 10.72, + 17.023, + 71.799, + 14.121, + 14.419, + 8.851, + 75.118, + 13.409, + 15.56, + 12.894, + 6.294, + 11.649, + 8.25, + 24.611, + 13.042, + 11.213, + 14.862, + 11.836, + 15.447, + 13.05, + 9.598, + 15.021, + 17.434, + 12.001, + 59.481, + 4.894, + 25.784, + 15.522, + 17.02, + 16.889, + 6.961, + 18.468, + 17.515, + 11.973, + 15.287, + 5.924, + 7.499, + 7.25, + 31.177, + 12.793, + 8.926, + 73.839, + 17.51, + 16.469, + 69.096, + 12.41, + 73.99, + 24.143, + 19.22, + 72.132, + 12.719, + 18.883, + 14.585, + 8.901, + 41.042, + 6.357, + 11.558, + 13.599, + 14.435, + 60.154, + 5.568, + 81.293, + 14.255, + 14.668, + 16.749, + 58.817, + 88.935, + 12.431, + 101.131, + 93.16, + 7.375, + 11.758, + 106.911, + 91.843, + 17.309, + 70.765, + 9.395, + 13.485, + 9.913, + 81.969, + 10.906, + 15.079, + 71.417, + 13.828, + 93.984, + 6.995, + 11.973, + 14.848, + 52.096, + 15.878, + 7.192, + 9.474, + 8.194, + 60.466, + 15.859, + 9.947, + 13.746, + 69.176, + 12.872, + 16.234, + 8.53, + 10.857, + 8.109, + 34.956, + 15.07, + 6.435, + 9.128, + 11.887, + 22.091, + 25.852, + 10.095, + 13.071, + 10.34, + 27.53, + 16.187, + 16.236, + 19.12, + 13.574, + 10.089, + 15.848, + 7.08, + 16.473, + 13.246, + 11.49, + 12.889, + 6.539, + 13.01, + 16.337, + 14.146, + 13.976, + 5.709, + 15.324, + 12.895, + 10.774, + 13.674, + 8.119, + 13.568, + 14.852, + 15.317, + 15.269, + 7.929, + 15.661, + 14.75, + 14.196, + 15.458, + 9.823, + 15.635, + 10.062, + 15.111, + 12.324, + 8.571, + 16.898, + 18.101, + 14.25, + 12.851, + 6.967, + 9.581, + 9.53, + 16.911, + 14.25, + 7.49, + 15.872, + 16.292, + 12.226, + 15.45, + 6.67, + 12.551, + 10.966, + 14.041, + 12.461, + 7.41, + 11.359, + 15.321, + 15.439, + 16.177, + 5.581, + 9.331, + 11.031, + 13.243, + 12.141, + 8.864, + 11.032, + 13.527, + 12.028, + 13.297, + 6.727, + 13.634, + 12.476, + 11.694, + 13.796, + 7.449, + 12.06, + 14.799, + 11.429, + 11.597, + 5.425, + 16.289, + 11.379, + 13.465, + 9.62, + 6.217, + 12.254, + 13.643, + 11.429, + 15.648, + 8.96, + 8.0, + 8.327, + 11.599, + 15.695, + 10.426, + 13.144, + 11.338, + 13.184, + 13.083, + 4.497, + 19.533, + 11.035, + 11.163, + 12.048, + 8.526, + 16.101, + 12.578, + 11.096, + 14.754, + 8.613, + 11.958, + 13.188, + 10.711, + 15.428, + 9.253, + 11.896, + 11.74, + 12.997, + 20.578, + 5.049, + 7.953, + 41.094, + 11.925, + 82.884, + 4.721, + 18.218, + 82.053, + 12.921, + 12.893, + 4.552, + 67.413, + 17.764, + 14.457, + 69.309, + 18.804, + 16.707, + 60.41, + 15.03, + 19.076, + 51.439, + 10.405, + 25.478, + 12.041, + 9.285, + 54.046, + 10.248, + 9.714, + 15.422, + 11.751, + 7.916, + 29.85, + 7.648, + 19.674, + 13.822, + 5.42, + 10.957, + 11.599, + 14.269, + 8.916, + 6.631, + 10.894, + 12.62, + 14.085, + 14.129, + 7.14, + 13.204, + 12.862, + 16.152, + 14.689, + 7.036, + 9.707, + 13.361, + 17.428, + 12.936, + 6.501, + 18.545, + 13.568, + 11.744, + 16.826, + 7.152, + 18.198, + 16.888, + 11.888, + 12.912, + 6.372, + 14.988, + 10.93, + 15.088, + 14.155, + 6.171, + 12.868, + 13.288, + 12.759, + 11.938, + 8.063, + 15.378, + 12.67, + 13.991, + 22.323, + 10.17, + 14.255, + 16.694, + 13.417, + 15.823, + 4.932, + 7.906, + 16.124, + 8.801, + 9.471, + 12.197, + 14.539, + 14.079, + 13.976, + 61.662, + 4.358, + 21.137, + 8.012, + 14.875, + 12.286, + 4.352, + 18.728, + 14.313, + 13.209, + 13.04, + 4.49, + 16.518, + 9.024, + 8.074, + 9.796, + 4.894, + 13.031, + 14.447, + 12.424, + 14.449, + 8.753, + 9.097, + 14.628, + 11.253, + 15.945, + 7.012, + 13.63, + 15.976, + 7.947, + 14.905, + 6.247, + 8.783, + 9.774, + 14.501, + 12.11, + 6.36, + 12.214, + 13.88, + 14.42, + 13.688, + 5.384, + 19.447, + 14.419, + 10.285, + 14.368, + 8.372, + 11.018, + 15.507, + 11.233, + 12.103, + 9.028, + 15.232, + 12.64, + 12.519, + 13.068, + 5.002, + 13.471, + 12.048, + 15.757, + 13.278, + 7.507, + 11.038, + 12.148, + 13.439, + 14.518, + 6.239, + 12.21, + 12.804, + 11.838, + 12.175, + 7.862, + 14.892, + 14.148, + 15.133, + 16.885, + 7.048, + 13.109, + 12.907, + 61.171, + 16.685, + 7.319, + 10.156, + 7.22, + 74.219, + 74.014, + 5.351, + 10.149, + 88.165, + 13.747, + 13.263, + 71.18, + 29.873, + 76.481, + 21.55, + 16.022, + 6.716, + 56.27, + 11.577, + 14.43, + 15.194, + 9.398, + 13.498, + 15.147, + 14.81, + 13.384, + 7.166, + 16.405, + 15.199, + 15.012, + 16.557, + 7.798, + 16.811, + 14.549, + 16.434, + 19.619, + 10.063, + 16.961, + 19.717, + 19.472, + 17.329, + 12.414, + 21.621, + 24.413, + 16.284, + 63.297, + 10.077, + 26.125, + 26.506, + 65.71, + 14.837, + 6.777, + 76.753, + 14.41, + 13.122, + 10.63, + 4.179, + 60.645, + 10.877, + 12.7, + 71.852, + 6.546, + 11.933, + 13.315, + 12.753, + 8.343, + 4.015, + 47.637, + 14.319, + 13.77, + 8.908, + 4.004, + 57.027, + 13.857, + 78.194, + 22.764, + 6.217, + 7.482, + 77.569, + 12.209, + 14.897, + 60.907, + 24.693, + 13.868, + 11.067, + 49.68, + 4.369, + 25.667, + 17.489, + 11.671, + 11.02, + 5.349, + 12.06, + 20.867, + 12.35, + 15.093, + 7.761, + 21.872, + 24.186, + 15.216, + 16.434, + 9.706, + 28.718, + 17.743, + 23.85, + 13.851, + 5.65, + 18.512, + 15.019, + 14.877, + 13.745, + 4.899, + 9.64, + 10.125, + 14.379, + 18.723, + 9.57, + 61.377, + 81.479, + 10.114, + 13.407, + 5.175, + 11.494, + 66.021, + 19.661, + 13.052, + 70.225, + 20.318, + 79.605, + 13.949, + 14.526, + 6.139, + 9.937, + 62.931, + 11.753, + 18.47, + 9.989, + 18.233, + 15.099, + 12.532, + 12.446, + 7.028, + 11.902, + 12.758, + 13.154, + 10.009, + 6.318, + 8.655, + 13.051, + 10.156, + 14.352, + 8.315, + 20.719, + 11.33, + 12.846, + 12.268, + 4.729, + 14.108, + 11.017, + 9.61, + 15.111, + 5.786, + 8.591, + 14.978, + 13.129, + 15.123, + 7.335, + 17.685, + 12.101, + 11.124, + 14.018, + 7.875, + 12.785, + 12.733, + 12.185, + 16.494, + 6.042, + 25.785, + 16.017, + 14.56, + 19.249, + 9.376, + 13.566, + 13.356, + 12.178, + 13.709, + 9.196, + 9.901, + 10.985, + 14.818, + 14.162, + 6.357, + 13.726, + 11.557, + 13.445, + 16.475, + 6.401, + 12.383, + 12.838, + 12.433, + 13.192, + 7.306, + 10.659, + 12.019, + 12.186, + 12.568, + 7.969, + 20.501, + 12.156, + 17.704, + 15.848, + 6.193, + 11.844, + 18.566, + 14.14, + 10.896, + 9.557, + 15.441, + 13.832, + 17.185, + 13.897, + 5.035, + 15.944, + 21.575, + 14.088, + 17.953, + 7.258, + 11.463, + 13.539, + 10.778, + 12.148, + 10.319, + 11.62, + 14.103, + 13.723, + 14.648, + 7.875, + 13.861, + 13.775, + 12.464, + 12.761, + 9.159, + 14.217, + 15.909, + 12.038, + 14.287, + 6.54, + 11.345, + 11.176, + 15.352, + 14.11, + 5.766, + 17.911, + 12.323, + 10.338, + 15.446, + 5.275, + 12.83, + 15.043, + 17.282, + 15.636, + 8.413, + 14.915, + 12.125, + 14.482, + 13.93, + 6.67, + 12.156, + 12.353, + 14.661, + 14.767, + 6.997, + 15.364, + 10.974, + 14.755, + 16.612, + 7.439, + 12.323, + 14.857, + 14.375, + 16.439, + 6.909, + 12.485, + 14.968, + 13.865, + 18.396, + 9.024, + 28.676, + 34.343, + 21.944, + 72.205, + 9.358, + 19.755, + 78.758, + 19.127, + 75.197, + 12.18, + 14.099, + 73.13, + 15.73, + 15.262, + 60.842, + 24.673, + 28.118, + 62.929, + 15.026, + 5.276, + 20.179, + 9.329, + 31.238, + 9.537, + 10.515, + 12.939, + 13.416, + 12.336, + 11.92, + 5.496, + 10.33, + 12.722, + 16.74, + 14.006, + 6.133, + 11.305, + 9.129, + 10.585, + 16.162, + 7.209, + 10.2, + 12.299, + 13.05, + 16.92, + 7.702, + 12.677, + 12.506, + 12.437, + 13.559, + 6.58, + 15.143, + 11.795, + 13.008, + 15.214, + 5.937, + 11.214, + 17.607, + 14.92, + 22.678, + 12.433, + 13.135, + 11.419, + 13.65, + 13.396, + 6.094, + 15.522, + 16.34, + 17.247, + 18.21, + 5.134, + 8.316, + 11.153, + 14.404, + 18.605, + 4.492, + 8.384, + 11.421, + 13.827, + 14.265, + 5.049, + 10.134, + 68.232, + 16.644, + 14.092, + 8.207, + 11.383, + 13.764, + 13.783, + 13.051, + 5.55, + 14.795, + 13.602, + 17.791, + 15.507, + 6.62, + 16.165, + 20.548, + 17.818, + 14.934, + 9.197, + 17.615, + 18.693, + 14.005, + 13.705, + 5.275, + 15.865, + 10.662, + 12.461, + 16.9, + 8.018, + 11.61, + 13.004, + 11.985, + 13.9, + 9.579, + 20.527, + 14.78, + 20.401, + 20.293, + 6.743, + 17.769, + 16.505, + 12.588, + 16.281, + 5.926, + 21.115, + 23.933, + 26.364, + 28.463, + 5.974, + 11.292, + 16.309, + 11.84, + 24.956, + 6.844, + 11.139, + 27.673, + 11.784, + 28.079, + 12.849, + 15.86, + 14.621, + 14.054, + 11.035, + 9.406, + 18.048, + 15.243, + 11.822, + 17.11, + 6.955, + 23.382, + 15.785, + 21.027, + 22.067, + 7.137, + 14.33, + 41.805, + 25.586, + 19.817, + 9.918, + 62.94, + 12.434, + 12.912, + 83.51, + 10.555, + 73.023, + 17.638, + 20.209, + 57.765, + 24.673, + 16.542, + 70.71, + 18.698, + 161.247, + 15.49, + 91.283, + 13.91, + 13.745, + 69.888, + 4.563, + 23.227, + 9.08, + 9.355, + 68.868, + 6.195, + 15.351, + 9.049, + 7.524, + 70.496, + 6.801, + 8.564, + 77.24, + 18.728, + 22.428, + 44.897, + 18.467, + 14.833, + 13.676, + 46.016, + 4.942, + 22.431, + 11.408, + 10.842, + 10.135, + 47.564, + 15.206, + 11.932, + 94.715, + 11.463, + 4.387, + 70.241, + 13.007, + 12.547, + 84.545, + 5.903, + 11.492, + 12.075, + 15.527, + 15.892, + 4.963, + 7.668, + 8.136, + 8.71, + 12.676, + 5.778, + 16.592, + 8.841, + 10.506, + 14.019, + 6.26, + 21.682, + 9.352, + 13.282, + 16.437, + 6.01, + 16.315, + 91.704, + 10.171, + 13.059, + 5.263, + 16.838, + 9.616, + 9.174, + 17.847, + 8.398, + 15.942, + 13.567, + 22.777, + 10.742, + 5.894, + 15.241, + 10.811, + 15.222, + 17.347, + 6.145, + 8.567, + 9.527, + 9.384, + 18.765, + 7.308, + 13.317, + 11.111, + 16.473, + 16.853, + 6.547, + 10.065, + 27.125, + 14.859, + 14.65, + 5.641, + 39.358, + 89.529, + 14.911, + 17.747, + 6.965, + 9.409, + 65.697, + 83.272, + 20.506, + 5.587, + 9.346, + 11.06, + 67.758, + 16.025, + 6.145, + 81.293, + 76.755, + 18.996, + 12.105, + 7.556, + 14.138, + 12.759, + 10.036, + 11.112, + 9.067, + 10.081, + 24.503, + 16.031, + 13.846, + 7.968, + 13.355, + 11.144, + 11.786, + 18.447, + 15.354, + 15.162, + 13.371, + 14.653, + 15.928, + 9.591, + 9.496, + 24.851, + 11.882, + 15.749, + 7.602, + 16.024, + 13.438, + 12.539, + 13.498, + 8.834, + 24.832, + 12.724, + 16.127, + 15.359, + 6.65, + 17.79, + 13.995, + 16.175, + 17.661, + 7.708, + 17.826, + 17.565, + 13.38, + 22.45, + 6.235, + 18.421, + 12.508, + 15.368, + 16.898, + 7.72, + 17.06, + 12.467, + 15.245, + 19.3, + 8.777, + 24.331, + 13.523, + 13.563, + 15.538, + 8.227, + 16.448, + 13.411, + 18.62, + 17.946, + 7.609, + 11.82, + 13.952, + 17.962, + 17.125, + 11.018, + 10.911, + 17.821, + 9.347, + 11.54, + 8.465, + 24.576, + 16.578, + 19.013, + 21.119, + 10.985, + 11.547, + 16.402, + 15.287, + 15.309, + 7.224, + 9.68, + 16.739, + 18.9, + 16.274, + 11.499, + 20.861, + 13.871, + 12.697, + 13.958, + 7.212, + 17.271, + 13.535, + 13.659, + 15.622, + 11.422, + 15.346, + 20.293, + 15.622, + 15.664, + 6.648, + 17.624, + 14.657, + 13.367, + 6.465, + 31.706, + 24.605, + 80.408, + 79.661, + 5.754, + 10.229, + 21.483, + 82.86, + 11.816, + 8.599, + 11.854, + 67.345, + 17.089, + 71.165, + 5.453, + 95.708, + 14.819, + 12.991, + 19.632, + 8.224, + 9.133, + 8.502, + 30.891, + 9.761, + 7.914, + 14.54, + 14.627, + 14.355, + 13.375, + 5.461, + 14.475, + 15.402, + 13.606, + 17.112, + 5.337, + 9.565, + 10.545, + 26.76, + 14.325, + 9.336, + 14.646, + 11.019, + 10.927, + 13.137, + 24.529, + 10.376, + 9.536, + 11.34, + 12.546, + 65.793, + 21.337, + 10.794, + 90.335, + 84.616, + 8.006, + 13.44, + 73.385, + 15.04, + 12.69, + 65.384, + 9.048, + 8.122, + 10.561, + 15.126, + 6.588, + 7.951, + 15.793, + 19.824, + 10.642, + 6.959, + 10.588, + 7.68, + 8.277, + 12.396, + 58.873, + 11.242, + 8.957, + 10.137, + 13.376, + 4.325, + 10.137, + 12.543, + 11.396, + 11.133, + 6.669, + 14.593, + 8.214, + 10.449, + 10.872, + 7.64, + 12.478, + 11.805, + 8.052, + 8.868, + 7.393, + 14.002, + 13.912, + 15.228, + 12.443, + 8.162, + 15.774, + 15.74, + 15.444, + 17.229, + 6.887, + 10.382, + 18.798, + 15.211, + 15.468, + 9.446, + 14.521, + 14.729, + 20.487, + 16.498, + 7.158, + 11.626, + 9.085, + 16.077, + 20.615, + 9.199, + 20.879, + 13.288, + 19.498, + 21.064, + 6.958, + 14.806, + 14.636, + 14.902, + 12.979, + 6.705, + 13.005, + 16.734, + 17.457, + 12.902, + 7.061, + 14.851, + 14.976, + 15.46, + 23.436, + 10.578, + 16.09, + 18.249, + 18.554, + 12.875, + 6.404, + 78.854, + 20.814, + 87.816, + 21.77, + 76.427, + 14.372, + 18.871, + 11.05, + 11.009, + 42.316, + 23.443, + 85.877, + 15.042, + 8.78, + 3.964, + 82.473, + 12.784, + 78.69, + 16.185, + 7.452, + 7.254, + 73.747, + 16.069, + 10.32, + 4.351, + 59.2, + 13.087, + 14.67, + 78.436, + 10.224, + 124.724, + 11.742, + 7.26, + 42.325, + 9.369, + 90.052, + 15.259, + 98.157, + 84.474, + 5.168, + 24.815, + 71.859, + 13.026, + 69.131, + 4.824, + 21.204, + 17.728, + 9.943, + 12.924, + 6.473, + 27.059, + 8.59, + 11.303, + 16.412, + 12.309, + 13.744, + 16.44, + 13.306, + 14.414, + 9.738, + 13.437, + 19.445, + 10.691, + 15.449, + 6.32, + 19.713, + 14.209, + 16.011, + 26.627, + 10.356, + 16.297, + 22.828, + 20.814, + 16.121, + 8.342, + 12.422, + 10.684, + 23.087, + 19.566, + 8.27, + 15.933, + 16.479, + 16.953, + 20.12, + 5.386, + 14.786, + 13.823, + 14.648, + 17.811, + 7.802, + 24.629, + 21.949, + 13.566, + 19.631, + 7.934, + 14.406, + 15.868, + 13.181, + 16.499, + 8.137, + 24.199, + 11.3, + 12.775, + 14.424, + 6.75, + 10.913, + 12.706, + 15.375, + 20.783, + 7.151, + 53.87, + 18.517, + 163.662, + 13.359, + 9.9, + 15.365, + 12.718, + 8.2, + 75.143, + 78.706, + 10.902, + 10.489, + 12.456, + 100.975, + 5.518, + 8.113, + 62.412, + 11.263, + 5.668, + 14.801, + 12.497, + 14.69, + 8.404, + 8.491, + 4.835, + 13.207, + 13.514, + 11.215, + 19.921, + 13.141, + 10.45, + 16.311, + 78.862, + 11.69, + 12.087, + 7.523, + 15.467, + 9.758, + 63.617, + 19.036, + 9.291, + 75.514, + 22.952, + 161.343, + 96.173, + 5.324, + 36.265, + 31.424, + 44.363, + 25.778, + 5.028, + 9.82, + 49.919, + 20.426, + 11.958, + 8.616, + 14.847, + 12.619, + 12.538, + 15.549, + 7.353, + 15.192, + 18.527, + 13.44, + 14.76, + 5.594, + 13.73, + 14.977, + 12.724, + 14.547, + 6.613, + 17.865, + 10.813, + 13.175, + 14.72, + 7.752, + 11.823, + 20.411, + 11.741, + 14.523, + 5.246, + 11.488, + 11.011, + 9.631, + 11.763, + 6.706, + 15.967, + 12.039, + 19.853, + 18.099, + 10.181, + 19.328, + 19.083, + 22.531, + 75.933, + 7.124, + 107.384, + 11.083, + 9.128, + 9.296, + 4.498, + 124.556, + 17.189, + 76.283, + 12.282, + 8.21, + 14.079, + 83.464, + 20.153, + 17.062, + 36.327, + 10.39, + 6.141, + 15.924, + 19.487, + 17.395, + 5.911, + 13.456, + 22.727, + 23.744, + 11.016, + 14.62, + 14.815, + 20.603, + 21.725, + 15.08, + 18.373, + 10.415, + 16.744, + 20.236, + 7.629, + 15.721, + 17.334, + 8.734, + 16.478, + 13.165, + 12.219, + 9.224, + 13.738, + 10.526, + 5.193, + 15.046, + 16.308, + 18.863, + 11.75, + 13.2, + 29.684, + 18.809, + 18.077, + 7.995, + 30.504, + 15.624, + 22.351, + 19.296, + 10.892, + 17.733, + 17.065, + 16.272, + 19.715, + 8.882, + 21.009, + 13.209, + 15.528, + 16.041, + 8.032, + 16.062, + 18.316, + 14.008, + 15.53, + 18.086, + 14.924, + 18.817, + 14.846, + 18.2, + 10.98, + 11.869, + 12.777, + 19.385, + 14.82, + 7.295, + 12.148, + 12.06, + 11.104, + 19.756, + 5.289, + 9.912, + 9.05, + 26.917, + 29.089, + 5.791, + 12.867, + 19.17, + 24.775, + 14.594, + 12.062, + 24.214, + 14.116, + 14.652, + 15.659, + 16.899, + 16.29, + 13.871, + 17.346, + 17.607, + 47.608, + 16.359, + 7.392, + 11.818, + 67.641, + 83.95, + 6.28, + 17.441, + 85.538, + 22.331, + 74.758, + 90.795, + 24.832, + 101.895, + 13.206, + 53.703, + 32.042, + 28.948, + 16.148, + 47.858, + 12.372, + 22.688, + 12.816, + 22.906, + 15.191, + 8.07, + 13.254, + 15.158, + 73.395, + 17.601, + 182.532, + 18.478, + 9.314, + 7.975, + 56.344, + 7.655, + 16.716, + 70.676, + 10.939, + 11.366, + 7.293, + 10.535, + 31.176, + 14.371, + 19.608, + 10.452, + 21.167, + 21.145, + 18.883, + 12.483, + 5.1, + 16.498, + 17.026, + 16.58, + 13.233, + 6.22, + 14.179, + 9.472, + 11.499, + 10.919, + 5.18, + 12.55, + 14.893, + 14.509, + 7.894, + 14.335, + 15.149, + 15.515, + 8.525, + 15.008, + 69.76, + 14.216, + 78.463, + 15.457, + 15.53, + 9.199, + 46.109, + 10.295, + 9.12, + 9.41, + 4.588, + 19.061, + 19.26, + 10.057, + 36.509, + 82.794, + 11.431, + 13.736, + 8.79, + 17.869, + 14.99, + 18.784, + 10.876, + 8.567, + 13.256, + 14.468, + 12.551, + 15.503, + 6.164, + 34.621, + 14.351, + 15.324, + 14.069, + 10.776, + 12.033, + 19.216, + 11.754, + 14.004, + 9.687, + 13.796, + 11.182, + 13.589, + 16.07, + 10.403, + 14.419, + 11.877, + 13.407, + 15.239, + 8.138, + 13.99, + 12.231, + 12.859, + 13.163, + 6.526, + 17.851, + 14.047, + 12.688, + 14.521, + 6.709, + 11.262, + 13.866, + 14.349, + 21.861, + 6.881, + 17.095, + 13.128, + 20.588, + 14.598, + 8.643, + 19.13, + 16.883, + 15.545, + 12.272, + 6.926, + 21.491, + 10.834, + 12.155, + 19.147, + 7.576, + 11.077, + 16.277, + 13.98, + 13.485, + 7.484, + 16.01, + 17.322, + 33.892, + 13.35, + 11.735, + 78.433, + 13.11, + 11.809, + 13.146, + 4.366, + 59.418, + 10.023, + 8.34, + 83.594, + 8.019, + 84.715, + 17.164, + 13.932, + 9.148, + 56.921, + 15.125, + 82.315, + 18.7, + 13.112, + 7.104, + 11.267, + 18.411, + 11.413, + 12.254, + 9.07, + 13.503, + 14.824, + 15.283, + 66.86, + 9.993, + 15.316, + 8.582, + 74.359, + 77.346, + 15.936, + 16.947, + 15.284, + 7.658, + 8.101, + 37.342, + 13.763, + 14.468, + 9.707, + 78.161, + 9.046, + 10.828, + 8.03, + 13.603, + 12.927, + 5.932, + 32.732, + 16.623, + 16.628, + 12.915, + 9.346, + 12.116, + 15.607, + 16.39, + 17.233, + 6.938, + 18.123, + 13.359, + 9.949, + 48.782, + 8.831, + 16.597, + 22.162, + 15.751, + 15.056, + 7.561, + 14.26, + 13.716, + 19.271, + 50.142, + 5.366, + 12.45, + 72.69, + 11.431, + 15.201, + 7.304, + 13.0, + 68.39, + 77.019, + 15.145, + 6.13, + 23.593, + 9.816, + 67.712, + 17.266, + 11.592, + 68.96, + 14.273, + 12.829, + 58.752, + 5.067, + 23.833, + 81.197, + 25.754, + 81.325, + 13.685, + 14.507, + 10.398, + 139.734, + 15.203, + 8.127, + 188.949, + 94.791, + 13.406, + 14.734, + 10.132, + 14.627, + 64.013, + 17.056, + 10.63, + 4.311, + 73.636, + 16.458, + 11.011, + 9.956, + 43.528, + 11.412, + 79.097, + 21.019, + 25.116, + 6.661, + 60.059, + 15.711, + 71.279, + 13.964, + 10.926, + 13.085, + 82.958, + 15.076, + 14.856, + 5.956, + 51.417, + 15.207, + 13.176, + 17.738, + 6.073, + 11.194, + 8.734, + 9.489, + 10.353, + 5.225, + 9.055, + 12.909, + 14.14, + 12.888, + 4.67, + 9.031, + 9.067, + 33.73, + 17.886, + 7.919, + 12.825, + 14.924, + 18.005, + 14.809, + 12.828, + 12.59, + 13.332, + 19.19, + 13.774, + 7.721, + 13.546, + 16.644, + 14.21, + 13.84, + 6.554, + 9.903, + 13.639, + 15.779, + 13.508, + 6.361, + 13.819, + 14.506, + 18.293, + 15.971, + 7.893, + 11.287, + 12.795, + 15.533, + 11.593, + 5.949, + 14.914, + 16.958, + 18.367, + 16.458, + 8.421, + 13.846, + 21.574, + 9.603, + 15.511, + 9.255, + 14.406, + 15.271, + 14.116, + 16.604, + 5.124, + 10.25, + 14.1, + 14.882, + 13.057, + 5.797, + 11.696, + 15.407, + 12.985, + 14.109, + 6.527, + 20.9, + 16.831, + 17.271, + 15.68, + 11.726, + 19.13, + 15.359, + 13.308, + 13.208, + 7.555, + 12.491, + 13.705, + 12.928, + 13.649, + 7.55, + 15.885, + 16.868, + 13.361, + 13.501, + 5.959, + 17.796, + 11.577, + 10.325, + 14.094, + 6.969, + 10.582, + 20.66, + 13.902, + 12.348, + 9.508, + 15.045, + 12.966, + 12.721, + 15.627, + 6.559, + 8.377, + 8.734, + 11.509, + 11.676, + 8.718, + 22.068, + 13.104, + 11.32, + 14.665, + 9.175, + 13.386, + 11.537, + 9.971, + 15.216, + 7.961, + 13.391, + 13.977, + 11.83, + 14.873, + 47.484, + 15.708, + 11.633, + 88.024, + 14.835, + 6.202, + 75.806, + 12.803, + 77.661, + 11.099, + 7.362, + 15.517, + 10.134, + 53.11, + 30.047, + 9.912, + 66.729, + 14.899, + 22.496, + 65.006, + 5.245, + 82.716, + 8.164, + 19.235, + 10.394, + 4.899, + 13.426, + 12.993, + 40.649, + 11.809, + 5.106, + 11.011, + 11.636, + 10.652, + 13.336, + 8.746, + 10.983, + 10.543, + 9.879, + 13.746, + 8.025, + 8.69, + 11.251, + 10.023, + 10.716, + 7.927, + 15.556, + 16.265, + 10.781, + 12.485, + 6.489, + 15.271, + 12.679, + 7.706, + 9.711, + 6.411, + 14.115, + 10.928, + 17.467, + 15.0, + 5.158, + 9.594, + 20.92, + 10.088, + 12.801, + 5.622, + 13.295, + 11.007, + 12.963, + 12.555, + 5.846, + 8.901, + 12.001, + 13.03, + 15.957, + 6.526, + 14.386, + 14.848, + 11.52, + 17.509, + 6.351, + 16.747, + 16.101, + 13.595, + 13.089, + 5.529, + 15.021, + 10.693, + 22.016, + 14.028, + 8.931, + 10.503, + 10.553, + 18.856, + 69.215, + 4.551, + 8.243, + 16.877, + 24.982, + 9.609, + 4.578, + 9.601, + 14.77, + 11.632, + 15.273, + 5.392, + 9.013, + 27.148, + 16.562, + 10.719, + 6.984, + 10.194, + 13.9, + 13.962, + 11.375, + 6.184, + 11.557, + 15.059, + 11.599, + 14.223, + 5.318, + 17.948, + 10.501, + 15.447, + 12.857, + 5.676, + 13.512, + 10.131, + 12.867, + 12.241, + 5.851, + 8.406, + 11.602, + 11.558, + 15.599, + 8.206, + 14.971, + 11.93, + 10.527, + 14.795, + 8.056, + 14.264, + 11.669, + 22.2, + 12.151, + 5.461, + 10.075, + 13.107, + 18.085, + 14.782, + 7.231, + 11.55, + 12.057, + 13.37, + 12.84, + 5.099, + 10.119, + 8.92, + 14.296, + 13.707, + 7.251, + 15.937, + 12.67, + 11.373, + 15.575, + 8.865, + 11.878, + 12.203, + 14.199, + 17.095, + 6.767, + 12.716, + 7.437, + 6.83, + 35.319, + 4.857, + 13.733, + 68.518, + 17.568, + 14.467, + 6.46, + 77.541, + 19.547, + 12.311, + 8.379, + 49.444, + 10.822, + 18.967, + 77.121, + 21.905, + 12.584, + 65.819, + 11.388, + 23.949, + 14.053, + 4.241, + 56.255, + 17.331, + 14.883, + 18.71, + 6.49, + 17.101, + 14.863, + 18.726, + 16.838, + 8.002, + 15.598, + 13.033, + 11.586, + 13.644, + 11.409, + 16.125, + 17.355, + 17.863, + 19.221, + 8.801, + 13.617, + 17.034, + 28.09, + 15.595, + 9.559, + 12.975, + 19.602, + 11.693, + 9.565, + 39.933, + 19.728, + 14.527, + 10.564, + 9.977, + 5.223, + 71.062, + 15.953, + 15.489, + 9.241, + 62.829, + 12.784, + 10.174, + 79.848, + 14.682, + 7.607, + 71.781, + 14.398, + 14.994, + 13.857, + 51.437, + 8.499, + 12.142, + 11.454, + 77.266, + 8.246, + 16.046, + 12.14, + 7.866, + 60.898, + 5.544, + 74.888, + 13.061, + 9.804, + 13.396, + 6.763, + 73.038, + 20.448, + 10.126, + 14.139, + 5.407, + 6.848, + 39.865, + 27.085, + 14.896, + 8.863, + 12.805, + 11.364, + 20.045, + 12.812, + 6.722, + 15.775, + 14.168, + 13.399, + 12.945, + 5.475, + 10.543, + 13.524, + 13.979, + 15.718, + 7.416, + 26.326, + 15.141, + 10.652, + 17.569, + 11.525, + 12.078, + 18.174, + 12.88, + 12.642, + 6.712, + 25.086, + 13.156, + 19.822, + 49.379, + 4.648, + 23.975, + 11.001, + 9.651, + 65.56, + 7.519, + 19.984, + 77.097, + 12.157, + 86.36, + 8.701, + 12.787, + 57.955, + 11.543, + 33.53, + 66.179, + 15.741, + 14.552, + 11.324, + 11.526, + 4.132, + 7.855, + 8.054, + 12.747, + 12.455, + 6.138, + 10.737, + 11.478, + 11.424, + 13.373, + 8.242, + 10.05, + 16.689, + 13.272, + 16.358, + 5.787, + 19.659, + 12.491, + 14.338, + 15.198, + 4.305, + 9.646, + 11.776, + 11.349, + 10.852, + 5.147, + 20.491, + 10.811, + 10.777, + 15.081, + 6.828, + 10.338, + 13.271, + 10.963, + 12.384, + 9.661, + 12.944, + 12.425, + 12.023, + 12.16, + 10.499, + 22.581, + 17.884, + 11.311, + 15.003, + 6.204, + 13.001, + 12.02, + 11.88, + 14.999, + 6.38, + 14.845, + 13.444, + 14.557, + 13.512, + 8.026, + 14.877, + 12.35, + 12.146, + 15.995, + 8.326, + 14.116, + 13.929, + 16.373, + 14.563, + 6.603, + 11.654, + 13.031, + 11.595, + 16.358, + 14.275, + 14.183, + 13.451, + 16.741, + 12.038, + 7.44, + 13.089, + 19.856, + 13.565, + 15.314, + 9.931, + 11.664, + 13.769, + 14.333, + 17.66, + 7.337, + 19.389, + 15.692, + 12.226, + 14.191, + 8.998, + 13.558, + 10.406, + 13.847, + 14.068, + 5.497, + 12.154, + 11.729, + 12.315, + 13.989, + 7.679, + 11.78, + 12.533, + 11.057, + 13.371, + 10.4, + 17.32, + 16.245, + 12.459, + 15.939, + 5.599, + 12.87, + 12.848, + 13.523, + 14.036, + 7.278, + 7.735, + 11.479, + 12.974, + 13.062, + 6.931, + 11.742, + 12.151, + 13.463, + 14.302, + 10.579, + 17.703, + 14.911, + 12.023, + 17.231, + 5.921, + 13.772, + 11.409, + 11.836, + 13.274, + 8.636, + 16.94, + 11.713, + 9.8, + 13.735, + 8.841, + 8.042, + 11.963, + 10.811, + 12.817, + 7.204, + 13.215, + 12.731, + 13.892, + 14.563, + 6.908, + 11.239, + 25.086, + 23.631, + 31.673, + 11.167, + 24.339, + 72.385, + 16.687, + 74.452, + 7.56, + 17.25, + 19.877, + 71.828, + 15.282, + 7.715, + 64.22, + 18.28, + 17.297, + 73.662, + 7.6, + 17.327, + 72.555, + 14.513, + 14.448, + 6.65, + 18.972, + 12.134, + 34.561, + 13.748, + 9.724, + 13.829, + 15.045, + 14.456, + 15.738, + 6.409, + 13.461, + 12.675, + 12.838, + 18.8, + 6.237, + 15.501, + 15.469, + 10.783, + 13.896, + 8.065, + 11.257, + 12.659, + 13.985, + 17.34, + 6.881, + 11.691, + 18.921, + 12.799, + 12.759, + 6.6, + 10.409, + 19.308, + 14.057, + 13.562, + 6.034, + 20.722, + 16.572, + 14.407, + 16.117, + 7.509, + 12.891, + 12.312, + 19.562, + 13.7, + 10.342, + 23.946, + 14.467, + 14.735, + 9.252, + 5.015, + 17.546, + 13.048, + 14.353, + 86.707, + 10.173, + 14.02, + 12.295, + 15.219, + 14.683, + 8.483, + 13.958, + 12.42, + 12.611, + 16.174, + 5.556, + 8.334, + 9.027, + 15.265, + 18.13, + 7.393, + 9.702, + 10.423, + 13.391, + 15.762, + 8.219, + 8.761, + 16.254, + 11.487, + 12.407, + 8.804, + 21.512, + 12.867, + 13.329, + 16.269, + 7.854, + 11.029, + 11.877, + 11.433, + 17.92, + 6.927, + 9.548, + 15.033, + 14.523, + 13.987, + 6.911, + 13.205, + 20.225, + 15.68, + 16.099, + 7.007, + 18.901, + 10.358, + 19.299, + 15.075, + 8.719, + 36.971, + 10.727, + 20.536, + 15.394, + 16.957, + 23.973, + 25.725, + 17.547, + 19.607, + 6.93, + 11.106, + 19.158, + 15.088, + 14.567, + 7.244, + 18.159, + 11.047, + 13.361, + 19.158, + 8.796, + 13.835, + 20.9, + 14.135, + 14.175, + 7.893, + 23.863, + 18.437, + 19.361, + 44.596, + 9.057, + 22.529, + 71.493, + 16.852, + 12.877, + 4.634, + 65.808, + 27.281, + 74.291, + 20.166, + 8.498, + 147.158, + 25.164, + 17.901, + 22.214, + 5.843, + 135.604, + 99.918, + 14.714, + 15.42, + 9.799, + 79.827, + 77.665, + 9.096, + 12.271, + 9.681, + 91.715, + 76.022, + 22.484, + 16.146, + 6.735, + 23.405, + 13.728, + 8.282, + 28.029, + 6.168, + 10.187, + 13.897, + 13.217, + 58.433, + 5.75, + 11.401, + 81.125, + 13.418, + 76.466, + 6.227, + 81.739, + 19.565, + 10.972, + 18.019, + 73.557, + 13.579, + 12.602, + 10.563, + 14.649, + 8.814, + 11.119, + 14.148, + 13.355, + 13.029, + 5.698, + 7.74, + 19.417, + 12.453, + 11.675, + 15.445, + 17.164, + 10.749, + 14.261, + 14.432, + 7.212, + 11.637, + 93.344, + 10.77, + 14.911, + 5.267, + 9.912, + 22.704, + 13.538, + 14.245, + 8.025, + 14.629, + 15.235, + 21.854, + 10.938, + 5.378, + 12.868, + 13.979, + 15.115, + 25.482, + 11.97, + 14.829, + 17.395, + 19.641, + 15.176, + 6.497, + 19.209, + 23.733, + 19.434, + 51.677, + 7.163, + 11.975, + 85.693, + 16.627, + 73.733, + 6.441, + 88.891, + 12.329, + 10.103, + 11.719, + 9.802, + 77.7, + 15.105, + 13.566, + 9.243, + 52.177, + 28.234, + 70.903, + 17.482, + 11.323, + 6.585, + 11.481, + 10.353, + 12.379, + 12.849, + 6.854, + 23.18, + 8.777, + 19.921, + 14.47, + 6.931, + 13.864, + 14.359, + 12.932, + 14.728, + 13.807, + 11.954, + 15.194, + 15.6, + 17.452, + 5.205, + 18.958, + 20.184, + 13.313, + 14.27, + 6.653, + 13.021, + 12.131, + 15.837, + 18.858, + 6.392, + 9.825, + 11.058, + 15.174, + 13.029, + 7.936, + 16.828, + 14.783, + 14.895, + 18.473, + 8.991, + 18.7, + 15.576, + 16.972, + 14.353, + 7.265, + 11.035, + 21.503, + 15.766, + 15.843, + 10.009, + 13.302, + 14.818, + 15.732, + 13.864, + 7.439, + 15.714, + 16.528, + 17.075, + 14.507, + 9.349, + 15.819, + 16.321, + 11.85, + 14.947, + 7.934, + 17.79, + 13.018, + 12.761, + 16.89, + 8.814, + 15.512, + 13.021, + 15.841, + 16.522, + 5.542, + 21.539, + 12.336, + 21.078, + 12.971, + 7.682, + 9.334, + 9.525, + 18.318, + 20.962, + 6.562, + 15.291, + 15.321, + 16.223, + 22.362, + 8.855, + 20.698, + 12.939, + 18.482, + 10.99, + 8.393, + 14.934, + 19.82, + 14.203, + 15.194, + 7.159, + 14.256, + 27.579, + 14.478, + 15.737, + 7.695, + 15.539, + 16.285, + 14.984, + 18.181, + 6.816, + 13.896, + 27.952, + 11.738, + 9.733, + 6.747, + 12.071, + 8.392, + 66.998, + 12.7, + 5.033, + 74.212, + 8.785, + 9.162, + 8.399, + 63.295, + 15.249, + 13.383, + 73.832, + 12.563, + 7.976, + 13.995, + 83.009, + 10.588, + 12.035, + 54.44, + 12.609, + 14.699, + 16.115, + 19.97, + 11.398, + 10.663, + 19.061, + 21.702, + 12.494, + 5.895, + 13.547, + 13.418, + 13.35, + 12.332, + 6.997, + 12.235, + 15.895, + 13.014, + 19.887, + 6.527, + 9.733, + 9.155, + 13.174, + 18.363, + 6.969, + 12.079, + 13.859, + 12.875, + 12.605, + 4.269, + 8.127, + 20.282, + 12.742, + 80.332, + 5.407, + 10.388, + 12.07, + 7.972, + 7.977, + 70.383, + 12.86, + 78.528, + 14.75, + 12.654, + 59.444, + 9.858, + 12.538, + 83.021, + 18.005, + 5.719, + 11.099, + 15.94, + 10.398, + 14.575, + 10.492, + 12.773, + 14.101, + 17.009, + 70.264, + 4.967, + 12.624, + 13.616, + 11.378, + 17.181, + 7.014, + 13.262, + 16.016, + 13.573, + 17.27, + 6.963, + 13.398, + 12.693, + 12.852, + 17.515, + 7.276, + 10.337, + 11.444, + 14.363, + 14.389, + 6.892, + 12.31, + 14.594, + 15.883, + 17.346, + 7.485, + 11.5, + 20.509, + 10.344, + 16.512, + 7.89, + 11.189, + 15.039, + 13.519, + 14.204, + 7.5, + 12.514, + 15.522, + 27.531, + 18.176, + 14.055, + 15.9, + 12.377, + 14.724, + 13.858, + 9.442, + 10.963, + 13.104, + 12.027, + 15.009, + 9.298, + 17.351, + 14.614, + 11.022, + 11.563, + 6.364, + 9.229, + 17.776, + 12.609, + 12.783, + 6.659, + 17.083, + 18.627, + 28.939, + 19.283, + 13.35, + 23.399, + 80.39, + 10.485, + 10.877, + 8.482, + 13.141, + 78.209, + 96.279, + 17.714, + 63.406, + 12.105, + 23.538, + 14.578, + 71.859, + 88.159, + 11.137, + 11.61, + 10.274, + 7.772, + 59.761, + 8.528, + 14.366, + 71.233, + 21.824, + 6.43, + 12.705, + 63.346, + 14.84, + 18.212, + 7.571, + 7.601, + 48.739, + 11.186, + 13.638, + 7.636, + 118.387, + 62.426, + 91.196, + 92.972, + 7.643, + 21.126, + 13.808, + 76.56, + 12.567, + 4.505, + 17.23, + 61.144, + 13.724, + 15.686, + 67.237, + 21.967, + 13.206, + 12.76, + 12.897, + 9.547, + 7.824, + 20.857, + 11.079, + 15.951, + 10.395, + 15.385, + 18.298, + 18.64, + 16.336, + 6.823, + 10.83, + 12.455, + 20.621, + 13.254, + 8.321, + 18.631, + 18.946, + 29.442, + 18.284, + 8.529, + 10.486, + 16.191, + 20.552, + 15.818, + 9.126, + 27.601, + 9.729, + 17.02, + 12.888, + 9.583, + 15.503, + 17.608, + 19.259, + 10.681, + 8.328, + 17.29, + 15.862, + 17.315, + 23.816, + 9.078, + 14.327, + 13.987, + 12.567, + 13.563, + 6.509, + 15.871, + 15.779, + 16.049, + 16.676, + 11.342, + 10.081, + 11.037, + 11.839, + 14.132, + 7.737, + 12.273, + 14.842, + 11.338, + 13.884, + 7.661, + 15.516, + 53.922, + 13.467, + 16.441, + 67.84, + 12.406, + 12.465, + 8.335, + 88.69, + 79.835, + 11.501, + 97.757, + 115.346, + 79.261, + 4.692, + 15.859, + 20.81, + 14.334, + 23.221, + 23.873, + 16.277, + 6.255, + 19.934, + 34.865, + 12.85, + 24.917, + 9.191, + 47.252, + 29.037, + 83.545, + 19.047, + 17.336, + 53.89, + 12.323, + 20.687, + 15.752, + 147.663, + 8.038, + 10.504, + 9.774, + 91.743, + 15.336, + 14.578, + 46.113, + 33.773, + 36.624, + 48.342, + 8.921, + 14.261, + 17.22, + 14.984, + 14.775, + 10.174, + 13.219, + 10.509, + 20.557, + 15.336, + 8.506, + 11.509, + 14.544, + 16.377, + 15.18, + 9.313, + 13.891, + 10.61, + 17.481, + 13.351, + 7.063, + 9.662, + 22.339, + 16.17, + 14.427, + 7.981, + 16.344, + 11.415, + 12.444, + 11.622, + 7.113, + 13.821, + 13.122, + 13.971, + 17.164, + 12.412, + 23.504, + 16.882, + 26.509, + 13.517, + 6.371, + 9.772, + 12.213, + 12.0, + 84.236, + 5.932, + 13.2, + 30.044, + 149.706, + 20.728, + 7.042, + 19.287, + 73.344, + 76.462, + 5.612, + 31.919, + 19.796, + 38.069, + 21.541, + 15.098, + 16.503, + 7.142, + 20.387, + 14.421, + 15.387, + 12.271, + 18.308, + 7.699, + 14.151, + 13.044, + 23.078, + 8.275, + 23.644, + 15.749, + 11.575, + 18.056, + 16.424, + 14.861, + 7.699, + 17.157, + 13.538, + 14.991, + 17.787, + 8.285, + 24.908, + 12.6, + 11.369, + 6.591, + 11.556, + 14.156, + 15.761, + 28.534, + 7.342, + 15.187, + 23.537, + 16.667, + 16.825, + 8.041, + 26.617, + 23.302, + 19.016, + 16.704, + 10.859, + 16.181, + 14.785, + 18.567, + 14.754, + 7.41, + 16.879, + 12.261, + 16.657, + 15.396, + 11.668, + 15.667, + 15.547, + 14.428, + 18.99, + 12.175, + 16.165, + 16.108, + 16.782, + 17.872, + 11.207, + 30.218, + 20.263, + 36.465, + 24.152, + 6.555, + 23.072, + 17.643, + 35.356, + 9.475, + 10.06, + 11.16, + 18.125, + 16.511, + 29.559, + 7.133, + 9.495, + 9.01, + 17.202, + 17.897, + 12.366, + 13.645, + 12.462, + 7.764, + 12.275, + 16.643, + 43.383, + 11.879, + 16.805, + 80.633, + 18.133, + 9.045, + 70.255, + 14.88, + 12.343, + 9.226, + 45.083, + 14.511, + 15.493, + 13.674, + 9.186, + 4.521, + 142.536, + 23.608, + 23.277, + 13.311, + 6.746, + 126.55, + 11.851, + 12.073, + 23.499, + 5.453, + 10.935, + 18.287, + 10.917, + 37.169, + 6.275, + 23.448, + 14.355, + 16.128, + 20.38, + 11.509, + 21.831, + 15.493, + 59.375, + 17.258, + 182.709, + 13.936, + 69.691, + 9.918, + 16.178, + 11.66, + 11.033, + 8.894, + 9.066, + 9.208, + 4.069, + 45.6, + 15.024, + 11.346, + 12.445, + 11.038, + 19.998, + 18.509, + 20.335, + 13.419, + 8.242, + 17.643, + 27.916, + 16.442, + 16.226, + 6.361, + 16.472, + 12.921, + 10.32, + 13.914, + 6.721, + 13.894, + 14.116, + 12.796, + 13.173, + 18.111, + 15.985, + 14.749, + 17.625, + 16.192, + 6.62, + 8.549, + 8.071, + 52.267, + 6.23, + 13.069, + 70.995, + 7.446, + 16.436, + 13.003, + 60.559, + 6.353, + 10.534, + 12.922, + 18.109, + 10.041, + 19.585, + 31.745, + 25.693, + 15.017, + 6.575, + 52.824, + 17.313, + 19.174, + 13.046, + 6.493, + 9.952, + 18.551, + 9.078, + 14.692, + 6.93, + 13.534, + 16.231, + 12.104, + 15.824, + 5.26, + 16.794, + 14.712, + 14.016, + 14.787, + 9.385, + 10.729, + 19.347, + 13.091, + 15.79, + 6.126, + 11.639, + 13.383, + 15.127, + 12.367, + 5.56, + 18.604, + 14.14, + 12.315, + 12.357, + 6.254, + 12.068, + 13.517, + 14.705, + 14.014, + 6.722, + 9.989, + 10.13, + 14.378, + 14.778, + 7.153, + 12.698, + 11.958, + 13.943, + 15.084, + 5.146, + 7.836, + 7.749, + 14.878, + 12.072, + 6.414, + 11.709, + 16.175, + 13.242, + 14.197, + 5.632, + 14.311, + 16.438, + 11.177, + 11.632, + 7.992, + 8.297, + 8.619, + 11.891, + 14.874, + 7.4, + 10.822, + 13.796, + 19.039, + 14.29, + 7.252, + 13.817, + 15.873, + 17.39, + 15.074, + 5.7, + 10.345, + 17.409, + 9.518, + 13.092, + 4.425, + 63.663, + 10.903, + 13.085, + 82.56, + 11.437, + 8.132, + 8.057, + 69.323, + 12.857, + 73.529, + 16.285, + 18.916, + 74.23, + 14.339, + 5.178, + 73.45, + 18.992, + 12.087, + 18.439, + 8.39, + 16.782, + 21.238, + 14.344, + 16.473, + 64.038, + 11.263, + 11.021, + 83.23, + 28.146, + 6.411, + 8.883, + 76.489, + 14.959, + 11.5, + 50.627, + 9.549, + 13.745, + 12.999, + 78.595, + 5.612, + 12.022, + 16.819, + 15.606, + 39.772, + 5.105, + 19.383, + 15.116, + 17.573, + 17.204, + 8.049, + 12.326, + 15.95, + 16.149, + 14.174, + 9.568, + 61.223, + 11.875, + 11.466, + 13.746, + 6.812, + 9.114, + 14.424, + 17.052, + 14.832, + 8.089, + 18.753, + 11.62, + 48.812, + 10.288, + 6.336, + 11.84, + 14.432, + 67.591, + 11.761, + 8.016, + 67.299, + 9.832, + 11.234, + 12.48, + 4.487, + 78.995, + 13.127, + 14.992, + 8.331, + 4.324, + 57.289, + 16.779, + 11.901, + 12.056, + 44.147, + 11.362, + 14.054, + 15.712, + 15.907, + 5.059, + 54.354, + 8.538, + 10.427, + 79.956, + 7.629, + 71.99, + 12.537, + 13.119, + 14.645, + 65.606, + 103.469, + 14.66, + 8.694, + 167.8, + 10.757, + 14.172, + 15.576, + 8.606, + 72.021, + 8.943, + 19.325, + 71.94, + 12.429, + 82.691, + 12.647, + 87.581, + 17.284, + 10.671, + 10.243, + 61.837, + 13.799, + 22.889, + 11.133, + 8.669, + 31.555, + 19.134, + 16.551, + 77.775, + 16.271, + 7.933, + 82.562, + 14.688, + 15.477, + 13.857, + 35.78, + 14.269, + 13.809, + 15.337, + 17.03, + 6.622, + 25.696, + 23.266, + 13.069, + 17.05, + 8.688, + 18.287, + 11.178, + 22.723, + 10.091, + 8.035, + 11.986, + 12.089, + 10.509, + 11.141, + 9.376, + 18.039, + 13.437, + 14.456, + 18.454, + 5.617, + 10.585, + 15.903, + 14.833, + 12.46, + 5.747, + 13.693, + 12.313, + 15.495, + 12.537, + 6.071, + 14.759, + 11.888, + 10.685, + 15.294, + 6.224, + 8.906, + 11.43, + 11.299, + 15.389, + 8.144, + 14.891, + 15.309, + 14.144, + 16.406, + 6.038, + 20.233, + 16.523, + 16.263, + 13.081, + 7.756, + 18.21, + 16.782, + 16.382, + 18.2, + 7.411, + 13.734, + 13.431, + 15.55, + 15.195, + 8.384, + 10.174, + 14.086, + 13.946, + 13.888, + 8.491, + 14.576, + 21.196, + 18.241, + 15.535, + 10.13, + 15.191, + 22.107, + 10.435, + 11.948, + 8.025, + 9.749, + 10.992, + 13.981, + 13.751, + 6.565, + 10.985, + 13.893, + 13.237, + 13.879, + 5.942, + 10.456, + 10.307, + 11.534, + 16.817, + 8.581, + 13.323, + 10.469, + 12.583, + 13.848, + 7.969, + 16.596, + 11.931, + 11.556, + 13.329, + 6.624, + 11.627, + 13.717, + 11.46, + 13.585, + 5.686, + 14.367, + 10.057, + 14.612, + 13.43, + 5.724, + 15.17, + 11.313, + 10.735, + 15.717, + 6.937, + 11.845, + 11.758, + 11.774, + 12.011, + 11.511, + 15.404, + 11.895, + 11.877, + 18.402, + 43.021, + 9.245, + 16.305, + 12.753, + 75.655, + 8.007, + 82.85, + 14.498, + 71.628, + 10.704, + 9.323, + 15.883, + 80.711, + 9.684, + 15.995, + 7.764, + 68.194, + 14.762, + 18.738, + 14.294, + 5.746, + 49.876, + 12.908, + 86.037, + 21.374, + 8.603, + 10.825, + 13.683, + 11.508, + 28.206, + 6.568, + 11.562, + 16.732, + 13.3, + 14.156, + 6.331, + 13.552, + 13.255, + 11.456, + 12.727, + 6.77, + 15.532, + 13.068, + 13.103, + 14.915, + 5.849, + 12.645, + 12.712, + 13.124, + 17.992, + 6.973, + 10.902, + 15.339, + 10.232, + 12.61, + 5.958, + 12.835, + 11.125, + 12.209, + 19.185, + 7.499, + 16.967, + 14.767, + 11.729, + 12.965, + 5.979, + 11.488, + 12.008, + 12.004, + 17.934, + 7.299, + 12.185, + 20.492, + 16.5, + 17.977, + 13.014, + 19.061, + 14.564, + 15.851, + 12.319, + 5.652, + 13.794, + 16.344, + 9.841, + 17.239, + 7.997, + 13.479, + 11.005, + 11.346, + 19.008, + 5.741, + 9.897, + 8.156, + 54.347, + 16.862, + 9.305, + 24.109, + 15.742, + 11.192, + 11.535, + 5.66, + 11.679, + 8.674, + 19.087, + 23.718, + 8.349, + 11.176, + 10.529, + 12.885, + 11.446, + 5.489, + 13.402, + 13.352, + 10.804, + 12.172, + 4.6, + 11.567, + 16.78, + 7.921, + 19.142, + 7.066, + 20.941, + 10.852, + 9.978, + 14.296, + 6.318, + 14.54, + 18.032, + 11.019, + 14.67, + 4.463, + 8.645, + 10.024, + 11.614, + 14.231, + 6.587, + 13.485, + 14.897, + 12.36, + 14.327, + 4.132, + 7.916, + 9.595, + 11.832, + 12.353, + 5.404, + 12.479, + 8.221, + 11.039, + 12.323, + 6.931, + 13.503, + 10.366, + 11.737, + 12.335, + 7.471, + 13.229, + 14.529, + 14.482, + 14.187, + 5.981, + 12.615, + 13.324, + 9.441, + 14.796, + 8.186, + 16.003, + 14.246, + 13.835, + 16.24, + 47.899, + 7.729, + 13.807, + 12.063, + 81.715, + 8.731, + 80.675, + 20.577, + 14.158, + 75.686, + 8.317, + 17.915, + 9.563, + 58.851, + 24.378, + 66.895, + 9.31, + 24.628, + 11.414, + 53.157, + 8.564, + 13.0, + 16.0, + 13.691, + 21.213, + 9.656, + 13.104, + 14.383, + 12.839, + 16.383, + 7.902, + 13.859, + 13.582, + 13.582, + 20.926, + 8.693, + 15.224, + 17.433, + 16.979, + 19.101, + 10.197, + 15.314, + 18.713, + 16.277, + 13.672, + 6.186, + 24.378, + 11.031, + 76.548, + 14.742, + 7.296, + 12.943, + 23.956, + 15.931, + 57.756, + 7.46, + 80.232, + 18.23, + 13.641, + 76.234, + 4.617, + 12.376, + 74.96, + 8.375, + 16.041, + 6.782, + 14.547, + 9.976, + 58.092, + 14.69, + 8.807, + 74.576, + 14.48, + 17.229, + 68.235, + 6.78, + 80.982, + 18.95, + 15.405, + 72.21, + 5.243, + 10.745, + 8.252, + 11.556, + 11.994, + 38.763, + 8.643, + 19.983, + 17.743, + 14.09, + 7.99, + 11.302, + 14.962, + 10.852, + 10.232, + 5.594, + 13.542, + 12.318, + 17.158, + 13.859, + 7.204, + 20.256, + 13.806, + 15.719, + 18.653, + 6.117, + 10.448, + 11.4, + 24.003, + 13.864, + 9.68, + 17.764, + 15.188, + 16.151, + 16.278, + 10.454, + 21.176, + 12.306, + 18.174, + 15.811, + 4.766, + 25.868, + 23.56, + 81.651, + 17.646, + 5.456, + 72.379, + 22.096, + 15.616, + 11.805, + 53.197, + 12.995, + 11.413, + 12.724, + 58.658, + 17.452, + 81.644, + 12.186, + 10.201, + 14.583, + 9.414, + 9.638, + 15.885, + 15.677, + 10.564, + 6.974, + 13.525, + 15.668, + 13.262, + 14.713, + 5.406, + 13.964, + 19.383, + 10.362, + 12.749, + 11.442, + 9.543, + 14.152, + 9.683, + 10.869, + 7.052, + 16.496, + 14.454, + 12.83, + 18.261, + 11.252, + 8.419, + 12.758, + 10.649, + 11.111, + 10.155, + 15.614, + 12.212, + 11.595, + 16.218, + 7.434, + 13.01, + 11.823, + 13.197, + 13.442, + 6.497, + 11.622, + 20.766, + 13.111, + 13.714, + 8.872, + 11.512, + 12.272, + 15.08, + 15.74, + 10.17, + 10.589, + 13.637, + 12.43, + 15.87, + 7.344, + 12.312, + 12.016, + 15.143, + 16.148, + 6.233, + 16.465, + 13.881, + 12.923, + 13.204, + 7.715, + 13.098, + 11.935, + 12.414, + 13.883, + 8.048, + 14.32, + 15.825, + 13.014, + 14.036, + 6.747, + 10.825, + 12.582, + 12.762, + 11.112, + 6.859, + 18.374, + 12.043, + 13.014, + 12.991, + 5.597, + 12.184, + 12.896, + 10.512, + 21.784, + 5.143, + 14.053, + 15.592, + 15.21, + 15.09, + 8.466, + 14.317, + 11.482, + 13.586, + 14.007, + 5.916, + 11.88, + 12.952, + 14.102, + 12.997, + 5.975, + 14.543, + 17.327, + 15.728, + 15.558, + 7.36, + 13.211, + 12.114, + 10.396, + 13.139, + 8.785, + 18.044, + 21.063, + 14.876, + 13.217, + 7.732, + 16.205, + 12.294, + 14.132, + 16.084, + 8.069, + 13.72, + 15.781, + 13.596, + 14.53, + 10.202, + 13.9, + 11.284, + 12.026, + 14.116, + 7.223, + 13.628, + 12.031, + 11.952, + 15.552, + 8.047, + 17.569, + 13.155, + 13.308, + 13.404, + 7.928, + 11.504, + 12.668, + 13.723, + 17.207, + 5.64, + 16.668, + 18.581, + 25.401, + 30.735, + 11.079, + 76.385, + 17.801, + 18.591, + 65.665, + 11.884, + 19.153, + 78.22, + 14.901, + 71.311, + 10.494, + 16.258, + 13.46, + 73.257, + 35.564, + 48.488, + 23.539, + 15.092, + 11.893, + 20.224, + 6.682, + 8.372, + 7.528, + 19.418, + 16.673, + 7.7, + 13.599, + 14.632, + 17.028, + 18.542, + 7.429, + 15.507, + 14.973, + 18.027, + 13.24, + 5.613, + 18.595, + 15.546, + 11.33, + 18.526, + 9.53, + 14.118, + 14.13, + 13.883, + 14.736, + 8.142, + 17.5, + 14.224, + 12.487, + 15.354, + 6.755, + 10.68, + 17.387, + 13.186, + 9.548, + 4.872, + 8.908, + 12.645, + 12.552, + 18.023, + 6.905, + 13.794, + 11.715, + 14.521, + 19.478, + 7.378, + 9.734, + 17.823, + 13.583, + 12.287, + 10.524, + 18.886, + 11.989, + 11.353, + 81.063, + 4.952, + 9.422, + 8.931, + 10.603, + 14.614, + 7.423, + 13.338, + 12.576, + 14.573, + 14.281, + 8.64, + 14.865, + 10.644, + 20.144, + 14.503, + 9.386, + 18.696, + 17.03, + 15.865, + 17.491, + 7.502, + 16.838, + 16.588, + 14.67, + 13.512, + 4.422, + 7.916, + 17.515, + 11.009, + 13.136, + 5.694, + 13.046, + 14.98, + 11.574, + 17.744, + 7.895, + 18.641, + 16.326, + 14.785, + 14.562, + 5.903, + 17.882, + 16.033, + 11.571, + 16.303, + 8.127, + 21.35, + 13.753, + 11.529, + 25.462, + 6.02, + 11.909, + 10.289, + 25.797, + 30.622, + 6.448, + 12.352, + 12.029, + 30.886, + 16.317, + 7.878, + 14.405, + 20.124, + 16.075, + 16.885, + 9.962, + 19.728, + 17.682, + 12.388, + 16.921, + 9.747, + 16.205, + 9.453, + 9.502, + 12.365, + 7.423, + 10.353, + 21.573, + 18.179, + 20.055, + 8.605, + 12.018, + 9.765, + 33.016, + 17.141, + 7.355, + 50.604, + 12.008, + 19.326, + 100.131, + 14.455, + 50.067, + 13.5, + 19.973, + 92.051, + 11.843, + 82.291, + 19.031, + 165.191, + 24.98, + 5.662, + 84.137, + 18.951, + 15.364, + 68.726, + 5.266, + 78.014, + 26.985, + 12.816, + 72.616, + 5.407, + 12.174, + 87.629, + 18.736, + 19.301, + 8.592, + 51.754, + 16.438, + 15.582, + 15.526, + 4.411, + 7.1, + 44.288, + 15.845, + 12.79, + 54.812, + 10.258, + 16.847, + 13.235, + 88.282, + 70.436, + 10.977, + 12.954, + 16.818, + 12.779, + 60.609, + 11.945, + 13.591, + 12.061, + 15.437, + 6.331, + 33.143, + 15.646, + 20.192, + 22.188, + 7.555, + 11.895, + 7.805, + 7.687, + 9.865, + 10.594, + 18.273, + 17.012, + 14.022, + 14.2, + 5.48, + 75.913, + 11.726, + 20.933, + 22.502, + 9.273, + 15.016, + 15.219, + 13.715, + 15.71, + 6.191, + 9.538, + 24.866, + 15.042, + 15.066, + 9.122, + 22.274, + 25.944, + 19.343, + 11.183, + 9.311, + 18.149, + 13.314, + 10.565, + 24.502, + 13.239, + 15.163, + 17.49, + 47.863, + 15.669, + 6.184, + 8.025, + 65.086, + 13.061, + 84.563, + 6.418, + 10.131, + 88.567, + 20.639, + 11.943, + 45.503, + 22.067, + 14.126, + 84.229, + 77.661, + 4.545, + 8.453, + 9.195, + 12.487, + 16.47, + 5.627, + 11.063, + 11.525, + 13.282, + 15.377, + 8.352, + 8.671, + 15.303, + 17.564, + 18.191, + 7.172, + 19.8, + 10.65, + 14.988, + 13.73, + 8.327, + 12.355, + 13.955, + 10.912, + 18.971, + 5.997, + 25.465, + 17.661, + 15.965, + 18.419, + 7.361, + 11.808, + 12.397, + 15.897, + 22.891, + 7.513, + 13.887, + 12.421, + 14.95, + 19.129, + 7.655, + 14.39, + 15.97, + 14.476, + 16.622, + 9.804, + 18.328, + 14.318, + 15.133, + 20.013, + 5.954, + 13.507, + 14.964, + 13.417, + 15.518, + 6.045, + 15.051, + 15.763, + 14.363, + 15.292, + 13.195, + 20.576, + 18.232, + 15.297, + 17.748, + 6.421, + 13.279, + 12.62, + 18.104, + 14.822, + 8.95, + 13.91, + 15.363, + 15.738, + 18.15, + 9.068, + 14.905, + 11.551, + 21.501, + 15.874, + 5.22, + 9.488, + 10.558, + 14.736, + 21.009, + 7.288, + 29.191, + 12.411, + 12.249, + 17.607, + 8.119, + 27.088, + 16.307, + 14.287, + 16.702, + 6.951, + 10.172, + 15.73, + 13.282, + 20.657, + 7.96, + 14.168, + 16.079, + 12.897, + 29.405, + 9.322, + 18.467, + 16.198, + 13.131, + 16.067, + 7.685 + ], + "action_events": [ + { + "start_seconds": 2.002455766996718, + "end_seconds": 13.51132709599915, + "result": null + } + ], + "verification": { + "confirmed_messages": 4254, + "stored_messages": 4254, + "status": "PASS" + }, + "attachment_original_verified": true, + "monitor": [ + { + "seconds": 4.378, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "72.95%", + "Container": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "ID": "9425d69912fc", + "MemPerc": "18.13%", + "MemUsage": "92.8MiB / 512MiB", + "Name": "mtusi2026lab4-api-1", + "NetIO": "1.53MB / 1.61MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "94.08%", + "Container": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "ID": "48fedaf342e1", + "MemPerc": "19.79%", + "MemUsage": "101.3MiB / 512MiB", + "Name": "mtusi2026lab4-api-2", + "NetIO": "11.9MB / 12.5MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.78%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.05%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "653kB / 1.05MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.37%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "9.95%", + "MemUsage": "50.93MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "8.94MB / 22.6MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.67%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "16.31%", + "MemUsage": "20.88MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "24.8MB / 25.1MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "76.8kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.97%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "93.35%", + "MemUsage": "478MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "1.17MB / 1.4MB", + "PIDs": "22" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 9 + }, + { + "state": "idle in transaction", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 1 + } + ], + "pools": [ + { + "instance": "9425d69912fc", + "stats": { + "connections_num": 4, + "connections_ms": 42, + "requests_num": 955, + "usage_ms": 3956, + "requests_queued": 9, + "requests_wait_ms": 10, + "pool_min": 1, + "pool_max": 8, + "pool_size": 4, + "pool_available": 3, + "requests_waiting": 0 + } + }, + { + "instance": "48fedaf342e1", + "stats": { + "connections_num": 6, + "connections_ms": 70, + "requests_num": 5604, + "usage_ms": 35427, + "requests_queued": 16, + "requests_wait_ms": 16, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 3, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 9.945, + "error": "EOF\n" + }, + { + "seconds": 17.38, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "65.82%", + "Container": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "ID": "b89ca7b5c7e8", + "MemPerc": "16.37%", + "MemUsage": "83.84MiB / 512MiB", + "Name": "mtusi2026lab4-api-3", + "NetIO": "3.46MB / 3.63MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "80.32%", + "Container": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039", + "ID": "26996fc7b6ac", + "MemPerc": "14.40%", + "MemUsage": "73.72MiB / 512MiB", + "Name": "mtusi2026lab4-api-4", + "NetIO": "1.2MB / 1.27MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.03%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.05%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "988kB / 1.61MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "26.69%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "10.24%", + "MemUsage": "52.41MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "13MB / 32.5MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.42%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "16.82%", + "MemUsage": "21.53MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "35.3MB / 35.7MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "77.8kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.95%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "90.54%", + "MemUsage": "463.5MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "1.33MB / 1.62MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "active", + "wait_event_type": "IO", + "wait_event": "WALSync", + "count": 1 + }, + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 7 + }, + { + "state": "idle in transaction", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 1 + }, + { + "state": "idle in transaction", + "wait_event_type": null, + "wait_event": null, + "count": 1 + } + ], + "pools": [ + { + "instance": "b89ca7b5c7e8", + "stats": { + "connections_num": 6, + "connections_ms": 77, + "requests_num": 1948, + "usage_ms": 6782, + "requests_queued": 11, + "requests_wait_ms": 36, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 5, + "requests_waiting": 0 + } + }, + { + "instance": "26996fc7b6ac", + "stats": { + "connections_num": 4, + "connections_ms": 45, + "requests_num": 1235, + "usage_ms": 4496, + "requests_queued": 8, + "requests_wait_ms": 8, + "pool_min": 1, + "pool_max": 8, + "pool_size": 4, + "pool_available": 2, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 24.581, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "89.31%", + "Container": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "ID": "b89ca7b5c7e8", + "MemPerc": "15.54%", + "MemUsage": "79.57MiB / 512MiB", + "Name": "mtusi2026lab4-api-3", + "NetIO": "8.15MB / 8.55MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "97.92%", + "Container": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039", + "ID": "26996fc7b6ac", + "MemPerc": "17.61%", + "MemUsage": "90.15MiB / 512MiB", + "Name": "mtusi2026lab4-api-4", + "NetIO": "6.94MB / 7.32MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.33%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.05%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "1.22MB / 2MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "36.95%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "10.57%", + "MemUsage": "54.14MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "16MB / 40.6MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.14%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "17.11%", + "MemUsage": "21.9MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "43.7MB / 44.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.14%", + "MemUsage": "48.99MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "77.8kB / 57.3kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.38%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "90.49%", + "MemUsage": "463.3MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "1.35MB / 1.65MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "active", + "wait_event_type": "IO", + "wait_event": "WALSync", + "count": 1 + }, + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 9 + } + ], + "pools": [ + { + "instance": "b89ca7b5c7e8", + "stats": { + "connections_num": 6, + "connections_ms": 77, + "requests_num": 4155, + "usage_ms": 12025, + "requests_queued": 11, + "requests_wait_ms": 36, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 5, + "requests_waiting": 0 + } + }, + { + "instance": "26996fc7b6ac", + "stats": { + "connections_num": 4, + "connections_ms": 45, + "requests_num": 3391, + "usage_ms": 10859, + "requests_queued": 8, + "requests_wait_ms": 8, + "pool_min": 1, + "pool_max": 8, + "pool_size": 4, + "pool_available": 0, + "requests_waiting": 0 + } + } + ] + }, + { + "seconds": 32.072, + "docker": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "83.23%", + "Container": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "ID": "b89ca7b5c7e8", + "MemPerc": "15.99%", + "MemUsage": "81.86MiB / 512MiB", + "Name": "mtusi2026lab4-api-3", + "NetIO": "13.2MB / 13.9MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "92.35%", + "Container": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039", + "ID": "26996fc7b6ac", + "MemPerc": "16.60%", + "MemUsage": "84.99MiB / 512MiB", + "Name": "mtusi2026lab4-api-4", + "NetIO": "11.9MB / 12.6MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.96%", + "Container": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "ID": "1f03107c8f59", + "MemPerc": "8.05%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab4-cache-1", + "NetIO": "1.45MB / 2.39MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "34.44%", + "Container": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "ID": "11b3a463b436", + "MemPerc": "10.78%", + "MemUsage": "55.18MiB / 512MiB", + "Name": "mtusi2026lab4-db-1", + "NetIO": "18.6MB / 47.5MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.08%", + "Container": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "ID": "a01916435f0d", + "MemPerc": "17.58%", + "MemUsage": "22.5MiB / 128MiB", + "Name": "mtusi2026lab4-lb-1", + "NetIO": "52.3MB / 53MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "ID": "f223fdbf3dfa", + "MemPerc": "19.24%", + "MemUsage": "49.25MiB / 256MiB", + "Name": "mtusi2026lab4-reconciler-1", + "NetIO": "113kB / 84.6kB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.87%", + "Container": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "ID": "68e73eb491e5", + "MemPerc": "92.68%", + "MemUsage": "474.5MiB / 512MiB", + "Name": "mtusi2026lab4-s3-1", + "NetIO": "1.36MB / 1.68MB", + "PIDs": "21" + } + ], + "postgres": [ + { + "state": "idle", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 8 + }, + { + "state": "idle", + "wait_event_type": null, + "wait_event": null, + "count": 1 + }, + { + "state": "idle in transaction", + "wait_event_type": "Client", + "wait_event": "ClientRead", + "count": 1 + } + ], + "pools": [ + { + "instance": "b89ca7b5c7e8", + "stats": { + "connections_num": 6, + "connections_ms": 77, + "requests_num": 6332, + "usage_ms": 18288, + "requests_queued": 11, + "requests_wait_ms": 36, + "pool_min": 1, + "pool_max": 8, + "pool_size": 6, + "pool_available": 3, + "requests_waiting": 0 + } + }, + { + "instance": "26996fc7b6ac", + "stats": { + "connections_num": 5, + "connections_ms": 61, + "requests_num": 5604, + "usage_ms": 16842, + "requests_queued": 9, + "requests_wait_ms": 10, + "pool_min": 1, + "pool_max": 8, + "pool_size": 5, + "pool_available": 4, + "requests_waiting": 0 + } + } + ] + } + ] +} diff --git a/evidence/rolling_containers.json b/evidence/rolling_containers.json new file mode 100644 index 0000000..dee1c44 --- /dev/null +++ b/evidence/rolling_containers.json @@ -0,0 +1,296 @@ +{ + "before": [ + { + "id": "9425d69912fc65c8e0c839a22f2250d11c966cc9a5a6b158ff0b72cd1207ed63", + "name": "mtusi2026lab4-api-1", + "hostname": "9425d69912fc", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:57.313322452Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:55.368313114Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:55.368377904Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ], + "steps": [ + [ + { + "id": "48fedaf342e1b5c9fd46e9d14dede59b9a8ff60cc61a6c5129b20d08e3cdf96f", + "name": "mtusi2026lab4-api-2", + "hostname": "48fedaf342e1", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:26:35.987840207Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "name": "mtusi2026lab4-api-3", + "hostname": "b89ca7b5c7e8", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:27:04.675168495Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + } + ], + [ + { + "id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "name": "mtusi2026lab4-api-3", + "hostname": "b89ca7b5c7e8", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:27:04.675168495Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039", + "name": "mtusi2026lab4-api-4", + "hostname": "26996fc7b6ac", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:27:09.192493456Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + } + ] + ], + "after": [ + { + "id": "b89ca7b5c7e80b8b91c632d0ac672480b6ddb5b59090c959bd80bd8241c4c9b2", + "name": "mtusi2026lab4-api-3", + "hostname": "b89ca7b5c7e8", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:27:04.675168495Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "26996fc7b6ac739b370ba13088d1d744853056c674d3ccc8500890b5b6e5e039", + "name": "mtusi2026lab4-api-4", + "hostname": "26996fc7b6ac", + "service": "api", + "running": true, + "started_at": "2026-10-16T19:27:09.192493456Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "1f03107c8f595f6bb5167160e8cc761cf1dbc170dd4da459514de051a09d54c2", + "name": "mtusi2026lab4-cache-1", + "hostname": "1f03107c8f59", + "service": "cache", + "running": true, + "started_at": "2026-10-16T19:24:14.51497638Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "6379/tcp": null + } + }, + { + "id": "11b3a463b4369d8c2dd70e941cb5bf4b77d9e1a892f7116d1aa33df9eb644973", + "name": "mtusi2026lab4-db-1", + "hostname": "11b3a463b436", + "service": "db", + "running": true, + "started_at": "2026-10-16T19:24:14.515575034Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": { + "5432/tcp": null + } + }, + { + "id": "a01916435f0d842324e3ee4886bbb637ffc28b4812396a92a0531ad984e9f3bc", + "name": "mtusi2026lab4-lb-1", + "hostname": "a01916435f0d", + "service": "lb", + "running": true, + "started_at": "2026-10-16T19:24:25.265114396Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + } + }, + { + "id": "425a7c4ea23278bddb2a2efebcb982dbabe0fc2ae5551081b6ed52df424269f9", + "name": "mtusi2026lab4-migrate-1", + "hostname": "425a7c4ea232", + "service": "migrate", + "running": false, + "started_at": "2026-10-16T19:26:55.368313114Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + }, + { + "id": "f223fdbf3dfa28020141c81a158c4dabba51c24c698b075fb0dbfcb16fb56bdc", + "name": "mtusi2026lab4-reconciler-1", + "hostname": "f223fdbf3dfa", + "service": "reconciler", + "running": true, + "started_at": "2026-10-16T19:24:26.040659771Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 268435456, + "published_ports": { + "8080/tcp": null + } + }, + { + "id": "68e73eb491e5fdf91ec1e52bb70a8e6fe9bad45f5b2166a031e9d2f5d27fa045", + "name": "mtusi2026lab4-s3-1", + "hostname": "68e73eb491e5", + "service": "s3", + "running": true, + "started_at": "2026-10-16T19:24:14.517842259Z", + "cpu_limit": 1.0, + "memory_limit_bytes": 536870912, + "published_ports": {} + }, + { + "id": "34e29dc315a1d55ca0ba174708e0d27adc811424d866935d7f281e01a31f6a3c", + "name": "mtusi2026lab4-s3-bootstrap-1", + "hostname": "34e29dc315a1", + "service": "s3-bootstrap", + "running": false, + "started_at": "2026-10-16T19:26:55.368377904Z", + "cpu_limit": 0.5, + "memory_limit_bytes": 134217728, + "published_ports": {} + } + ] +} diff --git a/evidence/s3_write_failure.json b/evidence/s3_write_failure.json new file mode 100644 index 0000000..034f60c --- /dev/null +++ b/evidence/s3_write_failure.json @@ -0,0 +1,8 @@ +{ + "status": "PASS", + "http_status": 503, + "metadata": "uploading", + "object": "absent", + "chat_id": "4849767c-8c80-4469-ba2f-7b439782abc4", + "attachment_id": "d2befe47-84d5-4cd2-888a-a2d6eaaf9b6c" +} diff --git a/evidence/upload_interrupted.json b/evidence/upload_interrupted.json new file mode 100644 index 0000000..5785648 --- /dev/null +++ b/evidence/upload_interrupted.json @@ -0,0 +1,7 @@ +{ + "status": "PASS", + "sent_bytes": 16, + "declared_bytes": 4096, + "metadata": 0, + "objects": 0 +} diff --git a/evidence/upload_memory.json b/evidence/upload_memory.json new file mode 100644 index 0000000..7f0b2f4 --- /dev/null +++ b/evidence/upload_memory.json @@ -0,0 +1,443 @@ +{ + "runs": [ + { + "dimension": 256, + "size_bytes": 197173, + "concurrent_uploads": 10, + "repetition": 1, + "elapsed_seconds": 0.2, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 50.00050475719273, + "successful_latency_ms": { + "p50": 177.823, + "p95": 199.251, + "p99": 199.251 + }, + "latency_samples_ms": [ + 199.251, + 176.934, + 178.783, + 176.489, + 179.393, + 177.823, + 195.595, + 165.586, + 180.372, + 167.019 + ], + "before": [ + { + "container": "81a289979a98a3cfe8ed140fe6e9d44dc9e4c1d97238731bf0e792ce6d8dab24", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 133656576, + "container_peak_bytes": 127946752, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "7de9c777c1bdd46d4a945010a99f20e9e57cc1bc0d77e575d7d2668f8ff51f8a", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 112111616, + "container_peak_bytes": 106348544, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "81a289979a98a3cfe8ed140fe6e9d44dc9e4c1d97238731bf0e792ce6d8dab24", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 136138752, + "container_peak_bytes": 130760704, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "7de9c777c1bdd46d4a945010a99f20e9e57cc1bc0d77e575d7d2668f8ff51f8a", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 116420608, + "container_peak_bytes": 111153152, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + }, + { + "dimension": 256, + "size_bytes": 197173, + "concurrent_uploads": 10, + "repetition": 2, + "elapsed_seconds": 0.145, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 69.09566154263844, + "successful_latency_ms": { + "p50": 118.179, + "p95": 143.348, + "p99": 143.348 + }, + "latency_samples_ms": [ + 143.348, + 114.799, + 118.179, + 115.119, + 115.609, + 124.187, + 115.907, + 126.252, + 141.024, + 119.354 + ], + "before": [ + { + "container": "2ca6d753d6a7308991464a7b96b23507766831bbc80a10c716f419956b2fef9d", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 111525888, + "container_peak_bytes": 105918464, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "e1ccb5bd30f8e154ae47a83d163a92b29a3723440dac86dd1c3f01ba9f75df0f", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 114376704, + "container_peak_bytes": 108515328, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "2ca6d753d6a7308991464a7b96b23507766831bbc80a10c716f419956b2fef9d", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 116072448, + "container_peak_bytes": 110772224, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "e1ccb5bd30f8e154ae47a83d163a92b29a3723440dac86dd1c3f01ba9f75df0f", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 116572160, + "container_peak_bytes": 122118144, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + }, + { + "dimension": 256, + "size_bytes": 197173, + "concurrent_uploads": 10, + "repetition": 3, + "elapsed_seconds": 0.151, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 66.42146959922258, + "successful_latency_ms": { + "p50": 124.273, + "p95": 149.041, + "p99": 149.041 + }, + "latency_samples_ms": [ + 125.702, + 118.463, + 121.036, + 135.132, + 142.124, + 149.041, + 87.277, + 138.344, + 86.5, + 124.273 + ], + "before": [ + { + "container": "e08871d00731b8a760906ea6fb8be77d81107a9e1dfbb3cd33ea760ace92c1de", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 111759360, + "container_peak_bytes": 105930752, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "328d37747428a59aae4130e2dafe03f39b5fff9dde561fe7f4751256a75073e0", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 111529984, + "container_peak_bytes": 105852928, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "e08871d00731b8a760906ea6fb8be77d81107a9e1dfbb3cd33ea760ace92c1de", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 116510720, + "container_peak_bytes": 111316992, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "328d37747428a59aae4130e2dafe03f39b5fff9dde561fe7f4751256a75073e0", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 116559872, + "container_peak_bytes": 111366144, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + }, + { + "dimension": 1024, + "size_bytes": 3151221, + "concurrent_uploads": 10, + "repetition": 1, + "elapsed_seconds": 0.403, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 24.810339037866473, + "successful_latency_ms": { + "p50": 339.61, + "p95": 402.166, + "p99": 402.166 + }, + "latency_samples_ms": [ + 210.903, + 265.17, + 375.366, + 402.166, + 400.501, + 266.088, + 339.61, + 374.81, + 210.982, + 342.662 + ], + "before": [ + { + "container": "be5e8022f8422cd906a56e1bbaf9de4301a02e6485be30fe98217355b251257b", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 114208768, + "container_peak_bytes": 118149120, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "d75aff6f8b9488e9f7c2b33e0653c495efec77601af7589249fa0cb77ac88026", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 113872896, + "container_peak_bytes": 108003328, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "be5e8022f8422cd906a56e1bbaf9de4301a02e6485be30fe98217355b251257b", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 133783552, + "container_peak_bytes": 127967232, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "d75aff6f8b9488e9f7c2b33e0653c495efec77601af7589249fa0cb77ac88026", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 131272704, + "container_peak_bytes": 126140416, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + }, + { + "dimension": 1024, + "size_bytes": 3151221, + "concurrent_uploads": 10, + "repetition": 2, + "elapsed_seconds": 0.431, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 23.199689911744827, + "successful_latency_ms": { + "p50": 319.572, + "p95": 429.619, + "p99": 429.619 + }, + "latency_samples_ms": [ + 319.572, + 294.825, + 374.821, + 429.619, + 377.772, + 296.36, + 215.412, + 423.914, + 231.341, + 398.349 + ], + "before": [ + { + "container": "33c568324dc6c6ba022889f9dc1fe9cf8ed4a064b0aba2a0b14b8a8e3ae2d213", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 113623040, + "container_peak_bytes": 108158976, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "b81f0a676713c37f0481157f4552020b9eff8823a183451c7303a5334aa6c6d0", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 111923200, + "container_peak_bytes": 106242048, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "33c568324dc6c6ba022889f9dc1fe9cf8ed4a064b0aba2a0b14b8a8e3ae2d213", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 131477504, + "container_peak_bytes": 134270976, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "b81f0a676713c37f0481157f4552020b9eff8823a183451c7303a5334aa6c6d0", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 131133440, + "container_peak_bytes": 137510912, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + }, + { + "dimension": 1024, + "size_bytes": 3151221, + "concurrent_uploads": 10, + "repetition": 3, + "elapsed_seconds": 0.446, + "successful_uploads": 10, + "failed_attempts": 0, + "burst_successful_rps": 22.432872376785337, + "successful_latency_ms": { + "p50": 354.611, + "p95": 444.786, + "p99": 444.786 + }, + "latency_samples_ms": [ + 364.278, + 444.786, + 229.569, + 233.565, + 387.33, + 419.847, + 271.838, + 354.611, + 277.369, + 398.801 + ], + "before": [ + { + "container": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 112033792, + "container_peak_bytes": 106389504, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 111742976, + "container_peak_bytes": 106307584, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "after": [ + { + "container": "d66f39ddc1de8d5837cb3caac22913896c76bc3867adfe84f073a726fa57a975", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 131276800, + "container_peak_bytes": 125980672, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + }, + { + "container": "7d0a6457eb210eded96d3c7a281467f43f6f971be053ab493a006b99f148daaa", + "limits": { + "cpu": 1.0, + "ram": 536870912 + }, + "process_peak_rss_bytes": 130502656, + "container_peak_bytes": 135319552, + "container_peak_source": "/sys/fs/cgroup/memory/memory.max_usage_in_bytes" + } + ], + "observed_instances": 2, + "original_checksums_verified": true + } + ], + "measurement": "process high-water RSS and cgroup peak if available" +} diff --git a/evidence/upload_memory.png b/evidence/upload_memory.png new file mode 100644 index 0000000000000000000000000000000000000000..d6f04d6f490bb27047a98713436aa23da1fae17b GIT binary patch literal 27582 zcmdSB2T;}NwlqgD22@+ILK#~MolvYvjC?eT_AmGCi^C zywN!;Q;RdAf;$AoHmx$TwzjmA5E455&j$o8&YcwE@v1YyMP^#cYg=(}EL}kV|8_FC z6t^`A3R#b~)T~Xx(fb!c1N@>5o6BPF3G4bWQ8s zb=4`a|Cn@0NUl}RPd(y~zIAd(zDjxRp0s*oY_h1(>f5`0XF6mCC&e%M_$8p;cgm~V zQYU%+g>z=$?i|NU#YM4?bGWo zO-+v)8je-O>K<*%xp7-K$9}1+OMv%At+uhj`ibhP{2VW8zrAUj#PK43+nJhh<3hjM zGU06&bC!5HqSZi!? z!tv~BHtp{Z$X#1>D=sc>bZkr`Qk`FFu*T|0y4}<49X>+(@2B%#xPNW&#{BwveQ9ZF zfg_3GdKbRQaZ6ZlD750M-}C(37X=!$h8GtF4n1_caN)uo(UZzaCMCO9ZaeknNwdE5 zkerJ2P{^z$n@qCe;!k~yr>A%yryqQxW28Gv$iFbOr>95EGa1)$Je{28H14rnv^?dk z!I!+-`_G&?Gcd9%?R=*IK87*RU-sy1Q%;0t%#nOw(Xi~f2XAuicqr{O9B$oO`n|XJ zHr_$u&V~n-dX5Kf-MTfeJ>5x-Rp>7sWsrL_KUgL!Jw3he&05cJ{VeBH>sHlBtB} z%Go^&*C`zw8}7%75VW_qkC44MsqNi^OPaAdzRl^1K7yvBKfbknc>0LPsqfI#>C+3U ztB+2dy`;QMMJA@+d1Q08)dVhUR-2l1?n^!&FE8(6Aw8Xm@xiRc3m4vQtikk0ovwa1 zYv#;bVPRpIlYV+;y!Q&}3wDwETYi(hGKZt#qUw`lyVmB-Ss}Ke_N?Pz*H)GXL)50N zTr5{=;DfhY?khm;cl0 zW^!_J-mSInV^%^!LZVK?*6uAzJfimB4b?*Ae;>J^udnYIj7QO?H8J$`>C@0X?(_J~ zitvq@GiE$-(Tml2zQ}6Y+!cll-&C9S(4iyIho8QZo;aUe)Lj`De9Q3EHvgHkW{Grl zbq&NB`3UNIFnT)@iv7-aRxM}d6c!dvnL2g3xVZSj-+wK=eBj%+CQq5rq2U_Ap^+c&%~N&1x0gIt^bu?E%U>N~{d^dOZYR{&wt}~M+PwuQQW4&Ezk>1#olBeJP*?!lqUHqBK+x!bXc%&o> z&3R;;FIz3iHMH`Ma7YAFMsYyBRA#7az>~!dnB(nk61S}l&7A0wY zL?nKV$UEAZ7oI;q+jNDgpdhkR{7jD*M$@iwqb~|J&i6_z~O zuU5|Dn&IT+go*ZPEqUaA)x$$9r)ExpL5{~$_rGlGox}^xncYcmJVqAJm^^8c65hsF z!Y1;$^H{2I&OHp;J*=}8%$)4(>{E5=LpO_pxpJhWqzX?4I~gOu;+X;@Z1N=$#|Gkh zwe+35&AL83H7LBl>(blM1O7_8cmILQgoH4xzP?-CTV)=%+`rJcrna{5?&hPag|+GS z3G@OiB|E#6A*>+};T)Z`^My*jBCn@Ro96f8g{=F<9!&JWs*iE{GFG4dn#rCg?F@$` z`_{)kbPKMm4;)_8ka{JLiL9IOuy$miGht$SqRYg% zZ;-h4*MlRo%8CMas`cwXKlj@0^5x~sP%Pqz;oIhgfjg6TST!p}j&tuk{~UT(p>s{5 zUj646_Ymc*`r9h3<9g2vWsSz)GEBAY4vG}APs%*}^u#(rK|zg)hjQ0Dl4^NoxD2d<(3Dc3;EktBub^SBNoHkCix?kjt=!6i8svSuBcA8w_=w1Z8cX-+%G5hF-y}Tb0`!UP_^)MX~@+` zt+=k9l#3THvJC6xmT`0~#d3J@0O^*=TCvrPVPVm&RQUXSr=L{F)>9w06rR1YVk?Wm z$H!+{aCh^(T=keETMLIcrTV-WkE>+YK5t&K`G{d;-AIi5tu^!ZbaV=6#RfE6t*(sK z-Pqb3DoGy%;ArV-8fpplVQ@=1lsBIJHb^iiMlQzWT+YInV=Iz z*dllEs}=FoTR`Iu)<&R&jeuI$k+Fo ziJ6{fd?W=kniRyd_NxSJztxt4^<1i^BK6qKZHl9RR#sMeMut(sN9)e2t-7h!{1pOq zF}Zmi^*4^RtWqm6iG48y7EK8is}5{?=Di!f7}Wk)t`WSNzn6gNU5(Ib}xg3+kqZ zo0L3Ey)YEtT<9N$2obgW@~qbJqZ}MJf%qFN+TM@!B9VGGr+m&-eXLMh@~!tv!FvAA zWQN27F0NCYJCn1Fx-^pqjLvIp7VQc(Zk^D2`>Cm^jO+GC$MUXa|R|tG>ZxJ_ge)-f?OjDFq zOATklB8D||{m(iyfUbSehNwUT$C>ksx5g}&Ho$GCFr8ajzB6E^MzZp{%@Vxu$ zkF93qUX0D!@r>Gu@_L|bHDF?I28MW3;aPwvd}rZZ0MEKZx~{TQMO#4$HdZ z!&9-`nu0y=y7vOA^VQB-ae=U!wl145!jCG&`1$$ODw9vVzBXs1rlv+6s9C9^ zKh9;WKw5k#D6_hgQqfy%pi>qqnRh?Ez8?1cmfH8hfldL~%&d7~YwQ06KjLy1@ zqx!&Vp)J6b%B6)PgIrUy5>6JkRa<0b$jBMr$7UTUJT^Sg8D(A>`*`R0zzv%&jqe%Z zhYlSYv<~=OY5hK0dtUYz#Mt0Qfk11kzGG)063u^3}m5N#h5!m!I%wKfkHh6OuE;!*gwZ{MEXG-b+^aNV@?e$q~<3l=Wq ztJQgFUYGuSpsRMHt*xz7&&%n)u~7=%0%n!5fs6>-?z$DC#vV>hk5T)zBcLbOPv36W z)39h~2P)UYsEShUzCA=>GSrt#M+OO14vfsXu`(>px47Zur7r-laW3_lPOOl>{@QyW zP(lMqQ%UrLGghoa$J5-ttcfw9*MrtTLG;Dr*;kmu6S;aEvY|$=vpIj`nCN~#-7;+O z*84p<;-{+&&o#d_<@H&+e;ylow*Xpf`}Xadt;W$4CmxRNs?Otec6MI3VZ)mbr?#!l z!)u24C1YCD?#uo*!ewD$(bU~-F;P=cP@v?$t*~*&?OfCbYYJ0QM4K~wcU_wHxb6Cl z8(Xp94jf8AbpEHHQ13;;yYtDWx4(aDU0vNi7C$OaUlAjPE1aRlUypaSO8q%@p0LV} zbDy6CiynP;YTe1AfSiJYjjxmDE*D|EPX9c;tbY7h?@?~)3m>E>u!Q-CWA(GnTw5x1 zrkfMx2ud5@(2LR3*m&!c;}MoURyr5YAP}>{*#p1*_S-gFIdo)$6cCUO!d1yU7#_&bzB}%HJ8L5Y;#{ z&9++;lc@nLqMPNc`!V4pQc6-^XL7w+M54T0qzv(Hh`qv{iWRB`}-^K zjlSMunJDZMhp!LZ7xD56adL8cZ$7FSmpL4nIn?a0-s>l3`VjR(l-%_tvu4dI#meaG z$>ubF;rbixbf<)+TBfLR$47fr=B^Nv$GhAL4i0|!@Zsk#Umm0Kjo9zWgBW<%Y4M;l z@bQDaIt^o8H#k!rK_mXNA<8Vua$$rA&BLre%Nib(B*P+Gjj(j%MH za%bS~SpQp(z2LZf(M{7t zj#9U@d_tp#sx)J4pt>sM?4z+d#r?e)nUzxg*MniOttCnzufTu0HgZL-Ny_jHv%zkMP-?E zx>|Gf>eWAa(|Nh;P-0-mx7BB531~!y0HMDrC{VmSd#PreUWWCLo<{59r1PCekpGmh zjXS8KYk4qFw-|^jDgd@V2uH-QJ~hZS z1By8EOLTQ$G>Z6EFYpLAM^BwOHp}X5{z=d+W82j$2 zyEr!Rc-OD4s$96SCRr?EsvQP(p|6Yn*AEBLK%r7FM+WtOp~%G4KhC?oK5Ru3h@%NB9i0GBnrb($ z-nik-_<{_p)>50{n4CS3s5y`3zCrKFZaaU#GFle1mkO#6egCr6qE2$?$B&Q|E#2L5 zNTSTrZ%w(}JR6^9Kn^I{xQy=GwcujUwhYI?_jmZ!)T0jzZ`MmcjN04u=4KSi463Ok zDE*`SI-gfvT|NGRJ@0m2^JiEHTQDp2^Sl9aH|wUVKX~w9c@Ys>c*U(Ji?|8{c9hjQ z4|Hhvch~E)`hYBV#u)$n_QZz+Mqu#bUS67BEt3Th$A};X?-|H1El}2#5IZ_v#(?ps z>vQuYAX!FgMBN?hg~V~--j-d!bn3|0i+Olr19n)Mfxx+T?V4(q^VlLDo-Vhj$+(NF z;R#wZUqKZ#d!ku|0Kn7U ztg!)qj8rHJ%2sbJy+eTaU6o(x!CyGhH8F@9U>-!bZj+R(;hE9B4|P})R!CY9^alCr14eI{(iG?tk4~GQjB%)Yqn%BLyWT^*k1zM!X@@u^!vH|KrKvR*+Z} zn^B!tfQ1|y9+pVkM1%iCGHlu_6r6jzGX{eQ@LD(m`1eD6Fa=rBhQ;Y2Jdg>nX7=)r zsaSH}l9B;I8tkxJ8ZdCT1d!MS5H1xQ7I|4Gj&M76t5BAz>X(#iy6o{2R+f z_aTZ^wgkJxPMSK)B>T$T;qL`Pff`Hb2^xfyk6u}c0tUQ4)y4!bS&O*2V?gw^BahrE zG%jnahyn^R3oZkN=Dp$2Zke&Z+ay$+^k&BxuZ3DOumht-jPgHZ-={dSXJV||g;KgZ6_F64s=`?d{SlHe`zYg2wi@%cZ25e*GLbN8dT z-=4sre}ADLuwur^xAz__7c~wsOa6Gubbv;VLnqtx;ubM6b!Y?H?$haCYS!XLYxm#W z-u{mM>N?_jz=;0v;^Fwyi~C*If}ML zRY~R*(b}SPx0Q=^hr}aU0y9ryXh1~=^ ze~cU(fy7kx)WH1;C$FVV$Oj z^=e^O@LdhC3L4MP8C|}7nIGt*@EfjG@wMcUH!wqtewH4p25$!a0kt=(LH}3T*^9w8 z1Y3RkBu5HlDr5_KDg;6W7M9;5tX0#${`{RZReZL)^$2@7mj^7F(Ow?0*1vF_zJC3p z;Iw>*8aDfzfbsd6E4EwSmztv>e#r>XrVqP@lsSj7{>o0LUy^t=t*}CWt>6pauhNYC zzINYFFEs`__|n=M zffc_&T3U~YnB5iZZ(Gc88=jbJ$#r=bGas25x)f3)A2e4*77)}!VEV9~LeIxAw1a7e z^o^90%8mtJkuADRok2)-sqrnpe)Z}t#>m*sn~=zK!50xyf}qo3RqF2}<#?VM@BYT) zj-YPU@e$-BjTFm9C2qLECOYM#^!DW^WVAomkt3m|r!pa)C~Mj-Ogu z)-$hNzpiMdbm~+zgP#V!nNi$3I9P0{ybncZ=M$+I?>+2?TdF+U4T%}qsuU^aj<|)M zRPS3+VtG<63f>AJg|b4Lay@r;cmiS|SN-wFA03TFP@hp&6P~%rD-(;n^w`hmLOR4O ziHLLT%Q`Aot2bjg`w1Id0dM~R0IdxZu7M}4cy4u)-~iTkkwicDM!obDD8bf9hJoA9 zuF;r4WyWA)#U3bOE16@-snD6gsq$em0o&frM74&En{oEYs=mHHb^mRr#URH7N;@3^ zUFQwbFdX5A!><;Y?5*lG0x*d zCtv60MuCBmyhLH=#pFCbPbyXq0)q8}b!-DZS|MWiCpef0$YxQ%QUcoXA=r*x*HlZ( z5%-y;UaR+5H@x`Gab&o^ee3y-!{D(a0do19vE%oQkB|S2kxeaR90#I@2fKxs{5*#2 zG|jodN5;LU(Vh1_7)I2!=Lsd5LAgMj4nqd{oEda-d~_&MGsbVM2|N zmD|eA<1#`4&%!SZeJ3TG$V^ZV8-#@q{r>y!v*yfEI(zePqyoCi47&x=LmXJNaJz4x zhQTaQ(juT-+0OQ#jJ$45^N={#R?dGQNK*Uux&x(n2)@T-mV;?Fn5feqo~+gwi8&20 zqgWztR=xp>2=^Hn>MLprLRDh2d2wv5{v4I@SRo6}1S}$9?%5daTrb#8<-;?PpJW4m5UZjK0=~Od+3q%_;apuNdMtk zTC-6=Htpi*5w|up>5Ud_w742Pn`8VcPe}00qPLTkbyq7XDV1Ygib1O$8|^Jlb?A?T zBt%35d!1J$Mo3D=8{S$#&#jmC|wNbNEH1_UwW3gC?P9x_x>u2hy$c!aQ zobR|-VMNdNLZ)=`;u&eyt*lSISaXJl_~=11gcM2>cVJr-Xc3yBwphO)vIT>brO|Y`W{DH0mLs z^#d8@gQ|k^_#WKiR>TYBK~vzQYk&S}`3qh-^D$nIPvOo%6BCojOb|g>0ifg^+u)H= z@DYr8A9K_kFb$=6Y+_p6VQCk2LxUq1=?RKl+3SmXd(ri4V&VOa;L2uRWr z9Z&Dd;evsUR*zskJ5w8pIPjsG3{3_(bMW9>P2Rm|Oow37{=l8*+%GRVfJl_HjC?6@ zNqI?6j9Ovmx7R$kA{P(&YRrlv)WQ;>x&i*=YqRH^WBgNN^yFkv7o@xnGEwlPU2*#=@4`D7m>CJ{iKdzTedce5BE2Bq66b6c-htz0KwnNwR7WGH_ z=DIg=ViI$CU35I92JVR#uxId1-o=Bd&s9GFYbf2|7C#ekJBqz07*t0 zMwzHl{-VNQ8GS4!t&5YUmS;LS)(Nr;Nx@Hvi(g?t!y%gv%r!b=v{P}*NnPo33d(onh{1z;6Ck|>YtuZA3<*6#$4nJ7^YVT!3VP#ME*QQ!Oe z%3)%747vaHKZBE)>;Z7E2hpW6YrH>B-3ieU>!8f4SSrjSYwUep9m>@tvkC?BDgo>S zYOJ7p$|~!umOT5(4b0EyCr+U9SRvwdc5CJIuP@oG{&4S11lpJXzK5(`Xn-4*C?EYv1WJD3Ns--d+oc0>d~;L*{m|7#GAz?bEWtw<#6fUh{lvHy zR!RQPK>1=pKzUfF40 zV{2I$c1&biTO3NDgnWgvzX{8)_&${EhfwEYs3b>0OH#{SLET6wJRft{^YKN1?V_Cv zbY6jyHc-Yz587QNtRhJ#UeD5qP<{R7XZ_@RT=eGMjIPP3pHTLRjoc769jHFFjc?sL ziAT|5_>F8B5`b!OrG`*yuN%Z;hQsnmnjej;Lw_5SMRFf<3{_DTfGC}VibeN?_jFb# zYk}PC!#Cb!H`Sf9#q{x!f|ME7R3mhJa`Q=>nH&v^_Fvgn+_qr+?vLjj4Z&Y`4f{5{ zm25!#BJF793|=w7#|k7)jdZ(ji;f>;L70j-8DzgN*lEDmd1B1gaTPe@5JoE~Pv%p5 zWvt+G7>tOfMJA@Or(?@T5?)L!2qb>%_YVSJL6}o(0*>WpN?iEy1l}7?FnZzkz2#X_ zCP0VHMS)R}h<}nFl3wxFYQEF+)_O`lgwSh_{q84ef9_f)*qybx_EUkN~wIyRt&yyE20v zLISoJswRjSB&9q=Y2te|xKBBS8D?7e&U!bxzCOaFdNdi;mM$gF@UDY9&x$>i%pBvU9E zMAgy?TFcwI_hN4z-eoxJlDRYb99af>Vq{vs`$tHd{SNKeQ#VP$mmRzWgJq&n6 zm|z~rN!^d^+!knCcL){j;>C-C9#;sTaN|Mbqk5z2xm6fywJ_A65qMzjb3aRn<=YHA&c411-{}4QyiV%K z_VLNI5B#>BCLiu&%m>WX;MBk}nS*wU-Oz5adg$pwQ>nYJDqc+s$vnA$V|?b#Ij4gv z_(nH>v-<6YOv$A_(U-pVFN`+2Efv%uqn&CZ`KZYvj*!961Zs`cc^@dPUT4V+MMNE7fU5dg66Ze46TCEk zzv*=fD76hPAXz^Mi-xeiy?*-vxfyKD7SYnC2(5UPH&Ouk^p)~_SCcSTv1cJ z><3V3PUAx@RIJD9Wkg~|;r!DE$V&7aLNN^W$cB)t1{;RD8a?k@ZBL*=54lTh#If(| z19&VovHJw0*~l-t^`e7WDkd9W}4&46rT(uF+o9Pn9#4?xN)Gw8MPe@{j!^n zB)(G=$_Rm=k{Zcie^ksTX8t@L$-_iGk!S|ulF1@v8UDX9SU*H^qA~lwWA6kooSpgg#`DuQcq(+nli^%#@v+NOuJshrmhaZjZot+rtG6+XHzsx68 zbclw>M_T;FE#BjM>Zn8(HQ^3X3T$kxXr;YslYVA8y&RItMhS@{czr^DScIw`J1rYr zY3=^{2J=79-5im?cP2Z62s{c=HjHZV!RN2n_6=mvrTAZ<$hgRb}DmCm!Dvbx=q6zj_e zS!CF|x^TT2@G}i5s+VI~StNj?NV>mb+o=t(#%uYoIkCH+N@cg7t`h4!|xACa7pWo7qwO`bYsiV0kf z|6b=C-FTh)Zo$u%XqdMEnXaU98B!aiJdD&i4K@XSOk42iY8@0*1k{X|DFH4>17qjQ zvH5*ho9CP6=e&N+4~cO3O)95nN3yp?%BJ~CV08mdP`33D-iPa{!ZW9%zL*B(un#sZ zMvw%{F-V@KHj8S$}@>JIu2XJNIHumr|*ZRHb!y3cbVKFQu{@C=(t;zYVFL zTEXmU$Z|F&VG(_z=h#_9XSn5w07e84+hGixYF7O$2FME;fj+|yOTdj$V9$y7GQG)< zbxn}g36%1`XX>IQ<}vc%gFldYZ#6G-adDwpLOYD*KeII34Tg*N$o&D0De#;e1~NIf zOO9@G?6lx)H|taTACTWE)%E7I<-p`Sh3h8%|MY0_Dhbj`y<$x6x4Fa)Z74Cwmj z{|xEWS0sA_JrGllNw^c~g6ShW4C$ZvSa~%id6p{v_d@w_GQzHP)ku#ZDP_2 ze`ZlHXy8GB2OP$~r+Vsc<}kc&{U!T(P4R{pF@F@!#@(|4u>jTji*|OW#Q%(Ln|!A2 z2GX6&wf&$)M*m$rO$d+%Gxd3b3F<*Gcdhy7=*?b~>a3V?n~*!kUJLj-;S(F=a~K_3 zMGiMyKNR~&o<#`^HlT!`+f?v#sq8Yy8DLCQa9s^BKG&qt#YKwK#89z|-8?qpNpZyz zhTRQ2HXO32B8!k8avhQi_l|RWtCKA%VLl+!17SfuJ-t|gx%5<<@{EdClSiG>G$a@` zDBfkDWGE9s+Eqg>S^}C7-U{GwnzIz~+%O!o`Ff;)zw88xH^+wtdkT3)!C3R@WV3<3dpwtlWM#>8Y z5q0Lj7g2_I0`?qLIhkVvN2wwu$b;O?V1oScVJtW3do^1q96a8?*ZoE}mKX?V#JU40 zqg0=r`pD+kciJlBS=6v*w<10|!TcG!iS0x{^4{nM=Z6;xLxL>8A^hNv0VcetL*Vue zL-x{mji7uu^MXAb+12CEF$~|E3!_k6N?RW9L8NDU)xk%goY=5^`(Y6F(h?9Qkg4n<>U7u_bTxZub7u{o~NWbujrZP+?#>dFHL4*aq6$$g1U>ZBOyxBw` z0UkOKMP;z^>e%G(+-CA_6;+u)p^A&A@kIbP@Jpj=NkmJ~15D*dAOrH8P+yrhur%a@ zof&6nKvPyCNsI#j6FU`-{u96LXCFeliiK^E{0&?a;5AYs#hBv3m$#q&w3qB*;NS|% z13r&T!CncDj6)8lP68x=uwu9)GT*b2(BLI(GZZke*Bg4!#z+F2iy~LO$wL@eP>|CN zEQBh0ATg5E_O@9k$OsGLQqF#PGd5P^DFm`&lZLy^3Rw23Z|05 zyn6NOaKj}|sw*fDqYL6br9CJkUs1v!LLFZ6(~>l}|E}ckcbdj<{Vc64 zO;<^xTGX@uvY1{m=iVZxZT<%k_sT&h1o6Fm0!s~Qtf^vnxQOBGSFadM^niIYV60R~ z9nviO@Zqrg;O}UClYIS@z(d2XPw}#L-#;@+j7M^$9!!)hr#Q9YSg8PTeh7Zv1bX){ z9IL7wUt4G8?YxZjseuFq^QuCL3-n&*M3%@hfIX~0xqbW0P=8?eXy?4w)PT$K z{4%O4dt}djlyd7%@4ZQ@1#Vd~W`P$gN8sFwGNBM82v$#9&GQ7(30)>e>SF~81i@PZ zxuPX_{Moyse95)Le_qAfRU}{?WOm`)QDDKqLxSvuAc}jz=NL znSmB8!AKsuQI`ic3m5~uC_X+YmYb56MhSSJ@UTuN$U#l4Lh`qR5mx|}tp%j0fs#NM z{W92c=@QmokMR->3+_M(LrH8G0#hgg6tS<=da&;%R|M)`!@MNApX@%J^PI;K<2uu> zg|?JOs8P4>Y=Lu|)~wl$d{&PBII_qgB@$Q>O7Hmt>zrkF0STRYi0C#*A9opJ?fMg{ zh*TBeX0?3SI-+kgsw#jga%@pgGt@h-o74(Ob|RExDUs3Ys0V^v#_gphMtgmO%HoX_ zyA~5N>q$l+2KwR7z=ogz8)Wxsrqs8;77pYJb4fXB?_qzZYw|X4_TyeZXSOOG z`ZpxpI9LJmz+)2238oUp{{1I|*etNuOi2GYvLKO!E70Znpn(B$312$2$0VcQX`2&*Bo>?3uU~0->-z}pKjz3> z6u&Z1dA1RLue;EDfPxBKb7wW#Dc;q3`{~O);FOWr_Fda@uLCO^YY261kpw&Nu9^&v z5O!~>#SXKsi>-|N`7Gc zny}+kUsaQ86NP{r=H=xT{nKeP{w*LiLdlp5%@m|hBQg_#{AO8phC6+$VUqTO8E6?N zkV7STlx7S|cwf#kY9dmR{N@(q&=wKBfz3s+K^qucz=vB%&l9FFDZtU0EOW08r4J$% zcJsEVOYEzzQOC^byi0^X_9(m)M4(fTzMfsUZM99+Qdgmz62@0%fbC%tQ8Fxn-7kv#rEuI+`UtkyQs!cP;O~E1k zDLvY=hrDJv%P5*5u&NT|gWRcgmvRfoT~rfxQB;mWKzl~563D8pP%fzSOhoycF)==( z{WtZo@9xxBM=4YWI}b-F2W)?Umu^R2*V})|+Qp{DtJ7D){J`$H`C#B_WW!H)>G2$t zjbF$-?6G(u6NE|u>E&+3*`-2y>X2swbl5TD1!@3TDk{mz z+jS2cfsId5mYH8;Q!ukZ!9>G6I^G*R@fAXv^#>Q|dc-jUeIM+u%lNfWKz4~nuS6T( zy8<})-otL}O5zjXRC+*8GC<*2fsMMv`C(kJbksxSZ&~L$qoXTQ_3<_IyCZ2vqozdn zhmBw)yG-c{D46sB?LD-HAmNa&j864nkg!Fy4!A#Z$C{&@j1}PD1L8NeC|D-AF@+9y zIGTJ0<$(`XZ;&lbFliL^jEjRhM|ZO!h+vAfPYF;B{fOca;?;esX8|*aGNrJD8Gq~j zCBz(+2%CrjQyRAO4CIj&fXrmC0hM4r(D9MBB(@bFGB_mg8@lXWCF!Lz{^2SL#xa;2 z0a{8Isz_3FZ9~6LDPllJn`)^wno%=)Ud_f6-T^eYVV%1!&SW0yUMMt4$>fPclqBzM zjwf$-xdZ#T&p`j(w{KtQ5c~%7$bk$4Uu8nEbktJ)3&IumqWgj;M)$kV<1U4{AO;-V z;~ib#%#VPjU_=0dqdB}9Fs3Eo=P2;lZEt_^^F)=_o5DF?0e*rhx;!?4PZ0#EwWet+ znfdm4EPHUKHZ=m}7BQ=S27|EspgZ!FFh=tE-1jI*$3^e#R?lcMu4y=m{Gqr7RSzS8v|681&`E z$C5J z>(|RrbC7WgM!JG!x6M4Dox_0HUYbfm3Yn-dXZ%%7O-+2v^~wPUaTWlh6?3f)zH`xv z6#`5N=h2jGgPfck0aPsTP1MMcJlt@xsZ{m{tVj`HrKw`cC|q{?M6d7XVqNYTeEj$* zX-L_bKYrLb>YtO}^#Agr?VdVok-YFpQkFun1tp%E(Ilko6ijos#L%?WWuyT|)!Daj z6WC@K+ZpJ@5>O5K(?gg9Y_a37rf7K^o%A^S<19E?rYd?FI!|Lwuo$SfXv4F_PFE$W z=fV=2%7{Rz8~gmHs683g)Y^R>UKj<8ix`69;>}0YS$&u`bI=M^u;_!5do*+5x^S59 zVg=fRK%}U!#0=leXLoC6I*&Qr;NGbTNP@?7)bwU6U8aycMsfNMAy;JFqT^X8-{D@1 zrOo>8{w}%>n%@t`WeoTc*9sCN3fm5Ryf5<5BS1I6NAQ1Jq6!F*xvE1KKxb)({}Xw% zgZNOB!tliPvsP;0>Zr9?%iBiZ1B^BdPUMrjyGi>Y9+M1qRC|-tq~Z(hJl8#f)#~dY zD3hi-*2{(+1Kst(pEf^4WGx3dqN|~K1!<70%grGT!RhN9n5JV-g!TB+8iWE)}K z@$$Nlr~vxpq})$(nIG%$AdAd_Xu$~w21PLPnKqA$X#MIDbWKFxzrP9+#~bgT(g`2v zW*(p=^!k(_qlQAG4YjM;8;{bB+Py`M-~LL75-vr|kAt_$6oHsrx@tG_V0g{<5sE|j zPWgr4L`;n!O#+^tQtqWomJm0X5-vbf^h6K^9PA~L@EL;ShLaN`p>yze`q^QB)Lg(R z2qwObDKIK+gpwo}&fWph7@8+YTEkebXqb8?mOF}sM0p4F*su8!=fPm!NvtFO0^LRO zC0Q4S&cSD-jqc$#2nta@ZD9pi{Lx2}BFQEkC~2>XkvVwm*nR9^vNdC^`>QZW z>S)d~qJ-dcr7Q^ljjk@cv^nfoHo1{JiH-xOs23OvCoM&oRzi`CiH0bW{)xutD$jKT ztS&|3-vG0fJPS)WtTEfYogyC)ZGgQ44Wz6D@&YvCIDWOzH`&>H0(aVKUY@m>tdt~8 zqx*-Q#^8PvShebf_P?N9qU;ZhZUG!zWD@~?>F6Y+MbE+O{9_mSTOb|o5(i) z{#|YeI&ES9so`Z~`On9Jg{09P_PB)l?Ce(0{p`xGz{>2qg7d)rsAe{*7AZEp=ruB# zowQ%({IrDq-*Cd(f2FCVM+nT3AI=)W=`Iw=s4tB&GuZ0Bm(zJ+P*_*^oH}tM)g#t` zV@EAPo@WYuv!f{5yYAe-YP0~rz9RI%N^&9~?odfju9<9iPIJV2s#Fli{LS=8hXHMc zLQ@fYTODAWWGz^m_G9m!AG^$6W$S*QF~LF&O7t_S#X(YzVlZ)GBe5-WSaEdVNPlbD z`qu$a9?jr*CPyJ;i lIZnW#u#bIEfm_MR$rT zv^7AU2FUln+xr4OKZAVvJY0d^CPfsV-i}H-#NT4$+3z^bj6+2kH~4;nLh ztZ`D@J5Zl!*4{RkNAg2m(CD&R=#3C-0c8Hg{4XM@^()@W%+9Cvy~-(iey9Oz0;(B7 z<32f_-7&a;eR-MR(D(;N*bSzENZdIB=cqY!CRM8TjSLMjQ-@?n5`Q8t&FOrm5J~4% zmmS%`&fAVJXFZrqH4DgZo4&7ykmCt*Q9FuXG>1O@h2*l`91sEH0v3^Ett zg#(X>voElDoA*!wqPiG#BBm-t6GU^N9bDoUepsRuRb-XJnn_e3)gF}HP?5JH)8J5{ zs~*BI2veGf5@bi7t{E45giz?lPwd#SOyWB<whLkK4;XO39O%RIH?CWVw}^s=*+IXZec_W$JZwnw-}!*Z|4R5`8wUS7oo@O6 z=fij32KU+q%cUBQOZnW{8AC)+vD9Dyh=*Ghk7+_(_e^;;vI?C;Sdx{HA8?l7BP4Ad zw2D1}*8_*(OrK(jzL4WAl!1S&+YB)PterT!3z2M_3Io@Lay$e)ZY!|v9!ZB>GsJJ@u^cdEU%UJ zj*9TOssCelwCrivg$FO>&s{fvd@c?y33HlUbK{{sZ$K8;kCQv0d@~}@Ab?X2T>E;g zev5roJoPYLv}5*x!IgMBzHWgPOP97hk@=U(GRG4~MszU+Y&c<7!;``?%%&30j6cT9lVDvSv^aDFv-ly608FM-w0`0JB(L+nVv2hn7#cn`y!lG!G(1oAe2ggqmi2xTv>m^R< zgHZ7!cOYnGp4xWQj~6~=`N0670!_FW(71308f80tTAC0O69L#AQ}5CtK6)8P0O+;g zDuC=Yk51*p3{ruDo((iHV5G}3f&fPzJJw+>(#chHz$S`FQj;OQqtL7=>&MXpPv94u z%PpafLDzs8bgcfF$A$kPwKXpn+@cnpd4=1AL3H7V9{}Mz_-_lz$<+7Ysc}-}0SFXQ z4t;O>()-XTs0aC73;10Fk~YldunDz-cPOyz!qqCEG!hMiGhs-aM1imYo|eZeRswgx z^w&juiqmGysS%C5K=`nHFBjOH^B=U(RVY@x<4n!=j31w_ zi{yj@n}O$kMy;I~Yf})`XalX*)#v1>;lv7ikGA$M6R`bGVh(*KWEMtcNar2^vGsw( zGyxw$OZghR9Strm>YK%g@WTW^U=ime(i*$LwL=wp1amL|Pz%R1wQ%d{=D_b~rPX*2 znwr_W`xj}IgSOiMDG4v73dM{39>g~D|Iv^7y&tO$o}>u8VkGK0QuSd>BvTk2=LAY~ z6VR1)@d){00Gw=fge>)!bKLy~7RVQn6_|P;#6$+7YXm$&w+u-LSmcE2M}(!cckf@^ zfgA1hK=~A6lvMkm@0E1aXHCS$uKtKp$`qBhA?jvh@B|&5=qe4xF&Fz-Gw06T3c^cK z*a8yFJ{DkH0e7GUxXm2n1Yc-=Ar51(5WdA1yM~XT(``9=jqS0B3*nb2M5~Pl);+U3 z%Mt3kFiQzDpf;M8j(2WqZ8e31y%14C^{pv}h76H!0e;mIT~LqjW1Y3N6(^fmauLY5 z6*x81q3`Q)hA)x1O)|!g7jRsX3eL} zX|Jk@;axsHX9O}9j-&;$Q+I-OWE%@kLn9HVBJ9zX%yl@wx=7?iG0X%B0?d%3cix29kDITQyJs=zH2osC*1lrT6263ZcAj<_B>K zfYkRlYl#j)+b}Dl$*6CId?{(hkbFq|1_5fTCuYfemUA@+Q@lIxb_nqeOK&~5 zBl04MetCUKN&(wT0x{Aimb-VJvEoyCAlA$drjZLhXVehL*?daTxtp zpd*QW__4oT!G`$x7R#sJhc<13&VAVS{HJm91U!s`H;Sw{q|xEr8sfAd$*Q6yPYkVy z)bdyHO;9(L!6cUwnf)+;Un%q92t)D}fs3xP-I<|*!ivm2XqMH$f=3(SZ84mB?ai;Y z5A#!4o&>G;ZDv|6bTza|#Vs1c5_Ld}UO#mnz$HR13nDdPBEmUM~4`W%1#L{Q!YRY2QTV#)irHifKzRJl4K?xMAoh83~` z>Qd_;heOGM16LF^ypFw}wyMC$X3bQuB!WD)b!Xsn(eJ%6?*b@**&yC|NmU z3OyKfs;YHyFRWK9n9aBL^DvVV5`}YMzsi;ufm*G1ZQ$)Gf!khQ$AGph7|XH#aH#oP zxZ6u()@vJU&baMjOrpf`37m9jfjt82dTWq5G=iM6vNFJZQ3z1#V>sAB7!MPQBT6Ic zGS+C$xOEu^=WptOuupso)=vn6PHR%7UO`_CB!cSTv}7E`y2yzwFu}0(!6m^j8j+t^ ztq^}szD!;8B@vB8K4erV#Awr!JuXE70XiPKyw~9=6wxPH)R!bL-feIJE(-<|itTwj zDS#?f5~ce9UR!_qB)~x1S}SwBQOKKk_IR-C3d~nbRRrKtu3jYOHdq2wVI4>U)vZ1_ zP4D?RIf#;)Aguaf!L2zPh~z~`OI4ra3Szl0pSq~_1M1G z3CG!y$sbT%qGJLt(BXoIv$|qK!r|Y7FR>k<#A++n3AO5~oU6UbNDi}+;^u*u>nYnYc3nF0?|Z%0XjHpe3HmeKXY8$G7ULDih8->bgFZi za3Y6kV+x)RCqx~kBFDz@inUd4C&~k6Y51W>r2VC(t!kKuHnrLC{{2U0O08qqS37U; zeQoBf!>G6M1ch}jX_#Gw;febEIyzpMDk)e5H=}WkPUb{vCZ^oqm>Vkn9E{Lj4_!zqywjY9gAKp!&j4PaqZ_eQMdxh65gg3k zlDT9vhZN3zB^LYZE66ZSyn}kE(sDYb1f=_eI>+t|4l?>jLdOGYKMs&@6&UK z`zivvmy)!eW9L?!;<%yK3VA$ii|BuL;70%E!%t+X zEdk#lPL58~DeT2b{{_pMde%abW|e0++AcRxotlGo#ci4tflIS% zpj6ESYx-JyMg@{S{MM_HE7nz{?WJHr+6;&n+9T+_DYGudapt$g^ryEr$IQ+PULUt; zl3?R-^^)HVy`gKOmR-3v52r#@gE9@>ISxA&R3YJrXDdB#-h5Uc9*lNZ^;{-GDJ(aL z%5VhSdPu%0aW>Xe!trn!-B{l1Z~zw@yCx^2kG3kabr+5_xYp}|7@>FRQLfzf@7Rcb zZ{ECFbQ*}X8E1-YgGz;XzN1D^>UkGRw~xG6kH4Le5y-Bt-dS<>w^dP(lOu5A{U+`B zzvM-b5|qAme}zIrCBsY{>(PJjK2{P;_q;g!^f>Bo8e#0Vv12_SbIvGJjr#p`nj zmJs!oKDC0P{`quv{vrYOH}WFzqb4Tj{`J>A#G{=7yen6lceevWMh2_cDx4Cyw8>Dk zIZj`Z<45jFRJCKb=0K{x1=C3OS?p6p>*v(tLC1q408`ARlNvBrbZQA@ViYd@Kqfm! z3wwYf$oj2|hzP<}4K1-9S*C@M1~j7&KR{M0mmUY^DZxoWk5LGdZDb(%$qEA;!=XkN zj}eQ0m zk6{RrC;~%EtpU`ljZ({GMhKNDkPEaTQkqofe4LbG>G{gp6MA=>n+=%N98fw^Z z@BO~te!$*wFmPDf6gZeIQVUFR!N3+Lu)(Vdbb{v5y$^x?(ksAb!|Q@L;Dqm+dOP60 zT+k%5yIiFUu$?%qfFD17G6GIdF9RNi@(g$wi#f11^8a1~tU8TA+4m%HaOEVh zLIw8Z4S~J{mvM%9K-Yolb>Jl;mNsdhl=Xm_{7rwY$9u-l+w1Q9i@pogd1gz-#Y3Pp z2)ZEz)LjHlhRyz!`?uOHKo^?>V+G`* zoV&YBukHc1PlJJH_bO{%RhBbawHCO5g)jRiupgBPJnH!*FiC#=_RS2~-!=ng4inI9 zvgtOUSu?9-jl9 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_order_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +-- Equality по chat_id и range/order по двум полям курсора. +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..d293758 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key varchar(128) NOT NULL, + body_hash char(64) NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, request_key) +); +CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..c35917e --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,41 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + object_key text NOT NULL UNIQUE, + status text NOT NULL CHECK (status IN ('uploading', 'ready', 'missing')), + lease_until timestamptz NOT NULL, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id, created_at, id); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id) +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +CREATE FUNCTION check_message_content() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF length(btrim(NEW.text))=0 AND NOT EXISTS ( + SELECT 1 FROM message_attachments WHERE message_id=NEW.id + ) THEN + RAISE EXCEPTION 'message needs text or attachment' USING ERRCODE='23514'; + END IF; + RETURN NEW; +END; +$$; +-- Проверяем в конце транзакции, когда связи вложений уже вставлены. +CREATE CONSTRAINT TRIGGER message_content +AFTER INSERT OR UPDATE ON messages DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW EXECUTE FUNCTION check_message_content(); +-- Прежние текстовые повторы остаются совместимыми с расширенной схемой Message. +UPDATE message_requests SET response=response || '{"attachment_ids": []}'::jsonb +WHERE NOT response ? 'attachment_ids'; diff --git a/requirements-experiment.txt b/requirements-experiment.txt new file mode 100644 index 0000000..f6b5455 --- /dev/null +++ b/requirements-experiment.txt @@ -0,0 +1,2 @@ +Pillow==12.0.0 +matplotlib==3.9.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4c0ac28 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +idna==3.20 +pydantic==2.11.5 +pydantic_core==2.33.2 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.34.2 +psycopg[binary]==3.2.9 +psycopg-pool==3.3.3 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.40.18 +Pillow==12.0.0 diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..c9b859e --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,156 @@ +"""Один профиль нагрузки для всех серий. Только синтетические данные, без токенов в отчёте.""" +from concurrent.futures import ThreadPoolExecutor +from collections import Counter +import math +from pathlib import Path +import sys +import threading +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, Failure + + +def context(base): + c = Client(base) + alice, bob, _, chat = c.bootstrap() + return dict(actor=alice, reader=bob, chat=chat) + + +def percentile(values, p): + if not values: + return None + return round(sorted(values)[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3) + + +def measure(base, ctx, duration, concurrency, action=None, retry_writes=False): + gate = threading.Event() + started = time.monotonic() + deadline = started + duration + action_events = [] + + def perform_action(): + nonlocal deadline + time.sleep(2) + start = time.monotonic() + try: + info = action() + deadline = max(deadline, time.monotonic() + 5) + action_events.append(dict(start_seconds=start-started, end_seconds=time.monotonic()-started, + result=info)) + except Exception as exc: + action_events.append(dict(start_seconds=start-started, error=str(exc))) + + def worker(worker): + c = Client(base, timeout=3) + gate.wait() + latencies, confirmed, bins = [], [], Counter() + errors, instances, failures, operation_counts = Counter(), Counter(), [], Counter() + step, pending = worker, None + while time.monotonic() < deadline or (action_thread and action_thread.is_alive()): + if time.monotonic() - started > 180: + raise RuntimeError('Infrastructure action exceeded workload time limit') + begin = time.monotonic() + # Один и тот же цикл: 60% POST, 20% история, 20% профиль. + operation = 'write' if pending or step % 5 < 3 else ('history' if step % 5 == 3 else 'profile') + operation_counts[operation] += 1 + try: + if operation == 'write': + key, text = pending or (uuid.uuid4().hex, 'x' * 128) + response = c.message(ctx['actor'], ctx['chat'], text, key=key) + confirmed.append(response['json']) + pending = None + elif operation == 'history': + response = c.call('GET', '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=50', + actor=ctx['reader'], schema='MessagePage') + else: + response = c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User') + elapsed = time.monotonic() - begin + latencies.append(elapsed) + instances[response['headers']['X-Instance-Id']] += 1 + bins[int(time.monotonic() - started)] += 1 + except Failure as exc: + # Сообщения ошибки клиента не содержат credentials/body. + errors[str(exc)] += 1 + failures.append(round(time.monotonic()-started, 3)) + if operation == 'write' and retry_writes: + pending = (key, text) + step += 1 + # Последний timeout мог произойти после COMMIT. Уточняем результат тем же + # ключом вне измеряемого окна, иначе проверка ошибочно сочтёт его дубликатом. + finished = time.monotonic() + resolved = 0 + if pending: + for attempt in range(10): + try: + confirmed.append(c.message(ctx['actor'], ctx['chat'], pending[1], key=pending[0])['json']) + pending = None + resolved = 1 + break + except Failure: + time.sleep(.5) + if pending: + raise RuntimeError('Final uncertain write could not be resolved') + return dict(latencies=latencies, confirmed=confirmed, errors=errors, instances=instances, + failures=failures, bins=bins, operation_counts=operation_counts, + finished=finished, resolved_outside_window=resolved) + + action_thread = threading.Thread(target=perform_action) if action else None + with ThreadPoolExecutor(max_workers=concurrency) as pool: + jobs = [pool.submit(worker, i) for i in range(concurrency)] + started = time.monotonic() + deadline = started + duration + if action_thread: + action_thread.start() + gate.set() + results = [job.result() for job in jobs] + elapsed = max(r['finished'] for r in results) - started + if action_thread: + action_thread.join(timeout=120) + if action_thread.is_alive(): + raise RuntimeError('Infrastructure action did not finish') + if any('error' in e for e in action_events): + raise RuntimeError(str(action_events)) + values = [x for r in results for x in r['latencies']] + errors, instances, bins, operation_counts = Counter(), Counter(), Counter(), Counter() + for r in results: + errors.update(r['errors']); instances.update(r['instances']) + bins.update(r['bins']); operation_counts.update(r['operation_counts']) + successes, failed = len(values), sum(errors.values()) + report = dict(duration_requested=duration, elapsed_seconds=round(elapsed, 3), concurrency=concurrency, + profile='closed-loop: 60% POST text128, 20% first50 history, 20% profile', + retry_writes=retry_writes, successful_requests=successes, failed_attempts=failed, + uncertain_writes_resolved_after_window=sum(r['resolved_outside_window'] for r in results), + successful_rps=round(successes/elapsed, 3), error_ratio=failed/max(1, successes+failed), + successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]}, + instances=dict(instances), errors=dict(errors), operations=dict(operation_counts), + successful_per_second=dict(sorted(bins.items())), + failure_seconds=[x for r in results for x in r['failures']], + latency_samples_ms=[round(x*1000, 3) for x in values], action_events=action_events) + if not successes: + raise RuntimeError('No successful requests') + return report, [m for r in results for m in r['confirmed']] + + +def verify_confirmations(base, ctx, confirmations): + from urllib.parse import quote + c = Client(base) + path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100' + stored = {} + while True: + page = c.call('GET', path, actor=ctx['reader'], schema='MessagePage')['json'] + for message in page['items']: + if message['id'] in stored: + raise AssertionError('Duplicate ID in history') + stored[message['id']] = message + if page['next_cursor'] is None: + break + path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor']) + if not all(stored.get(m['id']) == m for m in confirmations): + raise AssertionError('Confirmed message missing after infrastructure action') + # В fault-сценарии нет seed. Каждый новый ключ должен породить одну строку. + expected = {m['id'] for m in confirmations} + if set(stored) - expected: + raise AssertionError('Retry created an extra message or left an unconfirmed result') + return dict(confirmed_messages=len(expected), stored_messages=len(stored), status='PASS') diff --git a/scripts/bootstrap_s3.sh b/scripts/bootstrap_s3.sh new file mode 100755 index 0000000..d689684 --- /dev/null +++ b/scripts/bootstrap_s3.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu +endpoint="${S3_ENDPOINT:-http://s3:9000}" +bucket="messenger-attachments" +export MC_CONFIG_DIR=/tmp/mc-config +attempt=0 +until mc alias set local "$endpoint" "$(cat /run/secrets/s3_root_user)" \ + "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + test "$attempt" -lt 60 || exit 1 + sleep 1 +done +case "${1:-init}" in + init) + mc mb --ignore-existing "local/$bucket" >/dev/null + mc anonymous set none "local/$bucket" >/dev/null + mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null + mc admin user add local "$(cat /run/secrets/s3_access_key_id)" \ + "$(cat /run/secrets/s3_secret_access_key)" >/dev/null + mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null + echo 'Private bucket and limited API credentials ready' + ;; + deny-put) + mc admin policy create local messenger-api /policies/s3-no-put-policy.json >/dev/null + echo 'Synthetic S3 write failure enabled' + ;; + allow-put) + mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null + echo 'Normal private bucket policy restored' + ;; + *) exit 2 ;; +esac diff --git a/scripts/experiment.py b/scripts/experiment.py new file mode 100644 index 0000000..595269c --- /dev/null +++ b/scripts/experiment.py @@ -0,0 +1,184 @@ +"""Трек 2: реальные отказы, безопасная сверка и стоимость обхода; плюс RAM upload.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import io +import json +import os +from pathlib import Path +import random +import statistics +import sys +import threading +import time + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client +from scripts.stack import command, containers, scale, save +from scripts.benchmark import percentile + + +def api_run(*arguments): + running = [row for row in containers('api') if row['running']] + if not running: + raise RuntimeError('No running API container') + return command(['docker', 'exec', running[0]['id'], 'python', *arguments]) + + +def memory_stats(): + output = [] + for row in containers('api'): + if row['running']: + source = 'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/memory",timeout=5).read().decode())' + values = json.loads(command(['docker', 'exec', row['id'], 'python', '-c', source])) + output.append(dict(container=row['id'], limits=dict(cpu=row['cpu_limit'], ram=row['memory_limit_bytes']), **values)) + return output + + +def file_checks(): + result = json.loads(api_run('student-tests/storage_scenario.py', 'privacy')) + save('private_bucket.json', result) + interrupted = json.loads(api_run('student-tests/storage_scenario.py', 'interrupted')) + save('upload_interrupted.json', interrupted) + print('PASS: private bucket, limited credentials, closed console') + + +def storage_checks(): + active = any(row['running'] for row in containers('reconciler')) + command(['docker', 'compose', 'stop', 'reconciler']) + try: + command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'deny-put']) + try: + failure = json.loads(api_run('student-tests/storage_scenario.py', 's3-failure')) + save('s3_write_failure.json', failure) + finally: + command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'allow-put']) + result = json.loads(api_run('student-tests/storage_scenario.py', 'reconcile', '--failed-chat-id', failure['chat_id'])) + result['s3_write_failure'] = failure + save('reconcile_faults.json', result) + print('PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection') + finally: + if active: + command(['docker', 'compose', 'start', 'reconciler']) + + +def memory_experiment(base, repeats): + results = [] + for dimension in [256, 1024]: + rng = random.Random(41) + image = Image.frombytes('RGB', (dimension, dimension), rng.randbytes(dimension*dimension*3)) + output = io.BytesIO(); image.save(output, format='PNG'); image.close() + data = output.getvalue() + digest = hashlib.sha256(data).hexdigest() + for repeat in range(repeats): + print(f'Upload memory: {dimension}x{dimension}, repeat {repeat+1}/{repeats}', flush=True) + scale(2, 1, '512m', 8) # Новые процессы: high-water mark относится к этому случаю. + c = Client(base, timeout=60) + alice, bob, _, chat = c.bootstrap() + before = memory_stats() + gate = threading.Barrier(11) + def upload(_): + local = Client(base, timeout=60) + gate.wait() + start = time.monotonic() + response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=noise.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + return response, time.monotonic()-start + with ThreadPoolExecutor(max_workers=10) as pool: + jobs = [pool.submit(upload, index) for index in range(10)] + start = time.monotonic(); gate.wait() + batch = [job.result() for job in jobs] + elapsed = time.monotonic()-start + after = memory_stats() + instances = {response['headers']['X-Instance-Id'] for response, _ in batch} + assert len(instances) == 2, 'Memory burst did not reach both replicas' + for response, _ in batch: + att = response['json'] + assert att['sha256'] == digest and att['size_bytes'] == len(data) + content = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob)['bytes'] + assert content == data + values = [latency for _, latency in batch] + results.append(dict(dimension=dimension, size_bytes=len(data), concurrent_uploads=10, + repetition=repeat+1, elapsed_seconds=round(elapsed, 3), successful_uploads=10, + failed_attempts=0, burst_successful_rps=10/elapsed, + successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]}, + latency_samples_ms=[round(v*1000, 3) for v in values], + before=before, after=after, observed_instances=len(instances), + original_checksums_verified=True)) + save('upload_memory.json', dict(runs=results, measurement='process high-water RSS and cgroup peak if available')) + return results + + +def cost_experiment(sizes): + active = any(row['running'] for row in containers('reconciler')) + command(['docker', 'compose', 'stop', 'reconciler']) + try: + print('Reconciler cost: datasets '+', '.join(map(str, sizes))+', three scans each', flush=True) + output = command(['docker', 'compose', 'run', '--rm', '--no-deps', '-e', 'BASE_URL=http://lb:8080', + 'reconciler', 'python', 'student-tests/storage_scenario.py', 'cost', + '--sizes', *map(str, sizes)], timeout=900) + result = json.loads(output) + save('reconcile_cost.json', result) + return result['scans'] + finally: + if active: + command(['docker', 'compose', 'start', 'reconciler']) + + +def plots(cost, uploads): + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + sizes = sorted({row['dataset_added'] for row in cost}) + seconds = [statistics.median(row['elapsed_seconds'] for row in cost if row['dataset_added']==size) for size in sizes] + fig, ax = plt.subplots(figsize=(6, 4)) + ax.plot(sizes, seconds, marker='o') + ax.set(xlabel='Objects + metadata', ylabel='Median scan, seconds', title='Reconciler cost') + ax.grid(alpha=.3); fig.tight_layout() + fig.savefig(ROOT/'evidence/reconcile_cost.png', dpi=150); plt.close(fig) + dimensions = sorted({row['dimension'] for row in uploads}) + peaks = [max(api['process_peak_rss_bytes'] for row in uploads if row['dimension']==d for api in row['after'])/1024**2 for d in dimensions] + fig, ax = plt.subplots(figsize=(6, 4)) + ax.bar([f'{d}x{d}' for d in dimensions], peaks) + ax.set(ylabel='Peak API process RSS, MiB', title='10 concurrent uploads') + fig.tight_layout(); fig.savefig(ROOT/'evidence/upload_memory.png', dpi=150); plt.close(fig) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--mode', choices=['privacy', 'faults', 'memory', 'cost', 'all'], default='all') + p.add_argument('--repeats', type=int, default=3) + p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000]) + args = p.parse_args() + if args.repeats < 1 or sorted(set(args.sizes)) != args.sizes or min(args.sizes) < 1: + p.error('Positive repeats and increasing distinct dataset sizes required') + base = os.getenv('BASE_URL', 'http://localhost:8080') + save('experiment_environment.json', dict(docker=command(['docker', 'version', '--format', '{{.Server.Version}}']), + compose=command(['docker', 'compose', 'version', '--short']), + containers=containers(), client_cpus=os.cpu_count())) + if args.mode in {'privacy', 'all'}: + file_checks() + if args.mode in {'faults', 'all'}: + storage_checks() + uploads = memory_experiment(base, args.repeats) if args.mode in {'memory', 'all'} else [] + cost = cost_experiment(args.sizes) if args.mode in {'cost', 'all'} else [] + if uploads and cost: + plots(cost, uploads) + groups = [] + for size in args.sizes: + cases = [row for row in cost if row['dataset_added'] == size] + groups.append(dict(objects=size, repetitions=len(cases), + median_seconds=statistics.median(row['elapsed_seconds'] for row in cases), + heads_per_scan=[row['counters']['head_requests'] for row in cases])) + save('experiment_summary.json', dict(reconciler_cost=groups, concurrent_uploads=10, + verified_originals=sum(row['successful_uploads'] for row in uploads), + status='PASS')) + print('Experiment completed; results saved in evidence/') + + +if __name__ == '__main__': + main() diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py new file mode 100644 index 0000000..45a6b77 --- /dev/null +++ b/scripts/healthcheck.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import json +import os +import urllib.request + + +host = "127.0.0.1" +port = os.getenv("APP_PORT", "8080") +with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response: + payload = json.load(response) + if response.status != 200 or payload != {"status": "ok"}: + raise SystemExit(1) diff --git a/scripts/monitor.py b/scripts/monitor.py new file mode 100644 index 0000000..05db2a0 --- /dev/null +++ b/scripts/monitor.py @@ -0,0 +1,44 @@ +"""Редкие снимки Docker CPU/RAM и ожиданий PostgreSQL; overhead отражён в методике.""" +import json +import threading +import time +from scripts.stack import command, containers + + +class Monitor: + def __init__(self): + self.samples = [] + self.stop = threading.Event() + self.thread = threading.Thread(target=self.loop) + + def __enter__(self): + self.started = time.monotonic() + self.thread.start() + return self + + def __exit__(self, *args): + self.stop.set() + self.thread.join(timeout=120) + + def loop(self): + while not self.stop.is_set(): + try: + ids = [r['id'] for r in containers() if r['running'] and r['service'] != 'migrate'] + raw = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids]) + stats = [json.loads(line) for line in raw.splitlines()] + query = "SELECT coalesce(json_agg(x),'[]') FROM (SELECT state,wait_event_type," \ + "wait_event,count(*) FROM pg_stat_activity WHERE backend_type='client backend' " \ + "AND pid<>pg_backend_pid() GROUP BY state,wait_event_type,wait_event) x" + waits = json.loads(command(['docker', 'compose', 'exec', '-T', 'db', + 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query])) + pools = [] + for row in containers('api'): + if row['running']: + output = command(['docker', 'exec', row['id'], 'python', '-c', + 'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/pool",timeout=3).read().decode())']) + pools.append(dict(instance=row['hostname'], stats=json.loads(output))) + self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3), + docker=stats, postgres=waits, pools=pools)) + except Exception as exc: + self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3), error=str(exc))) + self.stop.wait(3) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..c7764af --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +docker compose stop reconciler lb api cache db s3 +docker compose up -d --wait db cache s3 +docker compose run --rm --no-deps migrate +docker compose run --rm --no-deps s3-bootstrap +docker compose up -d --no-deps --wait api +docker compose up -d --no-deps lb +python3 scripts/stack.py wait --count 2 +docker compose up -d --no-deps reconciler diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 0000000..74e6dc1 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +docker compose start api +python3 scripts/stack.py wait --count 2 diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..985c37e --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p .secrets +chmod 700 .secrets +python3 - <<'PY' +from pathlib import Path +import secrets +password_file = Path('.secrets/db_password') +if not password_file.exists(): + password_file.write_text(secrets.token_hex(24)) +password = password_file.read_text().strip() +Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger') +for name in ['s3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key']: + file = Path('.secrets') / name + if not file.exists(): + file.write_text(secrets.token_hex(16 if name.endswith(('user', 'id')) else 32)) +for file in Path('.secrets').iterdir(): + file.chmod(0o644 if file.name in {'database_url', 's3_access_key_id', 's3_secret_access_key'} else 0o600) +if not Path('.env').exists(): + Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\nAPI_CPU=1.0\nAPI_MEMORY=512m\n') +print('Локальные настройки подготовлены; секреты не выводятся.') +PY diff --git a/scripts/stack.py b/scripts/stack.py new file mode 100644 index 0000000..1893102 --- /dev/null +++ b/scripts/stack.py @@ -0,0 +1,129 @@ +"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] + + +def command(args, env=None, timeout=120): + result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError(result.stderr[-2000:]) + return result.stdout.strip() + + +def containers(service=None): + args = ['docker', 'compose', 'ps', '-a', '-q'] + if service: + args.append(service) + ids = command(args).split() + if not ids: + return [] + # Сохраняем только нужные поля, без environment с возможными секретами. + rows = json.loads(command(['docker', 'inspect', *ids])) + return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'], + service=r['Config']['Labels']['com.docker.compose.service'], + running=r['State']['Running'], started_at=r['State']['StartedAt'], + cpu_limit=r['HostConfig']['NanoCpus'] / 1e9, + memory_limit_bytes=r['HostConfig']['Memory'], + published_ports=r['NetworkSettings']['Ports']) for r in rows] + + +def save(name, data): + path = ROOT / 'evidence' / name + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + + +def wait(count, base=None): + base = base or os.getenv('BASE_URL', 'http://localhost:8080') + deadline = time.monotonic() + 75 + while time.monotonic() < deadline: + observed = set() + for _ in range(max(12, count * 8)): + try: + with urllib.request.urlopen(base + '/health/ready', timeout=3) as response: + observed.add(response.headers['X-Instance-Id']) + except (OSError, ValueError): + break + if len(observed) == count and None not in observed: + return sorted(observed) + time.sleep(.5) + raise RuntimeError(f'Не дождались {count} готовых API через LB') + + +def stop(instance): + hostname = instance.split(':')[0] + found = [r for r in containers('api') if r['hostname'] == hostname and r['running']] + if len(found) != 1: + raise RuntimeError('Instance alias не соответствует одной работающей API-реплике') + before = containers() + command(['docker', 'stop', '--time', '3', found[0]['id']]) + result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers()) + save('last_stop.json', result) + return result + + +def scale(count, cpu, memory, pool): + env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool)) + command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate', + '--scale', f'api={count}', 'api'], env) + instances = wait(count) + rows = containers('api') + if len(rows) != count or not all(r['running'] for r in rows): + raise RuntimeError('Фактическое число контейнеров отличается от запрошенного') + for row in rows: + if abs(row['cpu_limit'] - cpu) > .001: + raise RuntimeError('Docker не применил CPU limit') + if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024: + raise RuntimeError('Docker не применил RAM limit') + return dict(instances=instances, containers=rows) + + +def rolling(): + old = [r['id'] for r in containers('api') if r['running']] + if len(old) != 2: + raise RuntimeError('Rolling проверяется на двух репликах обычного стенда') + before = containers() + command(['docker', 'compose', 'build', 'api']) + steps = [] + for container in old: + # Не пересоздаём одновременно старые контейнеры при изменении image. + command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api']) + wait(3) + command(['docker', 'stop', '--time', '10', container]) + command(['docker', 'rm', container]) + wait(2) + steps.append(containers('api')) + save('rolling_containers.json', dict(before=before, steps=steps, after=containers())) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling']) + p.add_argument('--count', type=int, default=2) + p.add_argument('--instance') + p.add_argument('--output', default='containers.json') + args = p.parse_args() + if args.action == 'wait': + print('API ready:', ', '.join(wait(args.count))) + elif args.action == 'snapshot': + save(args.output, containers()) + print('Container identities and limits saved') + elif args.action == 'stop': + if not args.instance: + p.error('--instance required') + stop(args.instance) + print('Selected API stopped') + else: + rolling() + print('API replaced one at a time') + + +if __name__ == '__main__': + main() diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..1e3e396 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +docker compose up -d --build +python3 scripts/stack.py wait --count 2 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..78b4242 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +python3 scripts/stack.py stop --instance "$TARGET_INSTANCE_ID" diff --git a/scripts/test_expiry.sh b/scripts/test_expiry.sh new file mode 100755 index 0000000..7298b20 --- /dev/null +++ b/scripts/test_expiry.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +restore() { + docker compose up -d --no-deps --force-recreate --scale api=2 api + python3 scripts/stack.py wait --count 2 +} +trap restore EXIT +SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate --scale api=2 api +python3 scripts/stack.py wait --count 2 +python3 student-tests/checks.py --expiry diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..529a483 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,17 @@ +"""Ожидание готовности API без сторонних библиотек.""" +import os +import time +import urllib.request + +url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready' +for attempt in range(90): + try: + with urllib.request.urlopen(url, timeout=2) as response: + if response.status == 200: + print('API ready') + break + except OSError: + pass + time.sleep(1) +else: + raise SystemExit('API не стал готов за 90 попыток') diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..fdcda77 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,157 @@ +"""Ограниченный binary upload, проверка изображения и сохранение в двух хранилищах.""" +import asyncio +import base64 +from datetime import datetime +import hashlib +import os +import tempfile +import threading +import unicodedata +import uuid +import warnings + +from fastapi import Request +from fastapi.responses import StreamingResponse +from PIL import Image, ImageFile, PngImagePlugin, UnidentifiedImageError +from pydantic import BaseModel, ConfigDict +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from src.config import MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, UPLOAD_TIMEOUT +from src.errors import ApiProblem +from src.storage import ObjectStore, PREFIX, unlock_attachment + +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS +ImageFile.LOAD_TRUNCATED_IMAGES = False +PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 +PngImagePlugin.MAX_TEXT_MEMORY = 2 * 1024 * 1024 +storage = ObjectStore() +upload_slots = asyncio.Semaphore(4) +decode_slots = threading.Semaphore(1) + + +class Attachment(BaseModel): + model_config = ConfigDict(extra='ignore') + id: uuid.UUID + chat_id: uuid.UUID + owner_id: uuid.UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list = [] + error_code: str | None = None + created_at: datetime + updated_at: datetime + + +def public_attachment(row): + return Attachment(**row).model_dump(mode='json') + + +def filename_valid(name): + if (not 1 <= len(name) <= 255 or not name.strip() or name in {'.', '..'} or + any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in name)): + raise ApiProblem(400, 'invalid_request', 'Некорректное отображаемое имя файла') + + +def validate_image(file, declared_type): + # Декодируется максимум одно изображение на процесс, даже при нескольких upload. + with decode_slots, warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + try: + file.seek(0) + # Pillow может закрыть свой file object. Дубликат FD сохраняет исходный spool. + with os.fdopen(os.dup(file.fileno()), 'rb') as probe, Image.open(probe, formats=['PNG', 'JPEG']) as image: + if image.width * image.height > MAX_IMAGE_PIXELS: + raise ApiProblem(413, 'payload_too_large', 'Слишком много пикселей') + detected = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}[image.format] + if detected != declared_type: + raise ApiProblem(415, 'unsupported_media_type', 'MIME не совпадает с содержимым') + image.verify() + file.seek(0) + with os.fdopen(os.dup(file.fileno()), 'rb') as probe, Image.open(probe, formats=['PNG', 'JPEG']) as image: + image.load() # JPEG verify не проверяет все декодируемые данные. + image.close() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise ApiProblem(413, 'payload_too_large', 'Слишком много пикселей') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiProblem(415, 'unsupported_media_type', 'Повреждённое или неподдерживаемое изображение') from None + file.seek(0) + + +def persist(file, pool, chat_id, owner_id, filename, mime, size, sha256, md5): + attachment_id = uuid.uuid4() + key = PREFIX + str(chat_id) + '/' + str(attachment_id) + with pool.connection() as db: + # Session lock сохраняется между COMMIT намерения и записью ready. + # Reconciler использует тот же lock и не удаляет активную загрузку. + db.execute('SELECT pg_advisory_lock(hashtextextended(%s,0))', ('attachment:' + str(attachment_id),)) + db.commit() + try: + db.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256," + "object_key,status,lease_until) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'uploading'," + "clock_timestamp()+interval '120 seconds')", + (attachment_id, chat_id, owner_id, filename, mime, size, sha256, key)) + db.commit() # Устойчивое намерение ДО S3 PUT. + storage.put(file, key, size, mime, sha256, md5) + row = db.execute("UPDATE attachments SET status='ready', updated_at=clock_timestamp()," + "error_code=NULL WHERE id=%s RETURNING *", (attachment_id,)).fetchone() + db.commit() # 201 разрешён только после S3 PUT и COMMIT. + return public_attachment(row) + finally: + # При ошибке остаётся uploading с lease. Его безопасно обработает reconciler. + db.rollback() + unlock_attachment(db, attachment_id) + db.commit() + + +async def receive(request: Request, pool, chat_id, owner_id, filename): + filename_valid(filename) + mime = request.headers.get('content-type', '').split(';', 1)[0].strip().lower() + if mime not in {'image/png', 'image/jpeg'}: + raise ApiProblem(415, 'unsupported_media_type', 'Нужен PNG или JPEG') + length = request.headers.get('content-length') + if length is not None: + if not length.isdecimal(): + raise ApiProblem(400, 'invalid_request', 'Некорректный Content-Length') + if int(length) > MAX_UPLOAD_BYTES: + raise ApiProblem(413, 'payload_too_large', 'Лимит файла 10 MiB') + try: + async with asyncio.timeout(UPLOAD_TIMEOUT), upload_slots: + # На диске только временный bounded spool, не оригинал для будущего download. + with tempfile.TemporaryFile() as file: + size, sha, md5 = 0, hashlib.sha256(), hashlib.md5() + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ApiProblem(413, 'payload_too_large', 'Лимит файла 10 MiB') + await run_in_threadpool(file.write, chunk) + sha.update(chunk); md5.update(chunk) + if not size: + raise ApiProblem(415, 'unsupported_media_type', 'Пустой файл') + await run_in_threadpool(validate_image, file, mime) + return await run_in_threadpool(persist, file, pool, chat_id, owner_id, filename, + mime, size, sha.hexdigest(), base64.b64encode(md5.digest()).decode()) + except TimeoutError: + raise ApiProblem(503, 'unavailable', 'Время загрузки истекло') from None + except ClientDisconnect: + raise ApiProblem(400, 'invalid_request', 'Передача файла прервана') from None + + +def download(row): + result = storage.get(row['object_key']) + body = result['Body'] + if result['ContentLength'] != row['size_bytes']: + body.close() + raise ApiProblem(503, 'unavailable', 'Оригинал временно недоступен') + + def chunks(): + try: + while data := body.read(64 * 1024): + yield data + finally: + body.close() + return StreamingResponse(chunks(), media_type=row['content_type'], + headers={'Content-Length': str(row['size_bytes'])}) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..b0fc5f4 --- /dev/null +++ b/src/config.py @@ -0,0 +1,22 @@ +"""Настройки из окружения или Docker secrets; секреты не печатаем.""" +import os +from pathlib import Path + + +def setting(name, default=None): + file = os.getenv(name + '_FILE') + return Path(file).read_text().strip() if file else os.getenv(name, default) + + +DATABASE_URL = setting('DATABASE_URL') +CACHE_URL = setting('CACHE_URL', 'redis://localhost:6379/0') +SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +POOL_MAX = int(os.getenv('DB_POOL_MAX', '8')) +S3_ENDPOINT = setting('S3_ENDPOINT', 'http://s3:9000') +S3_REGION = setting('S3_REGION', 'us-east-1') +S3_BUCKET = setting('S3_BUCKET', 'messenger-attachments') +S3_ACCESS_KEY = setting('S3_ACCESS_KEY_ID') +S3_SECRET_KEY = setting('S3_SECRET_ACCESS_KEY') +MAX_UPLOAD_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) +MAX_IMAGE_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) +UPLOAD_TIMEOUT = int(os.getenv('UPLOAD_TIMEOUT_SECONDS', '45')) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..45c46cd --- /dev/null +++ b/src/errors.py @@ -0,0 +1,6 @@ +class ApiProblem(Exception): + def __init__(self, status, code, message, challenge=None): + self.status = status + self.code = code + self.message = message + self.challenge = challenge diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..d181983 --- /dev/null +++ b/src/main.py @@ -0,0 +1,637 @@ +from __future__ import annotations + +import base64 +import json +import logging +import os +import re +import time +import uuid +from datetime import datetime, timedelta, timezone +from contextlib import asynccontextmanager +import hashlib +import secrets +import socket +import resource +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool, PoolTimeout +import redis +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from starlette.exceptions import HTTPException +from src.config import DATABASE_URL, CACHE_URL, SESSION_TTL, POOL_MAX +from src.errors import ApiProblem +from src import attachments +from botocore.exceptions import BotoCoreError, ClientError +from typing import Annotated + +from fastapi import Depends, FastAPI, Header, Query, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool + + +logging.basicConfig( + level=os.getenv("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(message)s", +) +logger = logging.getLogger("messenger") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def as_json(value: BaseModel) -> dict: + return value.model_dump(mode="json") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class User(StrictModel): + id: uuid.UUID + username: str + display_name: str + created_at: datetime + + +class CreateUser(StrictModel): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("password") + @classmethod + def password_has_visible_characters(cls, value: str) -> str: + if not value.strip(): + raise ValueError("password must not contain only whitespace") + return value + + @field_validator("display_name") + @classmethod + def display_name_has_visible_characters(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must not contain only whitespace") + return value + + +class Chat(StrictModel): + id: uuid.UUID + title: str + member_ids: list[uuid.UUID] + created_at: datetime + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_has_visible_characters(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not contain only whitespace") + return value + + @field_validator("member_ids") + @classmethod + def member_ids_are_unique(cls, value: list[uuid.UUID]) -> list[uuid.UUID]: + if len(value) != len(set(value)): + raise ValueError("member_ids must be unique") + return value + + +class Message(StrictModel): + id: uuid.UUID + chat_id: uuid.UUID + sender_id: uuid.UUID + text: str + created_at: datetime + attachment_ids: list[uuid.UUID] = Field(default_factory=list) + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode='after') + def valid_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('Нужен текст или вложение') + if len(self.attachment_ids) != len(set(self.attachment_ids)): + raise ValueError('Повторяющиеся attachment_ids') + return self + + +class Store: + """Сессии и метаданные в PostgreSQL; Redis — только кеш профилей.""" + + def __init__(self) -> None: + self.pool = ConnectionPool( + DATABASE_URL, min_size=1, max_size=POOL_MAX, timeout=2, open=False, + kwargs={"row_factory": dict_row, "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"}, + ) + self.cache = redis.Redis.from_url( + CACHE_URL, decode_responses=True, socket_connect_timeout=0.25, socket_timeout=0.25, + ) + + def start(self) -> None: + self.pool.open() + self.pool.wait(timeout=10) + + def close(self) -> None: + self.pool.close() + self.cache.close() + + def chat(self, db, chat_id: uuid.UUID) -> Chat | None: + row = db.execute("SELECT * FROM chats WHERE id=%s", (chat_id,)).fetchone() + if row is None: + return None + members = db.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", + (chat_id,)).fetchall() + return Chat(**row, member_ids=[member["user_id"] for member in members]) + + def chat_messages(self, db, chat_id: uuid.UUID, limit: int, after) -> list[Message]: + # В lab1 здесь были общий список/список чата и bisect_right. + # Теперь границу ищет B-tree PostgreSQL (chat_id, created_at, id). + query = ("SELECT m.*, COALESCE((SELECT array_agg(a.attachment_id ORDER BY a.position) " + "FROM message_attachments a WHERE a.message_id=m.id),'{}'::uuid[]) AS attachment_ids " + "FROM messages m WHERE chat_id=%s") + params = [chat_id] + if after is not None: + query += " AND (created_at,id)>(%s,%s)" + params.extend(after) + rows = db.execute(query + " ORDER BY created_at,id LIMIT %s", params + [limit + 1]).fetchall() + return [Message(**row) for row in rows] + + +INSTANCE_ID = os.getenv("INSTANCE_ID") or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}" +store = Store() +passwords = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +dummy_hash = passwords.hash(secrets.token_urlsafe(32)) + + +@asynccontextmanager +async def lifespan(app): + store.start() + yield + store.close() + + +app = FastAPI(title="MTUCI PostgreSQL messenger", version="4.0.0", lifespan=lifespan, + docs_url=None, redoc_url=None, openapi_url=None) + + +def request_id(request: Request) -> str: + return request.state.request_id + + +def error_response(request: Request, status: int, code: str, message: str, challenge: str | None = None) -> JSONResponse: + rid = getattr(request.state, "request_id", str(uuid.uuid4())) + headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} + if challenge: + headers["WWW-Authenticate"] = challenge + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers=headers, + ) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request.state.request_id = str(uuid.uuid4()) + started = time.perf_counter() + status = 500 + try: + # Без токена отказываем ДО обращения к БД и проверки JSON. + if request.url.path.startswith("/api/v1/") and request.url.path not in { + "/api/v1/users", "/api/v1/auth/sessions" + }: + scheme, _, token = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "bearer" or not token or len(token) > 256: + status = 401 + return error_response(request, 401, "unauthorized", "Требуется Bearer", + 'Bearer realm="messenger"') + json_endpoints = { + "/api/v1/users", + "/api/v1/chats", + } + requires_json = request.method == "POST" and ( + request.url.path in json_endpoints + or re.fullmatch(r"/api/v1/chats/[^/]+/messages", request.url.path) is not None + ) + media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if requires_json and media_type != "application/json": + status = 415 + return error_response( + request, + 415, + "unsupported_media_type", + "Требуется Content-Type: application/json", + ) + if requires_json: + # Ограниченный JSON; после body() Starlette повторно отдаёт его маршруту. + raw = await request.body() + if len(raw) > 65536: + status = 413 + return error_response(request, 413, "payload_too_large", "Запрос слишком большой") + response = await call_next(request) + status = response.status_code + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + finally: + logger.info( + "request method=%s path=%s status=%s duration_ms=%.3f request_id=%s", + request.method, + request.url.path, + status, + (time.perf_counter() - started) * 1000, + request.state.request_id, + ) + + +@app.exception_handler(ApiProblem) +async def api_problem_handler(request: Request, exc: ApiProblem): + return error_response(request, exc.status, exc.code, exc.message, exc.challenge) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Запрос не соответствует контракту") + + +@app.exception_handler(Exception) +async def unexpected_error_handler(request: Request, exc: Exception): + logger.error("unexpected_error request_id=%s", request_id(request)) + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +@app.exception_handler(psycopg.Error) +@app.exception_handler(PoolTimeout) +async def database_error_handler(request: Request, exc: Exception): + return error_response(request, 503, "unavailable", "Хранилище временно недоступно") + + +@app.exception_handler(BotoCoreError) +@app.exception_handler(ClientError) +async def object_store_error_handler(request: Request, exc: Exception): + return error_response(request, 503, 'unavailable', 'Объектное хранилище временно недоступно') + + +@app.exception_handler(HTTPException) +async def http_error_handler(request: Request, exc: HTTPException): + code = "not_found" if exc.status_code == 404 else "invalid_request" + return error_response(request, exc.status_code, code, "Неверный маршрут или метод") + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + raise ApiProblem(401, "unauthorized", "Нужна действительная идентификация", challenge) + + +def token_hash(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def current_user(authorization: Annotated[str | None, Header()] = None) -> User: + scheme, _, token = (authorization or "").partition(" ") + if scheme.lower() != "bearer" or not token or len(token) > 256: + unauthorized() + with store.pool.connection() as db: + session = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()", + (token_hash(token),)).fetchone() + if session is None: + unauthorized() + key = "user:" + str(session["user_id"]) + cache_available = True + try: + cached = store.cache.get(key) + if cached: + user = User.model_validate_json(cached) + if user.id == session["user_id"]: + return user + except (redis.RedisError, ValueError): + cache_available = False + row = db.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s", + (session["user_id"],)).fetchone() + user = User(**row) + if cache_available: + try: + store.cache.setex(key, 60, user.model_dump_json()) + except redis.RedisError: + pass + return user + + +def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, uuid.UUID]: + try: + if not cursor or len(cursor) > 1024: + raise ValueError + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.b64decode(padded.encode(), altchars=b"-_", validate=True).decode()) + if not isinstance(payload, dict) or set(payload) != {"scope", "created_at", "id"}: + raise ValueError + if payload["scope"] != expected_scope: + raise ValueError + if not isinstance(payload["created_at"], str) or not isinstance(payload["id"], str): + raise ValueError + created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00")) + item_id = uuid.UUID(payload["id"]) + if created_at.tzinfo is None: + raise ValueError + return created_at, item_id + except (ValueError, TypeError, KeyError, json.JSONDecodeError, UnicodeError): + raise ApiProblem(400, "invalid_request", "Некорректный cursor") from None + + +def encode_cursor(scope: str, created_at: datetime, item_id: uuid.UUID) -> str: + payload = { + "scope": scope, + "created_at": created_at.isoformat().replace("+00:00", "Z"), + "id": str(item_id), + } + return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode().rstrip("=") + + +def page(items: list, limit: int, scope: str) -> dict: + # SQL уже выбрал limit+1 строк после курсора; весь список не загружаем. + selected = items[:limit] + next_cursor = None + if len(items) > limit and selected: + last = selected[-1] + next_cursor = encode_cursor(scope, last.created_at, last.id) + return {"items": [as_json(item) for item in selected], "next_cursor": next_cursor} + + +def visible_chat(chat_id: uuid.UUID, user: User) -> Chat: + with store.pool.connection() as db: + member = db.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", + (chat_id, user.id)).fetchone() + if member is None: + raise ApiProblem(404, "not_found", "Объект не найден") + return store.chat(db, chat_id) + + +def page_parameters(request: Request, scope: str, limit: int, cursor: str | None): + # Query(ge/le) сохранён из lab1, добавлена проверка дублей и формы limit. + if set(request.query_params) - {"limit", "cursor"} or any( + len(request.query_params.getlist(k)) > 1 for k in request.query_params + ): + raise ApiProblem(400, "invalid_request", "Некорректные параметры страницы") + if not re.fullmatch(r"[0-9]{1,3}", request.query_params.get("limit", "50")): + raise ApiProblem(400, "invalid_request", "Некорректный limit") + return decode_cursor(cursor, scope) if cursor is not None else None + + +@app.get("/health/live") +def liveness(): + return {"status": "ok"} + + +@app.get("/health/ready") +def readiness(): + with store.pool.connection() as db: + db.execute("SELECT 1") + attachments.storage.ready() + return {"status": "ok"} + + +@app.get("/internal/pool") +def pool_statistics(): + # Диагностика доступна только внутри Docker network: LB закрывает этот путь. + return store.pool.get_stats() + + +@app.get('/internal/memory') +def memory_statistics(): + result = {'process_peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024} + from pathlib import Path + for name in ['/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes']: + path = Path(name) + if path.is_file(): + result['container_peak_bytes'] = int(path.read_text().strip()) + result['container_peak_source'] = name + break + return result + + +@app.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + # Синхронные маршруты FastAPI исполняет в worker thread, не в event loop. + hashed = passwords.hash(body.password) + try: + with store.pool.connection() as db: + row = db.execute( + "INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid.uuid4(), body.username, body.display_name, hashed), + ).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiProblem(409, "conflict", "Username уже занят") from None + return as_json(User(**row)) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request, authorization: Annotated[str | None, Header()] = None): + if request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers: + raise ApiProblem(400, "invalid_request", "Вход не принимает тело запроса") + scheme, _, encoded = (authorization or "").partition(" ") + try: + if scheme.lower() != "basic" or len(encoded) > 2048: + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1) + if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: + raise ValueError + except (ValueError, UnicodeError): + unauthorized(basic=True) + with store.pool.connection() as db: + row = db.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + try: + passwords.verify(row["password_hash"] if row else dummy_hash, password) + except (VerificationError, InvalidHashError): + unauthorized(basic=True) + if row is None: + unauthorized(basic=True) + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL) + with store.pool.connection() as db: + db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", + (token_hash(token), row["id"], expires_at)) + user = User(**{k: row[k] for k in ("id", "username", "display_name", "created_at")}) + return {"token": token, "token_type": "Bearer", "user": as_json(user), + "expires_at": expires_at.isoformat().replace("+00:00", "Z")} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(user: Annotated[User, Depends(current_user)], + authorization: Annotated[str, Header()]): + token = authorization.partition(" ")[2] + with store.pool.connection() as db: + db.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def get_current_user(user: Annotated[User, Depends(current_user)]): + return as_json(user) + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, user: Annotated[User, Depends(current_user)]): + if user.id not in body.member_ids: + raise ApiProblem(400, "invalid_request", "Текущий пользователь должен быть участником") + with store.pool.connection() as db: + found = db.execute("SELECT id FROM users WHERE id=ANY(%s)", (body.member_ids,)).fetchall() + if len(found) != len(body.member_ids): + raise ApiProblem(400, "invalid_request", "Указан неизвестный участник") + row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", + (uuid.uuid4(), body.title)).fetchone() + with db.cursor() as cur: + cur.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", + [(row["id"], member) for member in body.member_ids]) + chat = store.chat(db, row["id"]) + return as_json(chat) # COMMIT уже выполнен для чата и всех участников. + + +@app.get("/api/v1/chats") +def list_chats( + request: Request, + user: Annotated[User, Depends(current_user)], + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +): + scope = f"chats:{user.id}" + after = page_parameters(request, scope, limit, cursor) + query = "SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s" + params = [user.id] + if after is not None: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + with store.pool.connection() as db: + rows = db.execute(query + " ORDER BY c.created_at,c.id LIMIT %s", params + [limit + 1]).fetchall() + chats = [store.chat(db, row["id"]) for row in rows] + return page(chats, limit, scope) + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)]): + return as_json(visible_chat(chat_id, user)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + request: Request, + chat_id: uuid.UUID, + body: CreateMessage, + user: Annotated[User, Depends(current_user)], + idempotency_key: Annotated[str | None, Header()] = None, +): + if (not idempotency_key or + not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) or + len(request.headers.getlist("idempotency-key")) != 1): + raise ApiProblem(400, "invalid_request", "Нужен корректный Idempotency-Key") + visible_chat(chat_id, user) # Повтор тоже проверяет сессию и участие. + canonical = {'text': body.text} + if body.attachment_ids: + canonical['attachment_ids'] = [str(value) for value in body.attachment_ids] + payload = json.dumps(canonical, ensure_ascii=False, sort_keys=True, + separators=(",", ":")) + digest = hashlib.sha256(payload.encode()).hexdigest() + lock_scope = f"{user.id}:POST:{chat_id}:{idempotency_key}" + with store.pool.connection() as db: + # Общая блокировка в PostgreSQL, а не локальный Lock одной реплики. + # Коллизия хеша только сериализует независимые запросы; ключи сравниваются полностью. + db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (lock_scope,)) + previous = db.execute( + "SELECT * FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s", + (user.id, chat_id, idempotency_key), + ).fetchone() + if previous and previous["expires_at"] > utc_now(): + if previous["body_hash"] != digest: + raise ApiProblem(409, "conflict", "Ключ уже использован с другим сообщением") + result = previous["response"] + else: + for attachment_id in body.attachment_ids: + found = db.execute('SELECT chat_id,status FROM attachments WHERE id=%s FOR KEY SHARE', + (attachment_id,)).fetchone() + if not found or found['chat_id'] != chat_id: + raise ApiProblem(404, 'not_found', 'Вложение недоступно') + if found['status'] != 'ready': + raise ApiProblem(409, 'conflict', 'Вложение ещё недоступно') + if previous: + db.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s", + (user.id, chat_id, idempotency_key)) + row = db.execute( + "INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", + (uuid.uuid4(), chat_id, user.id, body.text), + ).fetchone() + for position, attachment_id in enumerate(body.attachment_ids): + db.execute('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)', + (row['id'], attachment_id, position)) + result = as_json(Message(**row, attachment_ids=body.attachment_ids)) + db.execute( + "INSERT INTO message_requests(user_id,chat_id,request_key,body_hash,response,expires_at) " + "VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", + (user.id, chat_id, idempotency_key, digest, Jsonb(result)), + ) + # Сообщение и результат повтора зафиксированы одной транзакцией до ответа. + return result + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + request: Request, + chat_id: uuid.UUID, + user: Annotated[User, Depends(current_user)], + limit: Annotated[int, Query(ge=1, le=100)] = 50, + cursor: Annotated[str | None, Query(min_length=1)] = None, +): + visible_chat(chat_id, user) + scope = f"messages:{chat_id}" + after = page_parameters(request, scope, limit, cursor) + with store.pool.connection() as db: + messages = store.chat_messages(db, chat_id, limit, after) + return page(messages, limit, scope) + + +@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201) +async def upload_attachment(request: Request, chat_id: uuid.UUID, + user: Annotated[User, Depends(current_user)], filename: str = Query()): + await run_in_threadpool(visible_chat, chat_id, user) + return await attachments.receive(request, store.pool, chat_id, user.id, filename) + + +def accessible_attachment(attachment_id, user): + with store.pool.connection() as db: + row = db.execute('SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s', (attachment_id, user.id)).fetchone() + if not row: + raise ApiProblem(404, 'not_found', 'Вложение недоступно') + return row + + +@app.get('/api/v1/attachments/{attachment_id}') +def attachment_metadata(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)]): + row = accessible_attachment(attachment_id, user) + if row['status'] != 'ready': + raise ApiProblem(503, 'unavailable', 'Вложение временно недоступно') + return attachments.public_attachment(row) + + +@app.get('/api/v1/attachments/{attachment_id}/content') +def attachment_content(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)], + variant: str = Query('original')): + row = accessible_attachment(attachment_id, user) + if variant != 'original': + raise ApiProblem(404, 'not_found', 'Вариант отсутствует') + if row['status'] != 'ready': + raise ApiProblem(503, 'unavailable', 'Вложение временно недоступно') + return attachments.download(row) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..6543523 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,26 @@ +"""Версионируемые миграции: один файл = одна атомарная транзакция.""" +from pathlib import Path +import psycopg +from src.config import DATABASE_URL + + +def migrate(): + with psycopg.connect(DATABASE_URL) as db: + # Миграции запускает отдельный сервис. Lock защищает от второго запуска оператором. + db.execute('SELECT pg_advisory_lock(20260002)') + db.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(version text PRIMARY KEY, applied_at timestamptz DEFAULT now())') + db.commit() + for file in sorted(Path('migrations').glob('*.sql')): + with db.transaction(): + applied = db.execute('SELECT 1 FROM schema_migrations WHERE version=%s', + (file.name,)).fetchone() + if not applied: + db.execute(file.read_text()) + db.execute('INSERT INTO schema_migrations(version) VALUES (%s)', (file.name,)) + print('Applied', file.name) + db.execute('SELECT pg_advisory_unlock(20260002)') + + +if __name__ == '__main__': + migrate() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..8e71363 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,193 @@ +"""Сверка двух хранилищ. Dry-run по умолчанию, apply меняет только проверенные записи.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import resource +import time +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row +from src.config import DATABASE_URL +from src.storage import ObjectStore, PREFIX, lock_attachment, unlock_attachment + + +def checksum_matches(storage, row, counters): + result = storage.get(row['object_key']) + counters['get_requests'] += 1 + size, digest = 0, hashlib.sha256() + try: + while chunk := result['Body'].read(64 * 1024): + size += len(chunk) + counters['bytes_read'] += len(chunk) + if size > row['size_bytes']: + return False + digest.update(chunk) + finally: + result['Body'].close() + return size == row['size_bytes'] and digest.hexdigest() == row['sha256'] + + +def run(grace=600, apply=False, chat_id=None): + if grace < 1: + raise ValueError('Grace period must be positive') + started = time.monotonic() + now = datetime.now(timezone.utc) + storage = ObjectStore() + counters = dict(objects_scanned=0, metadata_scanned=0, list_requests=0, head_requests=0, + get_requests=0, bytes_read=0, protected_fresh=0, + protected_active=0, protected_used=0) + actions = [] + + def old(timestamp): + return (now - timestamp).total_seconds() >= grace + + def head(key): + counters['head_requests'] += 1 + return storage.head(key) + + def action(kind, row, reason=None): + item = dict(kind=kind, attachment_id=str(row['id']), object_key=row['object_key'], applied=apply) + if reason: + item['reason'] = reason + actions.append(item) + + def release(db, attachment_id): + db.rollback() + unlock_attachment(db, attachment_id) + db.commit() + + try: + with psycopg.connect(DATABASE_URL, row_factory=dict_row, connect_timeout=2, + options='-c statement_timeout=10000 -c lock_timeout=2000') as db: + # LIST завершается успешно до любой модификации. Ошибка зависимости не равна missing. + prefix = PREFIX if chat_id is None else PREFIX + str(UUID(str(chat_id))) + '/' + inventory = list(storage.objects(prefix, counters)) + for obj in inventory: + counters['objects_scanned'] += 1 + try: + attachment_id = UUID(obj['Key'].rsplit('/', 1)[-1]) + except ValueError: + continue # Не трогаем неизвестный формат ключа. + if not lock_attachment(db, attachment_id, wait=False): + counters['protected_active'] += 1 + db.commit() + continue + db.commit() + try: + row = db.execute('SELECT id FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone() + if row: + continue + current = head(obj['Key']) # Не полагаемся только на старый LIST snapshot. + if not current: + continue + if not old(current['LastModified']): + counters['protected_fresh'] += 1 + continue + action('delete_orphan', dict(id=attachment_id, object_key=obj['Key'])) + if apply: + storage.delete(obj['Key']) + finally: + release(db, attachment_id) + + if chat_id is None: + ids = db.execute('SELECT id FROM attachments ORDER BY id').fetchall() + else: + ids = db.execute('SELECT id FROM attachments WHERE chat_id=%s ORDER BY id', (chat_id,)).fetchall() + db.commit() + for identifier in ids: + attachment_id = identifier['id'] + counters['metadata_scanned'] += 1 + if not lock_attachment(db, attachment_id, wait=False): + counters['protected_active'] += 1 + db.commit() + continue + db.commit() + try: + row = db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', + (attachment_id,)).fetchone() + if not row: + continue + if not old(row['created_at']): + counters['protected_fresh'] += 1 + continue + if row['status'] == 'uploading' and row['lease_until'] > now: + counters['protected_active'] += 1 + continue + used = db.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1', + (attachment_id,)).fetchone() + current = head(row['object_key']) + if row['status'] == 'uploading': + if used: + counters['protected_used'] += 1 + continue + if current and not old(current['LastModified']): + counters['protected_fresh'] += 1 + continue + action('delete_stale_upload', row, 'lease_expired') + if apply: + if current: + storage.delete(row['object_key']) + db.execute('DELETE FROM attachments WHERE id=%s', (attachment_id,)) + db.commit() + continue + if not current: + if row['status'] != 'missing': + action('mark_missing', row, 'object_missing') + if apply: + db.execute("UPDATE attachments SET status='missing',error_code='object_missing'," + "updated_at=clock_timestamp() WHERE id=%s", (attachment_id,)) + db.commit() + continue + matches = current['ContentLength'] == row['size_bytes'] + if matches and (row['status'] == 'missing' or current.get('Metadata', {}).get('sha256') != row['sha256']): + matches = checksum_matches(storage, row, counters) + if not matches: + if row['status'] != 'missing': + action('mark_missing', row, 'object_corrupt') + if apply: + db.execute("UPDATE attachments SET status='missing',error_code='object_corrupt'," + "updated_at=clock_timestamp() WHERE id=%s", (attachment_id,)) + db.commit() + continue + if row['status'] == 'missing': + action('restore_ready', row, 'verified_original_restored') + if apply: + db.execute("UPDATE attachments SET status='ready',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", (attachment_id,)) + db.commit() + elif used: + counters['protected_used'] += 1 + finally: + release(db, attachment_id) + finally: + storage.client.close() + return dict(mode='apply' if apply else 'dry-run', grace_seconds=grace, chat_scope=str(chat_id) if chat_id else None, + elapsed_seconds=round(time.monotonic()-started, 6), counters=counters, actions=actions, + process_peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--apply', action='store_true') + p.add_argument('--grace-seconds', type=int, default=600) + p.add_argument('--loop', action='store_true') + p.add_argument('--interval', type=int, default=60) + p.add_argument('--chat-id', type=UUID, help='Optional safe scope for one chat') + args = p.parse_args() + while True: + try: + print(json.dumps(run(args.grace_seconds, args.apply, args.chat_id), ensure_ascii=False), flush=True) + except Exception: + # Не выдаём DSN, ключи и текст исключений SDK в логи. + print(json.dumps({'status': 'FAILED', 'reason': 'dependency_or_scan_error'}), flush=True) + if not args.loop: + raise SystemExit(1) + if not args.loop: + break + time.sleep(max(1, args.interval)) + + +if __name__ == '__main__': + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..d112d42 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,57 @@ +"""S3-клиент с ограниченными credentials; ключи назначаются приложением.""" +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +from src.config import S3_ENDPOINT, S3_REGION, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY + +PREFIX = 'originals/' + + +class ObjectStore: + def __init__(self): + self.client = boto3.client( + 's3', endpoint_url=S3_ENDPOINT, region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, + config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}, request_checksum_calculation='when_required', + response_checksum_validation='when_required')) + + def ready(self): + self.client.list_objects_v2(Bucket=S3_BUCKET, Prefix=PREFIX, MaxKeys=1) + + def put(self, file, key, size, mime, sha256, content_md5): + file.seek(0) + self.client.put_object(Bucket=S3_BUCKET, Key=key, Body=file, ContentLength=size, + ContentType=mime, Metadata={'sha256': sha256}, ContentMD5=content_md5) + + def head(self, key): + try: + return self.client.head_object(Bucket=S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in {'404', 'NoSuchKey', 'NotFound'}: + return None + raise + + def get(self, key): + return self.client.get_object(Bucket=S3_BUCKET, Key=key) + + def delete(self, key): + self.client.delete_object(Bucket=S3_BUCKET, Key=key) + + def objects(self, prefix=PREFIX, counters=None): + pages = self.client.get_paginator('list_objects_v2') + for page in pages.paginate(Bucket=S3_BUCKET, Prefix=prefix): + if counters is not None: + counters['list_requests'] += 1 + yield from page.get('Contents', []) + + +def lock_attachment(db, attachment_id, wait=True): + function = 'pg_advisory_lock' if wait else 'pg_try_advisory_lock' + return db.execute(f'SELECT {function}(hashtextextended(%s,0)) AS locked', + ('attachment:' + str(attachment_id),)).fetchone()['locked'] + + +def unlock_attachment(db, attachment_id): + db.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))', + ('attachment:' + str(attachment_id),)) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 0000000..7943e13 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,87 @@ +"""Контракт файлов: реальные изображения, лимиты, ACL и ключи сообщений.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import http.client +import io +from pathlib import Path +import struct +import sys +from urllib.parse import quote, urlsplit +import uuid +import zlib + +from PIL import Image +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Attachment negative and concurrent checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/attachments?filename=' + png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + headers = {'Content-Type': 'image/png'} + att = c.call('POST', path+'sample.png', 201, actor=alice, raw=png, headers=headers, schema='Attachment')['json'] + check('object_key' not in att and 'url' not in att, 'private storage details leaked') + for filename in ['../a.png', 'a\\b.png', 'x\x00.png', 'x\n.png', '..', ' ']: + c.call('POST', path+quote(filename, safe=''), 400, actor=alice, raw=png, headers=headers) + for data, mime in [(b'not an image', 'image/png'), (png[:30], 'image/png'), + (png[:-12], 'image/png'), (png, 'image/jpeg'), (png, 'application/octet-stream')]: + c.call('POST', path+'invalid.png', 415, actor=alice, raw=data, headers={'Content-Type': mime}) + jpeg = io.BytesIO() + Image.new('RGB', (64, 64), 'red').save(jpeg, format='JPEG') + data = jpeg.getvalue() + j = c.call('POST', path+'real.jpg', 201, actor=alice, raw=data, + headers={'Content-Type': 'image/jpeg'}, schema='Attachment')['json'] + downloaded = c.call('GET', '/api/v1/attachments/'+j['id']+'/content', actor=bob) + check(downloaded['bytes'] == data, 'JPEG bytes changed') + c.call('POST', path+'broken.jpg', 415, actor=alice, raw=data[:-20], headers={'Content-Type': 'image/jpeg'}) + # Валидный IHDR заявляет >40M пикселей. Отклонение происходит до декодирования. + ihdr = struct.pack('>IIBBBBB', 9000, 5000, 8, 2, 0, 0, 0) + header = b'\x89PNG\r\n\x1a\n' + struct.pack('>I', len(ihdr)) + b'IHDR' + ihdr + header += struct.pack('>I', zlib.crc32(b'IHDR'+ihdr)) + header += png[33:] + c.call('POST', path+'huge-pixels.png', 413, actor=alice, raw=header, headers=headers) + # Отправляем только headers заведомо большого файла: доказываем ранний byte limit. + base = urlsplit(c.base) + connection = http.client.HTTPConnection(base.hostname, base.port or 80, timeout=10) + connection.putrequest('POST', path+'too-big.png') + connection.putheader('Authorization', 'Bearer '+alice['token']) + connection.putheader('Content-Type', 'image/png') + connection.putheader('Content-Length', str(10485761)) + connection.endheaders() + response = connection.getresponse() + check(response.status == 413, 'oversized body was not rejected before reading') + response.read(); connection.close() + c.call('POST', path+'outsider.png', 404, actor=eve, raw=png, headers=headers) + for suffix in ['', '/content']: + c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob) + other_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json'] + message_path = '/api/v1/chats/'+other_chat['id']+'/messages' + c.call('POST', message_path, 404, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + message_path = '/api/v1/chats/'+chat['id']+'/messages' + for ids, code in [([str(uuid.uuid4())], 404), ([att['id'], att['id']], 400), ([att['id']]*11, 400)]: + c.call('POST', message_path, code, actor=alice, body={'text': '', 'attachment_ids': ids}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + key = uuid.uuid4().hex + def send(_): + return Client(args.base_url).message(bob, chat, '', key=key, attachments=[att['id']]) + with ThreadPoolExecutor(max_workers=8) as pool: + replies = list(pool.map(send, range(16))) + check(all(r['json'] == replies[0]['json'] for r in replies), 'attachment replay created different messages') + check(len({r['headers']['X-Instance-Id'] for r in replies}) >= 2, 'attachment race did not cross replicas') + # Отсутствие attachment_ids и [] одинаковы и для хеша повтора. + text_key = uuid.uuid4().hex + text_message = c.message(alice, chat, 'Только текст', key=text_key)['json'] + check(c.message(alice, chat, 'Только текст', key=text_key, attachments=[])['json'] == text_message, + 'default [] changes key semantics') + check(att['sha256'] == hashlib.sha256(png).hexdigest(), 'wrong original checksum') + print('PASS: PNG/JPEG, bytes/pixels limits, filename/MIME, ACL, attachment message replay') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/availability.py b/student-tests/availability.py new file mode 100644 index 0000000..5fe380a --- /dev/null +++ b/student-tests/availability.py @@ -0,0 +1,64 @@ +"""Каждая реплика отдельно и поочерёдная замена под одним профилем нагрузки.""" +import json +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from scripts.benchmark import context, measure, verify_confirmations +from scripts.stack import containers, stop, wait, command, rolling, save +from scripts.monitor import Monitor + + +def main(): + base = os.getenv('BASE_URL', 'http://localhost:8080') + aliases = wait(2, base) + results = [] + for target in aliases: + ctx = context(base) + # Сохраняем подтверждённую запись до отказа вместе с подтверждениями нагрузки. + from scripts.benchmark import Client + original = Client(base).message(ctx['actor'], ctx['chat'], 'До отказа')['json'] + attachment, content = Client(base).attachment(ctx['actor'], ctx['chat']) + try: + with Monitor() as monitor: + report, confirmations = measure(base, ctx, 16, 4, + action=lambda: stop(target), retry_writes=True) + event = report['action_events'][0] + successful_times = sorted(int(x) for x, count in report['successful_per_second'].items() if count) + first = next((x for x in successful_times if x > event['end_seconds']), None) + if first is None or first+1-event['start_seconds'] > 10: + raise AssertionError('Recovery exceeded 10 seconds') + stopped_host = target.split(':')[0] + if any(r['running'] for r in containers('api') if r['hostname'] == stopped_host): + raise AssertionError('Target is still running') + report['first_successful_bucket_after_stop_seconds'] = first + report['recovery_upper_bound_seconds'] = round(first+1-event['start_seconds'], 3) + report['verification'] = verify_confirmations(base, ctx, [original, *confirmations]) + assert Client(base).call('GET', '/api/v1/attachments/'+attachment['id']+'/content', + actor=ctx['reader'])['bytes'] == content + report['attachment_original_verified'] = True + report['monitor'] = monitor.samples + report['target'] = target + results.append(report) + save('failures.json', results) + finally: + command(['docker', 'compose', 'start', 'api']) + wait(2, base) + ctx = context(base) + attachment, content = Client(base).attachment(ctx['actor'], ctx['chat']) + original = Client(base).message(ctx['actor'], ctx['chat'], '', attachments=[attachment['id']])['json'] + with Monitor() as monitor: + report, confirmations = measure(base, ctx, 35, 4, action=rolling, retry_writes=True) + report['verification'] = verify_confirmations(base, ctx, [original, *confirmations]) + assert Client(base).call('GET', '/api/v1/attachments/'+attachment['id']+'/content', + actor=ctx['reader'])['bytes'] == content + report['attachment_original_verified'] = True + report['monitor'] = monitor.samples + save('rolling.json', report) + print('PASS: both API failures and rolling replacement, confirmations preserved') + + +if __name__ == '__main__': + main() diff --git a/student-tests/checks.py b/student-tests/checks.py new file mode 100644 index 0000000..57c5347 --- /dev/null +++ b/student-tests/checks.py @@ -0,0 +1,90 @@ +"""Дополнения к публичному smoke. Не печатает и не сохраняет tokens.""" +import base64 +import concurrent.futures +import json +import sys +import time +from pathlib import Path +from uuid import uuid4 +from urllib.parse import quote + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, arguments, check, run + + +def check_cursor_types(c, actor, path, cursor, schema): + check(bool(cursor), 'test requires a real pagination cursor') + payload = json.loads(base64.urlsafe_b64decode(cursor + '=' * (-len(cursor) % 4))) + c.call('GET', path + '?cursor=' + quote(cursor, safe=''), actor=actor, schema=schema) + invalid = [] + for field in ['created_at', 'id']: + for value in [1, 1.5, None, True, [], {}, '', 'invalid']: + invalid.append({**payload, field: value}) + invalid.extend([None, 1, True, [], 'cursor']) + for value in invalid: + encoded = base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip('=') + instances = set() + # Проверяем один и тот же cursor на обеих API за round-robin LB. + for _ in range(8): + response = c.call('GET', path + '?cursor=' + quote(encoded, safe=''), 400, actor=actor) + check(response['json']['error']['code'] == 'invalid_request', 'incorrect cursor error code') + instances.add(response['headers']['X-Instance-Id']) + if len(instances) >= 2: + break + check(len(instances) >= 2, 'invalid cursor was not checked on two API instances') + + +def main(): + def configure(p): + p.add_argument('--expiry', action='store_true', help='API must run with TTL=2') + args = arguments('Student API checks', configure) + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + if args.expiry: + time.sleep(2.2) + c.call('GET', '/api/v1/users/me', 401, actor=alice) + print('PASS: short TTL expiry') + return + registration = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Гонка', + 'password': 'Synthetic-password-42'} + # Для гонки регистрируем оба допустимых исхода через urllib. + import urllib.request + import urllib.error + def register(_): + req = urllib.request.Request(c.base + '/api/v1/users', + data=json.dumps(registration).encode(), + headers={'Content-Type': 'application/json'}) + try: + with urllib.request.urlopen(req, timeout=10) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + codes = list(executor.map(register, range(8))) + check(codes.count(201) == 1 and codes.count(409) == 7, 'username race failed') + path = '/api/v1/chats/' + chat['id'] + c.call('POST', '/api/v1/chats', 400, actor=alice, + body={'title': 'unknown member', 'member_ids': [alice['id'], str(uuid4())]}) + for body in [{'text': 'x', 'sender_id': eve['id']}, {'text': 4}, {'text': ' '}]: + c.call('POST', path + '/messages', 400, actor=alice, body=body) + c.call('GET', path + '/messages?limit=true', 400, actor=alice) + c.call('GET', path + '/messages?cursor=not-a-cursor', 400, actor=alice) + for n in range(3): + c.message(alice, chat, 'read-after-write ' + str(n)) + page = c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json'] + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json'] + c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + + quote(page['next_cursor'], safe=''), 400, actor=alice) + check_cursor_types(c, alice, path + '/messages', page['next_cursor'], 'MessagePage') + chats = c.call('GET', '/api/v1/chats?limit=1', actor=alice, schema='ChatPage')['json'] + check_cursor_types(c, alice, '/api/v1/chats', chats['next_cursor'], 'ChatPage') + history = c.call('GET', path + '/messages', actor=bob, schema='MessagePage')['json']['items'] + check(len(history) == 3 and history[-1]['text'] == 'read-after-write 2', 'read-after-write failed') + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + c.call('GET', '/api/v1/users/me', 401, actor=alice) + print('PASS: registration race, invalid input, cursor scope/types on two API, read-after-write, logout') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/database.py b/student-tests/database.py new file mode 100644 index 0000000..1a37580 --- /dev/null +++ b/student-tests/database.py @@ -0,0 +1,61 @@ +"""Проверка реального rollback после вставки чата и первого участника.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from uuid import uuid4 +import psycopg +import os +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from src.config import DATABASE_URL +from src.migrate import migrate + +migrate() +migrate() # Повтор миграции не создаёт дубликаты. +with psycopg.connect(DATABASE_URL) as db: + user = db.execute('SELECT id FROM users LIMIT 1').fetchone() + assert user, 'Сначала выполните smoke, чтобы создать синтетического пользователя' + chat = uuid4() + try: + with db.transaction(): + db.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (chat, 'rollback test')) + db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, user[0])) + # FK прерывает составную операцию ПОСЛЕ двух успешных INSERT. + db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat, uuid4())) + except psycopg.errors.ForeignKeyViolation: + pass + else: + raise AssertionError('Expected foreign-key failure') + assert db.execute('SELECT count(*) FROM chats WHERE id=%s', (chat,)).fetchone()[0] == 0 + assert db.execute('SELECT count(*) FROM chat_members WHERE chat_id=%s', (chat,)).fetchone()[0] == 0 + assert db.execute('SELECT count(*) FROM schema_migrations').fetchone()[0] == 3 +print('PASS: rollback of chat + members; migration reapplied twice') + +c = Client(os.getenv('BASE_URL', 'http://localhost:8080')) +alice, bob, _, chat = c.bootstrap() +key = uuid4().hex +original = c.message(alice, chat, 'До истечения ключа', key=key)['json'] +with psycopg.connect(DATABASE_URL) as db: + # Без ожидания суток: сдвигаем именно persisted expiry, а не clock приложения. + db.execute("UPDATE message_requests SET expires_at=now()-interval '1 second' " + "WHERE chat_id=%s AND request_key=%s", (chat['id'], key)) +renewed = c.message(alice, chat, 'После истечения ключа', key=key)['json'] +assert renewed['id'] != original['id'] +with psycopg.connect(DATABASE_URL) as db: + assert db.execute('SELECT count(*) FROM message_requests WHERE chat_id=%s AND request_key=%s', + (chat['id'], key)).fetchone()[0] == 1 +with psycopg.connect(DATABASE_URL) as db: + message = uuid4() + try: + with db.transaction(): + db.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)', + (message, chat['id'], bob['id'], 'atomic rollback')) + # Ошибка сохранения результата должна откатить и сообщение. + db.execute("INSERT INTO message_requests VALUES (%s,%s,%s,%s,%s,now())", + (uuid4(), chat['id'], 'rollback', '0'*64, psycopg.types.json.Jsonb({}))) + except psycopg.errors.ForeignKeyViolation: + pass + else: + raise AssertionError('Expected result-store failure') + assert db.execute('SELECT count(*) FROM messages WHERE id=%s', (message,)).fetchone()[0] == 0 +print('PASS: persisted key expiry, atomic message/result rollback') diff --git a/student-tests/dependencies.py b/student-tests/dependencies.py new file mode 100644 index 0000000..5d96d21 --- /dev/null +++ b/student-tests/dependencies.py @@ -0,0 +1,71 @@ +"""Redis необязателен для корректности; без PostgreSQL готовность снимается.""" +import os +from pathlib import Path +import sys +import time +import urllib.request +import urllib.error +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from scripts.benchmark import context, Client +from scripts.stack import command, wait, save, containers + +base = os.getenv('BASE_URL', 'http://localhost:8080') +c = Client(base) +ctx = context(base) +original = c.message(ctx['actor'], ctx['chat'], 'До отказов')['json'] +attachment, content = c.attachment(ctx['actor'], ctx['chat']) +results = {} +try: + command(['docker', 'compose', 'stop', 'cache']) + started = time.monotonic() + for _ in range(6): + c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User') + c.message(ctx['actor'], ctx['chat'], 'Без Redis') + results['redis'] = dict(status='PASS', elapsed_seconds=time.monotonic()-started) +finally: + command(['docker', 'compose', 'up', '-d', '--wait', 'cache']) +try: + command(['docker', 'compose', 'stop', 'db']) + # Даём LB время исключить все неготовые API. Его собственный 503 может быть HTML. + time.sleep(4) + codes = [] + for path in ['/health/ready', '/api/v1/users/me']: + request = urllib.request.Request(base+path, headers={'Authorization': 'Bearer '+ctx['actor']['token']}) + try: + with urllib.request.urlopen(request, timeout=8) as response: + codes.append(response.status) + except urllib.error.HTTPError as exc: + codes.append(exc.code) + assert codes == [503, 503], codes + results['postgres'] = dict(status='PASS', readiness_and_request_status=codes) +finally: + command(['docker', 'compose', 'up', '-d', '--wait', 'db']) + wait(2, base) +active = any(row['running'] for row in containers('reconciler')) +command(['docker', 'compose', 'stop', 'reconciler']) +try: + command(['docker', 'compose', 'stop', 's3']) + time.sleep(4) + codes = [] + paths = ['/health/ready', '/api/v1/attachments/'+attachment['id']+'/content'] + for path in paths: + request = urllib.request.Request(base+path, headers={'Authorization': 'Bearer '+ctx['actor']['token']}) + try: + with urllib.request.urlopen(request, timeout=10) as response: + codes.append(response.status) + except urllib.error.HTTPError as exc: + codes.append(exc.code) + assert codes == [503, 503], codes + results['s3'] = dict(status='PASS', readiness_and_download_status=codes) +finally: + command(['docker', 'compose', 'up', '-d', '--wait', 's3']) + wait(2, base) + if active: + command(['docker', 'compose', 'start', 'reconciler']) +page = c.call('GET', '/api/v1/chats/'+ctx['chat']['id']+'/messages', actor=ctx['reader'], schema='MessagePage')['json'] +assert original in page['items'] +c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User') +assert c.call('GET', '/api/v1/attachments/'+attachment['id']+'/content', actor=ctx['reader'])['bytes'] == content +results['recovery'] = 'old session and confirmed message preserved' +save('dependencies.json', results) +print('PASS: Redis fallback, PostgreSQL/S3 unready, old session and original preserved') diff --git a/student-tests/file_failover.py b/student-tests/file_failover.py new file mode 100644 index 0000000..8219bd1 --- /dev/null +++ b/student-tests/file_failover.py @@ -0,0 +1,43 @@ +"""Останавливаем именно API, принявшую upload, и скачиваем оригинал другой репликой.""" +import hashlib +import os +from pathlib import Path +import sys +import time +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, Failure +from scripts.stack import stop, wait, save, command + +base = os.getenv('BASE_URL', 'http://localhost:8080') +c = Client(base, timeout=3) +alice, bob, _, chat = c.bootstrap() +data = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() +response = c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') +target = response['headers']['X-Instance-Id'] +att = response['json'] +c.message(alice, chat, '', attachments=[att['id']]) +started = time.monotonic() +errors = 0 +try: + stopped = stop(target) + while True: + try: + downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob) + alias = downloaded['headers']['X-Instance-Id'] + assert alias != target and downloaded['bytes'] == data + assert hashlib.sha256(downloaded['bytes']).hexdigest() == att['sha256'] + break + except Failure: + errors += 1 + if time.monotonic()-started > 10: + raise + time.sleep(.2) + save('attachment_failover.json', dict(status='PASS', uploader=target, downloader=alias, + recovery_seconds=time.monotonic()-started, failed_attempts=errors, + original_size=len(data), checksum_verified=True, stop=stopped)) + print('PASS: upload API stopped, original downloaded by other API with existing token') +finally: + command(['docker', 'compose', 'start', 'api']) + wait(2, base) diff --git a/student-tests/idempotency.py b/student-tests/idempotency.py new file mode 100644 index 0000000..bd99106 --- /dev/null +++ b/student-tests/idempotency.py @@ -0,0 +1,58 @@ +"""Проверки общего результата повтора через LB и несколько API.""" +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Shared idempotency through LB') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + key = uuid.uuid4().hex + text = 'Один результат для конкурентных повторов' + + def send(_): + return Client(args.base_url, args.ca_file).message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(send, range(32))) + first = responses[0]['json'] + check(all(r['json'] == first and r['status'] == 201 for r in responses), 'replay differs') + instances = {r['headers']['X-Instance-Id'] for r in responses} + check(len(instances) >= 2, 'race did not cross API replicas') + path = '/api/v1/chats/' + chat['id'] + '/messages' + history = c.call('GET', path, actor=bob, schema='MessagePage')['json']['items'] + check(len(history) == 1 and history[0] == first, 'concurrent replay created duplicates') + c.call('POST', path, 409, body={'text': 'Другой текст'}, actor=alice, + headers={'Idempotency-Key': key}) + for invalid in ['', 'bad key', 'a' * 129]: + c.call('POST', path, 400, body={'text': text}, actor=alice, + headers={'Idempotency-Key': invalid}) + c.call('POST', path, 400, body={'text': text}, actor=alice) + # Тот же ключ независим для другого пользователя и другого чата. + other = c.message(bob, chat, text, key=key)['json'] + check(other['id'] != first['id'], 'key scope must include user') + chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Second', 'member_ids': [alice['id']]}, schema='Chat')['json'] + check(c.message(alice, chat2, text, key=key)['json']['id'] != first['id'], 'key scope must include chat') + # Пробелы JSON не изменяют смысл текста и не должны менять результат. + replay = c.call('POST', path, 201, actor=alice, raw=json.dumps({'text': text}, indent=2).encode(), + headers={'Content-Type': 'application/json', 'Idempotency-Key': key}, schema='Message') + check(replay['json'] == first, 'JSON formatting changed idempotency') + c.call('POST', path, 404, actor=eve, body={'text': text}, headers={'Idempotency-Key': key}) + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + c.call('POST', path, 401, actor=alice, body={'text': text}, headers={'Idempotency-Key': key}) + result = dict(status='PASS', concurrent_requests=32, observed_instances=len(instances), + messages_after_race=len(history), conflict=409, invalid_key=400, logout=401) + Path('evidence').mkdir(exist_ok=True) + Path('evidence/idempotency.json').write_text(json.dumps(result, indent=2)) + print('PASS: shared idempotency, scope, conflict and auth on replay') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/storage_scenario.py b/student-tests/storage_scenario.py new file mode 100644 index 0000000..7ddd975 --- /dev/null +++ b/student-tests/storage_scenario.py @@ -0,0 +1,237 @@ +"""Настоящие fault injection в PostgreSQL и S3, затем dry-run/apply/повтор.""" +import argparse +import base64 +from datetime import datetime, timezone +import hashlib +import http.client +import io +import json +import os +from pathlib import Path +import sys +import time +import urllib.request +import urllib.error +import uuid + +import psycopg +from psycopg.rows import dict_row +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from src.config import DATABASE_URL, S3_ENDPOINT, S3_BUCKET +from src.storage import ObjectStore, PREFIX, lock_attachment, unlock_attachment +from src.reconcile import run + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('mode', choices=['privacy', 'interrupted', 's3-failure', 'reconcile', 'cost']) + p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000]) + p.add_argument('--failed-chat-id', type=uuid.UUID) + args = p.parse_args() + c = Client(os.getenv('BASE_URL', 'http://localhost:8080'), timeout=30) + alice, bob, _, chat = c.bootstrap() + data = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + digest = hashlib.sha256(data).hexdigest() + md5 = base64.b64encode(hashlib.md5(data).digest()).decode() + storage = ObjectStore() + records = {} + + def scan(grace=20, apply=False): + return run(grace, apply, chat['id']) + + def put(key): + storage.put(io.BytesIO(data), key, len(data), 'image/png', digest, md5) + + def upload(name, code=201): + return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+name, + code, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, + schema='Attachment' if code == 201 else None) + + def row(identifier): + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db: + return db.execute('SELECT * FROM attachments WHERE id=%s', (identifier,)).fetchone() + + def seed(status='ready', active=False): + identifier = uuid.uuid4() + key = PREFIX + chat['id'] + '/' + str(identifier) + put(key) + with psycopg.connect(DATABASE_URL) as db: + db.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256," + "object_key,status,lease_until,created_at) VALUES (%s,%s,%s,'synthetic.png','image/png'," + "%s,%s,%s,%s,clock_timestamp()+%s*interval '1 second',clock_timestamp()-interval '15 minutes')", + (identifier, chat['id'], alice['id'], len(data), digest, key, status, 120 if active else -900)) + return dict(id=identifier, key=key) + + if args.mode == 'interrupted': + from urllib.parse import urlsplit + base = urlsplit(c.base) + connection = http.client.HTTPConnection(base.hostname, base.port or 80, timeout=5) + connection.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png') + connection.putheader('Authorization', 'Bearer '+alice['token']) + connection.putheader('Content-Type', 'image/png') + connection.putheader('Content-Length', '4096') + connection.endheaders() + connection.send(data[:16]) + connection.close() # Не отправляем остаток и не завершаем корректный HTTP body. + time.sleep(.5) + with psycopg.connect(DATABASE_URL) as db: + count = db.execute('SELECT count(*) FROM attachments WHERE chat_id=%s', (chat['id'],)).fetchone()[0] + objects = list(storage.objects(PREFIX+chat['id']+'/')) + assert count == 0 and not objects + print(json.dumps(dict(status='PASS', sent_bytes=16, declared_bytes=4096, metadata=0, objects=0))) + return + + if args.mode == 'privacy': + original = upload('private.png')['json'] + key = row(original['id'])['object_key'] + try: + urllib.request.urlopen(S3_ENDPOINT+'/'+S3_BUCKET+'/'+key, timeout=5) + except urllib.error.HTTPError as exc: + assert exc.code == 403 + else: + raise AssertionError('Anonymous object GET succeeded') + buckets = storage.client.list_buckets().get('Buckets', []) + assert all(bucket['Name'] == S3_BUCKET for bucket in buckets), 'Foreign bucket listed' + # MinIO может вернуть отфильтрованный ListBuckets, а не AccessDenied. + try: + storage.client.create_bucket(Bucket='forbidden-'+uuid.uuid4().hex) + except Exception as exc: + assert getattr(exc, 'response', {}).get('Error', {}).get('Code') == 'AccessDenied' + else: + raise AssertionError('API credential has bucket administration rights') + from urllib.parse import urlsplit + endpoint = urlsplit(S3_ENDPOINT) + console = f'{endpoint.scheme}://{endpoint.hostname}:9001/' + try: + urllib.request.urlopen(console, timeout=2) + except urllib.error.URLError: + pass + else: + raise AssertionError('Admin console is reachable from another container') + print(json.dumps(dict(status='PASS', anonymous_get=403, bucket_list='filtered', create_bucket='AccessDenied', + console='not reachable', attachment_id=original['id']))) + return + + if args.mode == 's3-failure': + # Оператор временно установил deny PUT policy; GET/LIST/readiness доступны. + upload('fault-s3.png', 503) + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db: + failed = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-s3.png'", + (chat['id'],)).fetchone() + assert failed and failed['status'] == 'uploading' + assert storage.head(failed['object_key']) is None + print(json.dumps(dict(status='PASS', http_status=503, metadata='uploading', object='absent', + chat_id=chat['id'], attachment_id=str(failed['id'])))) + return + + if args.mode == 'cost': + results, made = [], 0 + for size in args.sizes: + while made < size: + seed() + made += 1 + scan(600, False) # Прогрев обхода. + for repeat in range(3): + report = scan(600, False) + report.update(dataset_added=size, repetition=repeat+1) + results.append(report) + print(json.dumps(dict(status='PASS', scans=results))) + return + + # Используемое вложение: сообщение и готовый объект должны пережить очистку. + if args.failed_chat_id: + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db: + failure = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-s3.png'", + (args.failed_chat_id,)).fetchone() + assert failure and failure['status'] == 'uploading' + db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes'," + "lease_until=clock_timestamp()-interval '15 minutes' WHERE id=%s", (failure['id'],)) + dry_failure = run(20, False, args.failed_chat_id) + assert row(failure['id']) + fixed_failure = run(20, True, args.failed_chat_id) + assert row(failure['id']) is None and storage.head(failure['object_key']) is None + records['s3_failure_compensation'] = dict(dry_run=dry_failure, apply=fixed_failure, metadata_removed=True) + used = upload('used.png')['json'] + message = c.message(bob, chat, '', attachments=[used['id']])['json'] + used_row = row(used['id']) + with psycopg.connect(DATABASE_URL) as db: + db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s", + (used['id'],)) + + # Ошибка UPDATE ready ПОСЛЕ успешного S3 PUT. Только тестовое имя файла. + with psycopg.connect(DATABASE_URL) as db: + db.execute("CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS $$ " + "BEGIN IF NEW.filename='fault-db.png' AND NEW.status='ready' THEN " + "RAISE EXCEPTION 'synthetic DB failure'; END IF; RETURN NEW; END; $$") + db.execute('CREATE TRIGGER lab4_test_fail BEFORE UPDATE ON attachments ' + 'FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()') + try: + upload('fault-db.png', 503) + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as db: + failed = db.execute("SELECT * FROM attachments WHERE chat_id=%s AND filename='fault-db.png'", + (chat['id'],)).fetchone() + assert failed and failed['status'] == 'uploading' and storage.head(failed['object_key']) + immediate = scan(20, True) + assert row(failed['id']) and storage.head(failed['object_key']) + records['after_put_db_failure'] = dict(http_status=503, metadata='uploading', object='present', + immediate_cleanup=immediate) + with psycopg.connect(DATABASE_URL) as db: + db.execute("UPDATE attachments SET lease_until=clock_timestamp()-interval '15 minutes'," + "created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s", (failed['id'],)) + finally: + with psycopg.connect(DATABASE_URL) as db: + db.execute('DROP TRIGGER IF EXISTS lab4_test_fail ON attachments') + db.execute('DROP FUNCTION IF EXISTS lab4_fail_ready()') + + missing = upload('missing.png')['json'] + missing_message = c.message(alice, chat, '', attachments=[missing['id']])['json'] + missing_row = row(missing['id']) + storage.delete(missing_row['object_key']) + with psycopg.connect(DATABASE_URL) as db: + db.execute("UPDATE attachments SET created_at=clock_timestamp()-interval '15 minutes' WHERE id=%s", + (missing['id'],)) + + orphan_key = PREFIX + chat['id'] + '/' + str(uuid.uuid4()) + put(orphan_key) + active = seed('uploading', active=True) + locked = seed('uploading') + time.sleep(20.2) # Настоящий возраст объекта S3, не фиктивный LastModified. + fresh_key = PREFIX + chat['id'] + '/' + str(uuid.uuid4()) + put(fresh_key) + fresh = upload('fresh.png')['json'] + with psycopg.connect(DATABASE_URL, row_factory=dict_row) as guard: + lock_attachment(guard, locked['id']); guard.commit() + try: + dry = scan(20, False) + assert storage.head(orphan_key) and row(failed['id']) + assert row(missing['id'])['status'] == 'ready' + applied = scan(20, True) + assert storage.head(orphan_key) is None + assert row(failed['id']) is None and storage.head(failed['object_key']) is None + assert row(missing['id'])['status'] == 'missing' + assert row(used['id']) and storage.head(used_row['object_key']) + assert row(active['id']) and storage.head(active['key']) + assert row(locked['id']) and storage.head(locked['key']) + assert storage.head(fresh_key) and row(fresh['id']) + c.call('GET', '/api/v1/attachments/'+missing['id']+'/content', 503, actor=bob) + # После возврата именно исходных bytes reconciler может вернуть ready. + put(missing_row['object_key']) + repaired = scan(20, True) + assert row(missing['id'])['status'] == 'ready' + assert c.call('GET', '/api/v1/attachments/'+missing['id']+'/content', actor=bob)['bytes'] == data + repeated = scan(20, True) + assert not repeated['actions'], 'Repeat still changes already corrected state' + page = c.call('GET', '/api/v1/chats/'+chat['id']+'/messages', actor=bob, schema='MessagePage')['json'] + assert message in page['items'] and missing_message in page['items'] + finally: + unlock_attachment(guard, locked['id']); guard.commit() + records.update(status='PASS', dry_run=dry, apply=applied, restore=repaired, repeat=repeated, + used_and_fresh_preserved=True, active_lease_and_lock_preserved=True, + missing_original_restored_by_operator=True) + print(json.dumps(records)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..19ee622 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Black-box grading evidence beyond the public smoke test.""" +from concurrent.futures import ThreadPoolExecutor +import json +import sys +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def collect_messages(client, actor, chat, page_size=17): + items = [] + cursor = None + while True: + path = f"/api/v1/chats/{chat['id']}/messages?limit={page_size}" + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + + +def main(): + args = arguments("Student negative, pagination and concurrency tests") + client = Client(args.base_url, args.ca_file) + alice, bob, outsider, chat = client.bootstrap() + + client.call( + "POST", + "/api/v1/users", + 400, + raw=b'{"username":', + headers={"Content-Type": "application/json"}, + ) + client.call( + "POST", + "/api/v1/users", + 415, + raw=b'{"username":"wrong_media","display_name":"Wrong"}', + headers={"Content-Type": "text/plain"}, + ) + client.call("POST", "/api/v1/users", 409, body=alice["registration"]) + client.call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + actor=alice, + body={"text": " \t"}, + ) + client.call("GET", f"/api/v1/chats/{chat['id']}", 404, actor=outsider) + + prefix = "race-" + uuid.uuid4().hex + "-" + + def send(worker_id): + local = Client(args.base_url, args.ca_file) + return [ + local.message(alice, chat, f"{prefix}{worker_id}-{index}")["json"] + for index in range(10) + ] + + with ThreadPoolExecutor(max_workers=10) as pool: + created = [message for batch in pool.map(send, range(10)) for message in batch] + + check(len(created) == 100, "not all concurrent requests succeeded") + check(len({message["id"] for message in created}) == 100, "duplicate message ids returned") + + history = collect_messages(client, bob, chat) + concurrent_history = [message for message in history if message["text"].startswith(prefix)] + check(len(concurrent_history) == 100, "history does not contain exactly 100 concurrent messages") + check(len({message["id"] for message in concurrent_history}) == 100, "history contains duplicates") + check( + history == sorted(history, key=lambda message: (message["created_at"], uuid.UUID(message["id"]).int)), + "multi-page history is out of order", + ) + + print("PASS: malformed JSON/media type, duplicate username, blank message, outsider access, pagination") + print("PASS: 10 clients created exactly 100 unique messages") + + +if __name__ == "__main__": + run(main)