From 96007f4c5a5c7772766e82b31d8e38764f2afa36 Mon Sep 17 00:00:00 2001 From: mayorovasonya <112+mayorovasonya@noreply.gitea.privcloud.tech> Date: Tue, 27 Oct 2026 13:17:07 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9B=D0=A0=204:=20S3-=D0=B2=D0=BB=D0=BE?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B8=20=D0=B8=D1=81=D1=81?= =?UTF-8?q?=D0=BB=D0=B5=D0=B4=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D1=81?= =?UTF-8?q?=D0=BE=D0=B3=D0=BB=D0=B0=D1=81=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B8=20=E2=80=94=20=D0=9C=D0=B0=D0=B9=D0=BE?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=20=D0=91=D0=92=D0=A22301?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 25 +- .github/workflows/application.yml | 26 + Dockerfile | 13 + Makefile | 11 + REPORT.md | 227 ++- compose.yaml | 185 ++ evidence/bucket-bootstrap-repeat.txt | 3 + evidence/build.txt | 55 + evidence/clean-clone.txt | 220 +++ evidence/failover-recovery.txt | 7 + evidence/faults-initial-stderr.txt | 6 + evidence/faults-stderr.txt | 12 + evidence/faults.json | 0 evidence/final-tests.txt | 31 + evidence/openapi-validation.txt | 1 + evidence/persistence.txt | 3 + evidence/public-failover.txt | 4 + evidence/public-smoke.txt | 2 + evidence/rebuild-startup.txt | 12 + evidence/reconciler-default-stderr.txt | 2 + evidence/reconciler-default.json | 20 + evidence/reconciliation-stderr.txt | 2 + evidence/reconciliation.json | 332 ++++ evidence/research.png | Bin 0 -> 88676 bytes evidence/research.svg | 2391 +++++++++++++++++++++++ evidence/restart-after.json | 284 +++ evidence/restart-before.json | 284 +++ evidence/runtime.json | 284 +++ evidence/slow-upload-initial-stderr.txt | 4 + evidence/slow-upload.json | 1 + evidence/startup.txt | 64 + evidence/stopped-instances.jsonl | 1 + evidence/storage-security-stderr.txt | 2 + evidence/storage-security.json | 14 + evidence/student-api.txt | 20 + evidence/student-attachments.txt | 11 + evidence/uploads-initial-s3-state.json | 1 + evidence/uploads-initial-stderr.txt | 41 + evidence/uploads-second-api-state.json | 1 + evidence/uploads-second-stderr.txt | 70 + evidence/uploads-stderr.txt | 30 + evidence/uploads.json | 475 +++++ infra/haproxy.cfg | 46 + migrations/001_initial.sql | 42 + migrations/002_unicode_nul.sql | 17 + migrations/003_idempotency.sql | 12 + migrations/004_attachments.sql | 30 + requirements-research.txt | 2 + requirements.txt | 12 + scripts/bootstrap.py | 35 + scripts/restart.sh | 9 + scripts/snapshot.py | 11 + scripts/stop-one.sh | 4 + scripts/stop_one.py | 32 + scripts/wait_ready.py | 16 + src/bucket_bootstrap.py | 13 + src/common.py | 98 + src/migrate.py | 13 + src/objects.py | 156 ++ src/postgres.py | 399 ++++ src/reconcile.py | 129 ++ src/server.py | 290 +++ student-tests/faults.py | 125 ++ student-tests/images.py | 24 + student-tests/plot.py | 36 + student-tests/research.py | 123 ++ student-tests/slow_upload.py | 32 + student-tests/storage_checks.py | 69 + student-tests/test_api.py | 177 ++ student-tests/test_attachments.py | 83 + student-tests/uploads.py | 75 + 71 files changed, 7246 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/bucket-bootstrap-repeat.txt create mode 100644 evidence/build.txt create mode 100644 evidence/clean-clone.txt create mode 100644 evidence/failover-recovery.txt create mode 100644 evidence/faults-initial-stderr.txt create mode 100644 evidence/faults-stderr.txt create mode 100644 evidence/faults.json create mode 100644 evidence/final-tests.txt create mode 100644 evidence/openapi-validation.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/public-failover.txt create mode 100644 evidence/public-smoke.txt create mode 100644 evidence/rebuild-startup.txt create mode 100644 evidence/reconciler-default-stderr.txt create mode 100644 evidence/reconciler-default.json create mode 100644 evidence/reconciliation-stderr.txt create mode 100644 evidence/reconciliation.json create mode 100644 evidence/research.png create mode 100644 evidence/research.svg create mode 100644 evidence/restart-after.json create mode 100644 evidence/restart-before.json create mode 100644 evidence/runtime.json create mode 100644 evidence/slow-upload-initial-stderr.txt create mode 100644 evidence/slow-upload.json create mode 100644 evidence/startup.txt create mode 100644 evidence/stopped-instances.jsonl create mode 100644 evidence/storage-security-stderr.txt create mode 100644 evidence/storage-security.json create mode 100644 evidence/student-api.txt create mode 100644 evidence/student-attachments.txt create mode 100644 evidence/uploads-initial-s3-state.json create mode 100644 evidence/uploads-initial-stderr.txt create mode 100644 evidence/uploads-second-api-state.json create mode 100644 evidence/uploads-second-stderr.txt create mode 100644 evidence/uploads-stderr.txt create mode 100644 evidence/uploads.json create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_unicode_nul.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 requirements-research.txt create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100755 scripts/restart.sh create mode 100644 scripts/snapshot.py create mode 100755 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/wait_ready.py create mode 100644 src/bucket_bootstrap.py create mode 100644 src/common.py create mode 100644 src/migrate.py create mode 100644 src/objects.py create mode 100644 src/postgres.py create mode 100644 src/reconcile.py create mode 100644 src/server.py create mode 100644 student-tests/faults.py create mode 100644 student-tests/images.py create mode 100644 student-tests/plot.py create mode 100644 student-tests/research.py create mode 100644 student-tests/slow_upload.py create mode 100644 student-tests/storage_checks.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/uploads.py diff --git a/.env.example b/.env.example index d99d26a2..229988d7 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,9 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +APP_PORT=18089 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +API_CPUS=1 +API_MEMORY=512m +HTTP_WORKERS=16 +DB_POOL_MAX=8 +DB_ACCESS_MODE=optimized +POSTGRES_CPUS=1 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..9e15045b --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,26 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BASE_URL: http://127.0.0.1:18089 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: | + python scripts/bootstrap.py + docker compose build api + docker compose up -d --wait + python scripts/wait_ready.py + make check test test-student BASE_URL=$BASE_URL + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL + docker compose up -d --wait --no-deps api + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5e287bd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2 +WORKDIR /app +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ ./src/ +COPY migrations/ ./migrations/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] +STOPSIGNAL SIGTERM +CMD ["python", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..32258f81 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,14 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-student reconcile reconcile-apply + +test-student: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v + +reconcile: + docker compose run --rm --no-deps reconciler + +reconcile-apply: + docker compose run --rm --no-deps reconciler python src/reconcile.py --apply diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..4d8bad22 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,218 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4 — S3-хранилище и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Майорова Софья, БВТ2301**. Целевая оценка: **5**. Выбран **трек 2: согласованность двух хранилищ**. Проверки выполнены 27 октября 2026 года. Окончательную оценку выставляет преподаватель. + +Работа продолжает [мою lab3, PR 55](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab3/pulls/55). Сохранены PostgreSQL, Redis-кэш профиля, сессии, keyset-пагинация, балансировка и транзакционная идемпотентность сообщений на 24 часа. Добавлены приватное S3, бинарные вложения, проверки изображений и повторяемая безопасная сверка S3/БД. Исходные README, COURSE, contracts и публичные tests этой лабораторной сохранены. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine с Compose v2 и Python 3.9+ на хосте. Сервер работает на Python 3.12.8 внутри образа; устанавливать серверные библиотеки на Mac не нужно. Для используемого Mac Docker запускается через отдельный профиль Colima: -## Архитектура и решения +```sh +colima start --profile mtusi-labs +export DOCKER_CONTEXT=colima-mtusi-labs +cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab4" +python3 scripts/bootstrap.py +docker compose build api +docker compose up -d --wait +export BASE_URL=http://127.0.0.1:18089 +python3 scripts/wait_ready.py +make check test test-student BASE_URL=$BASE_URL +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +На другой машине вместо этого `cd` клонировать публичный форк и выбрать ветку `solution/BVT2301-Mayorova`. `build api` идёт перед `up`: одноразовые migrate/bootstrap используют тот же уже построенный образ. `bootstrap.py` создаёт только локальные `.env` и `secrets/`, не печатает значения и не перезаписывает существующие. Пароли случайные; путь секрета — не его значение. Каталог secrets имеет 0700, файлы читаются UID 10001 в контейнере. Секреты, `.env`, runtime/build и данные исключены из Git и Docker build context. -## Доказательства критериев +`migrate` один раз применяет версии 001–004 под advisory lock; API DDL не выполняет. Затем административный `bucket-bootstrap` создаёт bucket. Уже существующий bucket допустим, повторный bootstrap проверен. API стартуют после обоих jobs. Контейнер s3-store хранит **и volume-файлы, и filer metadata** под `/data` на одном persistent volume; PostgreSQL использует другой volume. Redis — восстанавливаемый кэш, сессии находятся в PostgreSQL. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Остановка с сохранением данных: -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +```sh +docker compose down +``` -## Эксперименты и отказы +Только для намеренного удаления **данных этого учебного проекта**, когда они больше не нужны: `docker compose down -v`. Эта команда не используется в restart/failover. Закрытые тестовые volumes отдельного clean-clone стенда удаляются после окончания его проверки. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`.env.example` содержит реальные настройки моего решения: APP_PORT=18089 — только порт LB на localhost, API_REPLICAS=2, API_CPUS=1, API_MEMORY=512m, HTTP_WORKERS=16, DB_POOL_MAX=8, DB_ACCESS_MODE=optimized, POSTGRES_CPUS=1, SESSION_TTL_SECONDS=3600. Внутренний API слушает 8080. Endpoint/bucket/пути S3 credentials задаются в Compose; ограничения 10 MiB и 40 млн пикселей закреплены в коде по контракту и не увеличиваются переменной окружения. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Архитектура и данные -## Ограничения и следующие шаги +```text +Клиент → 127.0.0.1:18089 → HAProxy → API × 2 + ├→ PostgreSQL (общие метаданные/сессии/идемпотентность) + ├→ Redis (необязательный кэш профилей) + └→ S3 gateway → отдельная сеть storage → SeaweedFS mini + └→ s3_data +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Выбрана официальная SeaweedFS **4.51**, digest `sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8`. Образ и реальные параметры проверены запуском. Шлюз `weed s3` принимает только S3; IAM API, Iceberg/Lance отключены. Mini в s3-store запускает master/volume/filer, а его S3, WebDAV и административная консоль отключены. Из API недоступен filer: API не состоит в сети storage. Ни S3, ни PostgreSQL, ни Redis, ни отдельные API-порты на хост не публикуются. Только LB подключён к edge. + +Образы Python 3.12.8, PostgreSQL 17.2-bookworm, Redis 7.4.1-bookworm, HAProxy 3.0.5-alpine также закреплены digest в Dockerfile/Compose. Pillow 13.0.0 проверяет изображения, boto3 1.40.61 обращается к S3; точные прямые и транзитивные Python-зависимости закреплены в requirements.txt. Их выбор — проверенная версия, не обещание постоянного отсутствия уязвимостей. + +| Роль | CPU | Память | Устойчивые данные | +| --- | ---: | ---: | --- | +| API, каждая из двух реплик | 1 | 512 MiB | Нет | +| PostgreSQL | 1 | 512 MiB | postgres_data | +| SeaweedFS master/volume/filer | 1 | 512 MiB | s3_data | +| S3 gateway | 0,5 | 512 MiB, GOMEMLIMIT=384MiB | Нет | +| HAProxy | 0,5 | 64 MiB | Нет | +| Redis | 0,25 | 64 MiB, maxmemory=32mb | Нет | + +CPU-лимиты суммарно превышают 4 vCPU локальной VM; это допускаемый overcommit локального стенда, не выделенные физические ядра каждому сервису. Поэтому измерения не выдаются за максимальную производительность production. + +Миграция 004 создаёт `attachments`: UUID, chat_id, owner_id, filename, случайный object_key, истинный MIME, размер, SHA-256, внутреннее состояние и timestamps. `messages.attachment_ids` сохраняет порядок ссылок; `message_attachments` с FK защищает целостность и запрещает удалить используемое вложение. Пустой текст допустим с вложениями; текст до 4000 Unicode code points, включая U+0000, по-прежнему сохраняется lossless в BYTEA. + +`POST /chats/{id}/attachments?filename=...` принимает сырые PNG/JPEG, не multipart. Сервер создаёт ключ `originals/`, имя файла никогда не является путём. Ответ 201 содержит ready, variants=[], error_code=null, checksum оригинала. Участник другого чата/посторонний получает 404 для metadata/content и при попытке прикрепить чужое вложение. Отсутствие Bearer даёт 401. Download идёт через API без redirect; истинный Content-Type и `nosniff`, `private, no-store`, байты не перекодируются. Ключи S3 и адреса backend не входят в публичные модели. Любой участник своего чата может прикрепить доступное вложение. + +## Границы операции upload + +1. Аутентификация и проверка членства предшествуют приёму тела файла. Проверяются query/filename, один Content-Length, отсутствие Transfer-Encoding, лимит bytes и MIME. +2. Файл читается кусками до 64 KiB в `TemporaryFile` внутри ограниченного tmpfs `/tmp` (128 MiB на API). Это только временный буфер незавершённой операции, не источник хранения подтверждённого файла. При исключении/обрыве контекст закрывает файл; при падении процесса tmpfs исчезает. +3. Вычисляется SHA-256. Pillow открывает только PNG/JPEG, проверяет размеры **до load**, затем `verify()` и полное декодирование первого изображения. MIME сравнивается с форматом по содержимому. Ошибки дают 415, превышение pixels — 413. Управляющие символы, NUL, slash/backslash в filename — 400. Для защиты от сжатых PNG-текстов установлен дополнительный лимит декодированных текстовых метаданных 1 MiB. +4. PostgreSQL фиксирует устойчивый intent со state=uploading **до S3 PUT**. Он уже содержит всю ожидаемую checksum/size и назначенный сервером key. +5. В новой транзакции upload берёт row lock на intent, выполняет S3 PUT, затем обновляет ready. COMMIT завершён до возврата 201. Lock удерживается до COMMIT, поэтому reconciler не может собрать выполняющуюся операцию. +6. Обрыв до intent не создаёт S3-объекта. Отказ после intent/PUT оставляет учтённое состояние uploading, которое безопасно убирается последующей сверкой. Ошибка записи в БД отдаёт 503 без SQL/credentials. Неопределённый результат HTTP upload нельзя прозрачно повторять: upload по контракту не идемпотентен. Сообщения с Idempotency-Key остаются идемпотентными через общую транзакцию PostgreSQL. + +Не обещается распределённая ACID-транзакция между PostgreSQL и S3. Именно окно между двумя хранилищами исследуется в выбранном треке. + +## Ограничение ресурсов и таймауты + +На API максимум 16 выполняющихся HTTP handlers и 32 принятых соединения; заполнение очереди даёт 503. Одновременно принимаются максимум 8 upload на реплику, декодируется **один** файл. Пиксельный буфер явно освобождается `image.close()` до освобождения decoder semaphore; duplicate file descriptor сохраняет исходный временный файл для S3. Это важно: один только выход из контекста Pillow не должен оставлять предыдущий декодированный объект живым, пока следующий запрос уже декодируется. + +Content-Length >10 485 760 отклоняется до чтения файла; dimension check >40 000 000 — до распаковки. Приём тела имеет общий deadline 15 секунд и idle timeout ≤3 секунд; `read1` позволяет пересчитывать общий deadline даже при постоянной подаче очень маленьких порций. Ожидание декодера ограничено 10 секундами. S3 connect timeout 1 секунда, socket read timeout 3 секунды, SDK total_max_attempts=1; автоматического retry upload нет. HTTP LB client/server timeouts 30 секунд. Download также идёт кусками 64 KiB, с ограничением времени, первый кусок читается до 200. Ошибка S3 посреди уже начатого ответа закрывает соединение; клиент обязан считать недополученное тело ошибкой. + +Пул PostgreSQL ≤8 соединений на API, ожидание 2 секунды, statement_timeout=2s, lock_timeout=1,5s. SIGTERM прекращает приём новых соединений, drain ограничен 5 секундами. Readiness проверяет PostgreSQL и S3 bucket — зависимости обязательного синхронного пути; Redis не критичен. Liveness проверяет жизнь процесса. При недоступном S3 LB убирает API из ready backend; ответы самого LB о невозможности соединения могут отличаться от JSON-ошибки API. + +## Приватность bucket и права + +`bootstrap.py` генерирует три разные S3 identities. Административная имеет Admin/Read/Write/List и доступна только bucket-bootstrap и явно запускаемому тестовому audit. API и reconciler получают отдельные пары с Read/Write/List **только messenger-attachments**, без Admin. Write включает удаление объектов этого bucket; API не использует этот метод в публичном endpoint, reconciler использует для компенсации. Это ограничение по bucket, не по одному пользовательскому чату: ACL чата обеспечивает приложение. + +Нет anonymous identity/публичного policy. `storage_checks.py` вне API действительно получает 403 на прямой unauthenticated GET известного key и bucket listing. С реальными ключами API создание другого bucket, listing другого bucket и изменение policy дают 403. Внутренний filer и административный порт также недоступны из сети API. `audit` — только tools profile, не запускается обычным up; его административные секреты никогда не подключаются к API. Bucket bootstrap успешно повторён. + +## Трек 2: повторяемый reconciler + +**Гипотеза:** устойчивый intent перед PUT, row lock и консервативная сверка позволят исправлять частичные записи без удаления свежего или используемого файла; цена полного прохода будет расти примерно линейно с числом метаданных/объектов. + +Оператор сначала запускает dry-run, затем после просмотра — apply: + +```sh +make reconcile +make reconcile-apply +``` + +Обе команды используют отдельные ограниченные credentials reconciler. По умолчанию grace **3600 секунд** и dry-run; команда без `--apply` не меняет ни S3, ни строки БД. Значение grace <3600 отвергается, кроме явно отмеченного флага `--allow-short-test-grace` на синтетическом стенде. Для регулярного обслуживания оператор должен повторять эти команды; скрытого фонового удаления нет. + +Сверка берёт общий advisory lock, чтобы два reconciler не работали одновременно. Cutoff фиксируется в начале прохода. PostgreSQL читается страницами по 100 UUID, S3 list — страницами по 100 ключей; findings ограничены 200 записями, счётчики полные. Внутри каждой операции берётся `FOR UPDATE SKIP LOCKED`: занятая живым upload строка пропускается, а свежая по updated_at защищена grace period. + +| Обнаруженное состояние | Dry-run | Apply и повторный запуск | +| --- | --- | --- | +| Старый uploading, PUT не было | stale_intent | Удаляет intent, S3 delete идемпотентен | +| Старый uploading, PUT уже был, ready/COMMIT сорвался | stale_intent | Удаляет объект, затем intent; при сбое между delete и БД следующий проход завершит очистку | +| Старый объект originals/UUIDv4 без строки БД | orphan | Перед delete повторно проверяет отсутствие строки и свежесть HEAD; ключи никогда не переиспользуются | +| Ready-метаданные, S3 отсутствует/size или metadata checksum не совпадают | missing_or_mismatched | Сохраняет строку и все ссылки, ставит внутренний storage_missing; API возвращает 503 вместо ложного оригинала | +| storage_missing, оригинал восстановлен извне | recoverable | Потоково проверяет **реальные байты** SHA-256/size, только затем возвращает ready | +| Ready и корректный HEAD, даже без сообщения | Защищено | Никогда не удаляет | +| Свежее, занято row lock, неуправляемое имя/prefix | Защищено/пропущено | Не удаляет | + +`storage_missing` — внутреннее состояние, не дополнительное значение публичного enum lab4. Такой ресурс возвращает 503; восстановленная запись снова отдаёт нормальную Attachment со status=ready. Reconciler не может изобрести потерянные bytes: для восстановления нужна внешняя сохранённая копия/backup, в эксперименте это исходный fixture в памяти теста. + +Ключевое доказательство безопасности: корректный API **сначала** фиксирует intent и **потом** создаёт объект с новым UUID. Поэтому старый key без строки не может быть обычным upload в окне до фиксации метаданных. Используемая строка защищена state, row lock и FK; ready-вложения сборщик вообще не удаляет. Если БД/S3 недоступны, проход прерывается без fail-open удаления. Уже выполненные S3 delete не откатываются вместе с PostgreSQL, но устойчивый intent делает повтор безопасным. + +`student-tests/research.py` использует настоящие S3 и PostgreSQL: вводит исключение trigger непосредственно в ready UPDATE после успешного PUT; создаёт intent без PUT, объект без метаданных, удаляет оригинал используемого вложения, удерживает row lock как живой uploader, создаёт свежий intent/объект. Реальная выдержка 3 секунды делает старый orphan старше тестового grace=2, S3 LastModified и часы не подменяются. Dry-run, apply, повтор и восстановление имеют assert на фактические строки, bytes и ссылки. Тестовые grace=2 и SQL fault injection предназначены только для данного синтетического стенда. + +Команды воспроизведения security и исследования (audit получает отдельные локальные тестовые секреты): + +```sh +docker compose run --rm -T --no-deps -v "$PWD:/work:ro" -w /work audit python student-tests/storage_checks.py +docker compose run --rm -T --no-deps -v "$PWD:/work:ro" -w /work audit python student-tests/research.py +``` + +## Измерения + +Стенд: macOS arm64, Colima/Ubuntu Linux, 4 vCPU и примерно 4 GiB RAM. Лимиты контейнеров приведены выше. Остальные учебные стенды не останавливались; это локальный эксперимент без гарантии отсутствия соседней нагрузки. Генерация fixtures и bootstrap выполняются до начала измерения. + +Для стоимости сверки использованы три популяции — 10, 110, 510 строк и столько же S3-объектов, три последовательных dry-run на каждой. Исходная популяция включает данные проверок; каждый этап добавляет соответственно 100 и 400 подтверждённых тестовых ready-объектов. При повторе сценария абсолютная исходная популяция может вырасти: результаты всегда содержат фактические counts, а не предполагаемый N. + +| Метаданных / объектов | Медиана полного обхода | Наблюдаемые HEAD | +| --- | ---: | ---: | +| 10 / 10 | 19,56 мс | 9 | +| 110 / 110 | 136,28 мс | 109 | +| 510 / 510 | 603,06 мс | 509–510 | + +Количество HEAD немного различается из-за grace. Полный проход включает создание S3 client, advisory lock, PostgreSQL-страницы/lookup, S3 list и HEAD. Основные bytes не скачиваются при обычном HEAD; GET/SHA-256 используются только при восстановлении storage_missing. Память прохода ограничена размером страниц. Последовательный алгоритм имеет O(N+M) обход, но **не является snapshot между двумя хранилищами**; пропущенное/изменившееся состояние проверяется следующим запуском. Увеличение 110→510 объектов дало примерно пропорциональный рост времени; это поддерживает гипотезу на данном диапазоне, не прогноз миллиардов объектов. + +На каждом профиле **10 клиентов одновременно** начинают upload, затем скачивают результат с тем же SHA-256. Один прогон на профиль; fixtures детерминированы, подготовлены заранее. Нет скрытых retry. Перед каждым профилем API перезапускаются, поэтому cgroup memory.peak относится к этому запуску. Опрос memory.current/peak/events и cpu.stat идёт дополнительными docker exec; это вносит накладные расходы. Измеряется end-to-end upload+download, не только POST и не предельный RPS. Для десяти запросов p95/p99 по nearest-rank оба равны максимуму. + +| PNG: bytes / pixels | Успех / ошибки | Весь сценарий | p50 / p95=p99 end-to-end | Пик API 1 / API 2 | +| --- | --- | ---: | ---: | ---: | +| 529,848 / 176,400 | 10/10 / 0 | 0.387 с | 141.3 / 149.4 мс | 67.05 / 67.18 MiB | +| 7,563,763 / 2,520,000 | 10/10 / 0 | 2.042 с | 1711.5 / 1743.2 мс | 127.41 / 127.04 MiB | +| 533,859 / 40,000,000 | 10/10 / 0 | 0.831 с | 416.2 / 602.8 мс | 326.36 / 319.54 MiB | + +В финальных трёх профилях 30/30 загрузок вернули 201, скачанные SHA-256 совпали, OOM kill = 0. Лимит каждой API — 512 MiB. Высокий пик третьего профиля связан с 40 млн декодированных RGB-пикселей при небольшом сжатом файле: размер HTTP-тела не равен размеру изображения в RAM. Различие реплик отражает распределение нагрузки и память аллокатора; это не доказательство разницы алгоритмов. + +Средний CPU API за сценарий оценивается по разнице cgroup cpu.stat usage_usec, делённой на время сценария (100% = одно ядро; короткая длительность и docker exec влияют на оценку): +- 176400 pixels: API 1 / API 2 = 45.7% / 43.7%. +- 2520000 pixels: API 1 / API 2 = 36.2% / 35.1%. +- 40000000 pixels: API 1 / API 2 = 77.7% / 79.5%. + +![Сверка и память при загрузках](evidence/research.png) + +Ранние прогоны сохранены, а не скрыты: с лимитом S3 gateway 192 MiB обнаружен его OOM; после изменения gateway на 512 MiB, GOMEMLIMIT=384MiB и ограничения concurrentUploadLimitMB до 32 повтор выявил отдельный OOM API на RGB-изображениях 40 Мп. Причина API устранена освобождением пиксельного буфера **до** освобождения decoder semaphore. Финальные прогоны с исходным поддерживаемым лимитом 512 MiB/API проверяют именно этот случай. Ошибочные запуски не включены в успешные итоговые измерения, их stderr и State с OOMKilled=true оставлены в evidence. + +## Проверки и доказательства + +| Критерий | Команда/сценарий | Результат / evidence | +| --- | --- | --- | +| Оригинальные контракт и tests | make check; git diff относительно skeleton | Не изменены; final-tests.txt | +| Минимум lab3 + вложения | make test | Публичный lab4 smoke PASS; final-tests.txt | +| Дополнительные API/Unicode/сессии/гонки | make test-student | 21 тест PASS; final-tests.txt | +| PNG/JPEG, checksum, приватность, чужой chat/id | test_attachments.py и storage_checks.py | Побайтное совпадение; 400/401/404/413/415; storage-security.json | +| 10 MiB/40 Мп и повреждённый файл | test_bad_images_and_mime, test_valid_pixel_boundary | 413 выше границы, 415 для подмены/повреждения, ровно 40 Мп принято | +| Остановка именно загрузившей API | student-tests/faults.py | Каждая реальная реплика остановлена отдельно, файл скачан другой с прежним Bearer; faults.json | +| Обязательный публичный failover сообщений | make test-failover ACTION_SCRIPT=scripts/stop-one.sh | 2 реплики, 116 подтверждённых записей, 1 транзитная ошибка, восстановление 2,594 с; public-failover.txt | +| Restart API, LB, PostgreSQL, Redis и обоих S3-компонентов | make test-persistence ACTION_SCRIPT=scripts/restart.sh | Прежние сессия, сообщения, attachment metadata/bytes сохранены; persistence.txt, restart-before/after.json | +| Недоступность S3, fallback Redis | student-tests/faults.py | S3: ready/content 503, live 200; после восстановления прежние bytes; Redis fallback; faults.json | +| Обрыв приёма | student-tests/faults.py | Нет intent для прерванного filename, PUT не выполнялся | +| Медленный непрерывный клиент | student-tests/slow_upload.py | 400 около общего deadline 15 с; slow-upload.json | +| Отдельные ограниченные S3 credentials и закрытая сеть | storage_checks.py | Anonymous GET/list 403, три запрещённых операции API 403, filer/admin недоступны; storage-security.json | +| Идемпотентный bootstrap bucket | Повторный compose run bucket-bootstrap | Exit 0; bucket-bootstrap-repeat.txt | +| Потоковое ограничение RAM, ≥10 одновременных файлов | student-tests/uploads.py | uploads.json, график; cgroup peaks и OOM counters | +| Согласованность, свежие/используемые/занятые объекты | research.py | Все 7 assertions-групп true; dry/apply/repeat/recover; reconciliation.json | +| Стоимость обхода | research.py | 9 реальных измерений на трёх популяциях; reconciliation.json | +| Чистый clone/новые volumes | Отдельный проект, порт и секреты | PASS: новые секреты, пустые PostgreSQL/S3 volumes, smoke + 21 тест + полный restart; clean-clone.txt | + +Публичные команды отказов управляют только lab4; после stop-one реплика остаётся остановленной по контракту теста и восстанавливается явно `docker compose up -d --wait --no-deps api`. Дополнительный faults.py сам восстанавливает каждую зависимость в finally. Снимки runtime содержат только разрешённые поля состояния, лимитов, mounts и портов, без environment/credentials. Не сохраняются тела Basic/Bearer или полные session responses. + +```sh +export DOCKER_CONTEXT=colima-mtusi-labs +export BASE_URL=http://127.0.0.1:18089 +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL +docker compose up -d --wait --no-deps api +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL +python3 student-tests/faults.py +python3 student-tests/slow_upload.py +python3 student-tests/uploads.py +``` + +Для повторной генерации графиков нужен Python 3.11+ и `pip install -r requirements-research.txt`, затем `python student-tests/plot.py`; приложенные PNG/SVG уже готовы. Workflow Application добавлен отдельно от исходной проверки skeleton; hosted CI требует разрешений платформы и здесь не объявляется прошедшим автоматически. + +## Ограничения + +Доказаны отказ API-реплики и restart с сохранёнными volumes; один PostgreSQL, один S3-store, один LB и один host остаются точками отказа. Это не multi-node HA и не гарантия сохранности при отключении питания. TLS отсутствует, поэтому публикуется только localhost с синтетическими данными. Root-доступ к Docker равносилен доступу к секретам стенда. + +Распределённой транзакции нет, до очередного apply старые uploading/orphan могут оставаться. Готовые, но неиспользуемые вложения сборщик намеренно сохраняет, поскольку контракт не содержит удаления. Данные экспериментов/публичных тестов также накапливаются. Для постоянно работающего сервиса нужны расписание обслуживания, backup S3+БД и контроль расходов. + +Обычный проход сверяет HEAD и сохранённую S3 checksum metadata, а не перечитывает каждый корректный объект: одинаково изменённые bytes и metadata сторонним привилегированным оператором так не обнаруживаются. Восстановление missing всегда проверяет реальные bytes. Параллельные изменения БД/S3 и значительный clock skew требуют повторного прохода и аккуратного grace; штатные API не переиспользуют ключи и не изменяют ready-объекты. Grace 3600 ориентирован на значительно более короткую обычную загрузку; уменьшать его на реальных данных без исследования нельзя. + +Проверка изображений декодирует первый кадр, не выполняет антивирус и не создаёт thumbnail; дополнительные PNG-текстовые метаданные ограничены 1 MiB как защита parser RAM. UI, presigned upload и трек 1 не реализуются: выбран законченный трек 2. Показатели маленького локального эксперимента не задают SLA или максимальный RPS. + +## Источники и помощь + +Перенесено собственное решение lab3 (Майорова Софья, PR 55); чужие студенческие решения не использовались. Задание/контракты текущего skeleton — основание обязательного поведения. + +- [SeaweedFS S3 API](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API), [S3 credentials и bucket-scoped actions](https://github.com/seaweedfs/seaweedfs/wiki/S3-Credentials), параметры mini проверены у закреплённого Docker-образа. +- [Pillow Image: ограничения, verify/load/close](https://pillow.readthedocs.io/en/stable/reference/Image.html). +- [Boto3 put_object](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html). +- [PostgreSQL row-level/advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html). + +Использован Codex для разработки, подготовки тестов, запуска измерений и оформления. Все приведённые PASS/числа получены реальными командами данного стенда; утверждения ограничены фактически проверенными сценариями. Имя автора в Git — Майорова Софья; генеративная помощь раскрыта явно. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f8dc59f0 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,185 @@ +name: mtusi-lab4 +x-application: &application + image: mtusi-messenger:lab4 + networks: [backend] + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + HTTP_WORKERS: ${HTTP_WORKERS:-16} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized} + LOG_LEVEL: WARNING + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret + secrets: [database_url, cache_url, s3_api_key, s3_api_secret] + tmpfs: ["/tmp:size=128m,mode=1777"] + mem_limit: ${API_MEMORY:-512m} + cpus: ${API_CPUS:-1} + stop_grace_period: 10s +services: + migrate: + <<: *application + command: [python, src/migrate.py] + healthcheck: + disable: true + deploy: + replicas: 1 + depends_on: + postgres: + condition: service_healthy + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + bucket-bootstrap: + condition: service_completed_successfully + bucket-bootstrap: + <<: *application + command: [python, src/bucket_bootstrap.py] + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: [s3_admin_key, s3_admin_secret] + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + reconciler: + <<: *application + profiles: [tools] + command: [python, src/reconcile.py] + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret + secrets: [database_url, s3_reconciler_key, s3_reconciler_secret] + healthcheck: + disable: true + deploy: + replicas: 1 + audit: + <<: *application + profiles: [tools] + environment: + <<: *environment + BASE_URL: http://lb:8080 + secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_reconciler_key, s3_reconciler_secret] + healthcheck: + disable: true + deploy: + replicas: 1 + s3-store: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64] + networks: [storage] + volumes: [s3_data:/data] + mem_limit: 512m + cpus: 1 + stop_grace_period: 20s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/] + interval: 2s + timeout: 2s + retries: 45 + s3: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16] + secrets: [s3_config] + networks: [backend, storage] + mem_limit: 512m + environment: + GOMEMLIMIT: 384MiB + cpus: 0.5 + depends_on: + s3-store: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'nc -z 127.0.0.1 8333'] + interval: 2s + timeout: 1s + retries: 30 + lb: + image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949 + ports: ["127.0.0.1:${APP_PORT:-18089}:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + mem_limit: 64m + cpus: 0.5 + stop_grace_period: 12s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 2s + retries: 10 + postgres: + image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [backend] + healthcheck: + test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1} + redis: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 64m + cpus: 0.25 +volumes: + postgres_data: + s3_data: +networks: + edge: + storage: + internal: true + backend: + internal: true +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_api_key: + file: ./secrets/s3_api_key + s3_api_secret: + file: ./secrets/s3_api_secret + s3_reconciler_key: + file: ./secrets/s3_reconciler_key + s3_reconciler_secret: + file: ./secrets/s3_reconciler_secret + s3_config: + file: ./secrets/s3_config diff --git a/evidence/bucket-bootstrap-repeat.txt b/evidence/bucket-bootstrap-repeat.txt new file mode 100644 index 00000000..5ff110db --- /dev/null +++ b/evidence/bucket-bootstrap-repeat.txt @@ -0,0 +1,3 @@ + Container mtusi-lab4-bucket-bootstrap-run-d95f463bab66 Creating + Container mtusi-lab4-bucket-bootstrap-run-d95f463bab66 Created +Private bucket ready; static authenticated identities loaded from local secret config. diff --git a/evidence/build.txt b/evidence/build.txt new file mode 100644 index 00000000..ae9bb62f --- /dev/null +++ b/evidence/build.txt @@ -0,0 +1,55 @@ + Image mtusi-messenger:lab4 Building +#1 [internal] load local bake definitions +#1 reading from stdin 650B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 6.92kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [2/7] WORKDIR /app +#7 CACHED + +#8 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#8 CACHED + +#9 [4/7] COPY requirements.txt ./ +#9 CACHED + +#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#10 CACHED + +#11 [6/7] COPY src/ ./src/ +#11 DONE 0.0s + +#12 [7/7] COPY migrations/ ./migrations/ +#12 DONE 0.0s + +#13 exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:6cabe9729ea3e45195f623f07f0974303b8b392b2e4e6aac4a3c4bb565dd6a1a done +#13 exporting config sha256:84a7ceda5c86abe367437b43f6f54e6eb16a19abb99bfd2257771bb9119cf759 done +#13 exporting attestation manifest sha256:a206b14cc9c3bfd49e2bf15bb90189e58c9a89871e87fcb4454c483bdde60677 done +#13 exporting manifest list sha256:6de46cd4b5e20a5826fcdf2289f055877f477800dcc27100a470e1fb8c267001 done +#13 naming to docker.io/library/mtusi-messenger:lab4 done +#13 unpacking to docker.io/library/mtusi-messenger:lab4 0.0s done +#13 DONE 0.1s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:lab4 Built diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..7467d6a5 --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,220 @@ +Cloning into '/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab4/build/lab4-clean-6e826b67/repo'... +done. +Local configuration ready; no secrets printed. + Image mtusi-messenger:lab4 Building +#1 [internal] load local bake definitions +#1 reading from stdin 721B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 52.94kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [4/7] COPY requirements.txt ./ +#8 CACHED + +#9 [2/7] WORKDIR /app +#9 CACHED + +#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#10 CACHED + +#11 [6/7] COPY src/ ./src/ +#11 CACHED + +#12 [7/7] COPY migrations/ ./migrations/ +#12 CACHED + +#13 exporting to image +#13 exporting layers done +#13 exporting manifest sha256:b9111c9adf12479532b94464933e51b833be4486c559d60f233006a4f2d448a8 done +#13 exporting config sha256:e59e3410c92a9e2663734ff4a9c33edcc65414616230d01ede4285bc1ab1f973 done +#13 exporting attestation manifest sha256:9b07c898e2f46418c89c2514d79a08b01b37144849451e0704173ea0c7c5c8c2 done +#13 exporting manifest list sha256:6db63c0e6e0d2f8d4f650a7ed60998284a8e81d98bc294b41883cef612ba2bce done +#13 naming to docker.io/library/mtusi-messenger:lab4 done +#13 unpacking to docker.io/library/mtusi-messenger:lab4 done +#13 DONE 0.0s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:lab4 Built + Network lab4-clean-6e826b67_storage Creating + Network lab4-clean-6e826b67_storage Created + Network lab4-clean-6e826b67_backend Creating + Network lab4-clean-6e826b67_backend Created + Network lab4-clean-6e826b67_edge Creating + Network lab4-clean-6e826b67_edge Created + Volume lab4-clean-6e826b67_s3_data Creating + Volume lab4-clean-6e826b67_s3_data Created + Volume lab4-clean-6e826b67_postgres_data Creating + Volume lab4-clean-6e826b67_postgres_data Created + Container lab4-clean-6e826b67-s3-store-1 Creating + Container lab4-clean-6e826b67-redis-1 Creating + Container lab4-clean-6e826b67-postgres-1 Creating + Container lab4-clean-6e826b67-redis-1 Created + Container lab4-clean-6e826b67-s3-store-1 Created + Container lab4-clean-6e826b67-s3-1 Creating + Container lab4-clean-6e826b67-postgres-1 Created + Container lab4-clean-6e826b67-migrate-1 Creating + Container lab4-clean-6e826b67-s3-1 Created + Container lab4-clean-6e826b67-bucket-bootstrap-1 Creating + Container lab4-clean-6e826b67-migrate-1 Created + Container lab4-clean-6e826b67-bucket-bootstrap-1 Created + Container lab4-clean-6e826b67-api-2 Creating + Container lab4-clean-6e826b67-api-1 Creating + Container lab4-clean-6e826b67-api-1 Created + Container lab4-clean-6e826b67-api-2 Created + Container lab4-clean-6e826b67-lb-1 Creating + Container lab4-clean-6e826b67-lb-1 Created + Container lab4-clean-6e826b67-postgres-1 Starting + Container lab4-clean-6e826b67-s3-store-1 Starting + Container lab4-clean-6e826b67-redis-1 Starting + Container lab4-clean-6e826b67-s3-store-1 Started + Container lab4-clean-6e826b67-s3-store-1 Waiting + Container lab4-clean-6e826b67-postgres-1 Started + Container lab4-clean-6e826b67-postgres-1 Waiting + Container lab4-clean-6e826b67-redis-1 Started + Container lab4-clean-6e826b67-postgres-1 Healthy + Container lab4-clean-6e826b67-migrate-1 Starting + Container lab4-clean-6e826b67-migrate-1 Started + Container lab4-clean-6e826b67-s3-store-1 Healthy + Container lab4-clean-6e826b67-s3-1 Starting + Container lab4-clean-6e826b67-s3-1 Started + Container lab4-clean-6e826b67-s3-1 Waiting + Container lab4-clean-6e826b67-s3-1 Healthy + Container lab4-clean-6e826b67-bucket-bootstrap-1 Starting + Container lab4-clean-6e826b67-bucket-bootstrap-1 Started + Container lab4-clean-6e826b67-migrate-1 Waiting + Container lab4-clean-6e826b67-redis-1 Waiting + Container lab4-clean-6e826b67-bucket-bootstrap-1 Waiting + Container lab4-clean-6e826b67-migrate-1 Exited + Container lab4-clean-6e826b67-redis-1 Healthy + Container lab4-clean-6e826b67-bucket-bootstrap-1 Exited + Container lab4-clean-6e826b67-api-2 Starting + Container lab4-clean-6e826b67-api-2 Started + Container lab4-clean-6e826b67-api-1 Starting + Container lab4-clean-6e826b67-api-1 Started + Container lab4-clean-6e826b67-api-2 Waiting + Container lab4-clean-6e826b67-api-1 Waiting + Container lab4-clean-6e826b67-api-2 Healthy + Container lab4-clean-6e826b67-api-1 Healthy + Container lab4-clean-6e826b67-lb-1 Starting + Container lab4-clean-6e826b67-lb-1 Started + Container lab4-clean-6e826b67-api-2 Waiting + Container lab4-clean-6e826b67-api-1 Waiting + Container lab4-clean-6e826b67-lb-1 Waiting + Container lab4-clean-6e826b67-s3-1 Waiting + Container lab4-clean-6e826b67-bucket-bootstrap-1 Waiting + Container lab4-clean-6e826b67-postgres-1 Waiting + Container lab4-clean-6e826b67-migrate-1 Waiting + Container lab4-clean-6e826b67-s3-store-1 Waiting + Container lab4-clean-6e826b67-redis-1 Waiting + Container lab4-clean-6e826b67-bucket-bootstrap-1 Exited + Container lab4-clean-6e826b67-redis-1 Healthy + Container lab4-clean-6e826b67-s3-store-1 Healthy + Container lab4-clean-6e826b67-s3-1 Healthy + Container lab4-clean-6e826b67-postgres-1 Healthy + Container lab4-clean-6e826b67-migrate-1 Exited + Container lab4-clean-6e826b67-api-2 Healthy + Container lab4-clean-6e826b67-api-1 Healthy + Container lab4-clean-6e826b67-lb-1 Healthy +Ready through load balancer. +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18090" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18090" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_access_and_cross_chat (test_attachments.Attachments) ... ok +test_attachment_message_replay (test_attachments.Attachments) ... ok +test_bad_images_and_mime (test_attachments.Attachments) ... ok +test_binary_roundtrip_and_metadata (test_attachments.Attachments) ... ok +test_dangerous_names_and_query (test_attachments.Attachments) ... ok +test_valid_pixel_boundary (test_attachments.Attachments) ... ok + +---------------------------------------------------------------------- +Ran 21 tests in 5.458s + +OK +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18090" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab4-clean-6e826b67-lb-1 Stopping + Container lab4-clean-6e826b67-lb-1 Stopped + Container lab4-clean-6e826b67-lb-1 Removing + Container lab4-clean-6e826b67-lb-1 Removed + Container lab4-clean-6e826b67-api-1 Stopping + Container lab4-clean-6e826b67-api-2 Stopping + Container lab4-clean-6e826b67-api-2 Stopped + Container lab4-clean-6e826b67-api-2 Removing + Container lab4-clean-6e826b67-api-2 Removed + Container lab4-clean-6e826b67-api-1 Stopped + Container lab4-clean-6e826b67-api-1 Removing + Container lab4-clean-6e826b67-api-1 Removed + Container lab4-clean-6e826b67-bucket-bootstrap-1 Stopping + Container lab4-clean-6e826b67-migrate-1 Stopping + Container lab4-clean-6e826b67-redis-1 Stopping + Container lab4-clean-6e826b67-bucket-bootstrap-1 Stopped + Container lab4-clean-6e826b67-bucket-bootstrap-1 Removing + Container lab4-clean-6e826b67-migrate-1 Stopped + Container lab4-clean-6e826b67-migrate-1 Removing + Container lab4-clean-6e826b67-migrate-1 Removed + Container lab4-clean-6e826b67-postgres-1 Stopping + Container lab4-clean-6e826b67-bucket-bootstrap-1 Removed + Container lab4-clean-6e826b67-s3-1 Stopping + Container lab4-clean-6e826b67-postgres-1 Stopped + Container lab4-clean-6e826b67-postgres-1 Removing + Container lab4-clean-6e826b67-postgres-1 Removed + Container lab4-clean-6e826b67-s3-1 Stopped + Container lab4-clean-6e826b67-s3-1 Removing + Container lab4-clean-6e826b67-s3-1 Removed + Container lab4-clean-6e826b67-s3-store-1 Stopping + Container lab4-clean-6e826b67-redis-1 Stopped + Container lab4-clean-6e826b67-redis-1 Removing + Container lab4-clean-6e826b67-redis-1 Removed + Container lab4-clean-6e826b67-s3-store-1 Stopped + Container lab4-clean-6e826b67-s3-store-1 Removing + Container lab4-clean-6e826b67-s3-store-1 Removed + Volume lab4-clean-6e826b67_postgres_data Removing + Network lab4-clean-6e826b67_storage Removing + Volume lab4-clean-6e826b67_s3_data Removing + Network lab4-clean-6e826b67_edge Removing + Network lab4-clean-6e826b67_backend Removing + Volume lab4-clean-6e826b67_postgres_data Removed + Volume lab4-clean-6e826b67_s3_data Removed + Network lab4-clean-6e826b67_storage Removed + Network lab4-clean-6e826b67_edge Removed + Network lab4-clean-6e826b67_backend Removed +PASS: clean clone, new credentials, empty PostgreSQL/S3 volumes, all 21 own tests, full restart and byte persistence. diff --git a/evidence/failover-recovery.txt b/evidence/failover-recovery.txt new file mode 100644 index 00000000..bfc5a1bc --- /dev/null +++ b/evidence/failover-recovery.txt @@ -0,0 +1,7 @@ + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy diff --git a/evidence/faults-initial-stderr.txt b/evidence/faults-initial-stderr.txt new file mode 100644 index 00000000..17c8ef76 --- /dev/null +++ b/evidence/faults-initial-stderr.txt @@ -0,0 +1,6 @@ +Traceback (most recent call last): + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/faults.py", line 125, in + main() + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/faults.py", line 78, in main + target = urlsplit(c.base_url) +AttributeError: 'Client' object has no attribute 'base_url' diff --git a/evidence/faults-stderr.txt b/evidence/faults-stderr.txt new file mode 100644 index 00000000..ebf0e41e --- /dev/null +++ b/evidence/faults-stderr.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/faults.py", line 125, in + main() + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/faults.py", line 71, in main + c.message(alice, chat, 'write survives stop') + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/tests/client.py", line 183, in message + return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message') + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/tests/client.py", line 132, in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: POST /api/v1/chats/1b519f28-dbdb-4400-a923-073d940c8952/messages: expected [201], received 503 diff --git a/evidence/faults.json b/evidence/faults.json new file mode 100644 index 00000000..e69de29b diff --git a/evidence/final-tests.txt b/evidence/final-tests.txt new file mode 100644 index 00000000..f5cd996b --- /dev/null +++ b/evidence/final-tests.txt @@ -0,0 +1,31 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18089" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18089" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_access_and_cross_chat (test_attachments.Attachments) ... ok +test_attachment_message_replay (test_attachments.Attachments) ... ok +test_bad_images_and_mime (test_attachments.Attachments) ... ok +test_binary_roundtrip_and_metadata (test_attachments.Attachments) ... ok +test_dangerous_names_and_query (test_attachments.Attachments) ... ok +test_valid_pixel_boundary (test_attachments.Attachments) ... ok + +---------------------------------------------------------------------- +Ran 21 tests in 5.309s + +OK diff --git a/evidence/openapi-validation.txt b/evidence/openapi-validation.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi-validation.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..b969d325 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18089" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover.txt b/evidence/public-failover.txt new file mode 100644 index 00000000..f381bd88 --- /dev/null +++ b/evidence/public-failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18089" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 1, "first_recovery_seconds": 2.594} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-smoke.txt b/evidence/public-smoke.txt new file mode 100644 index 00000000..8305d3b8 --- /dev/null +++ b/evidence/public-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18089" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/rebuild-startup.txt b/evidence/rebuild-startup.txt new file mode 100644 index 00000000..36215156 --- /dev/null +++ b/evidence/rebuild-startup.txt @@ -0,0 +1,12 @@ + Container mtusi-lab4-api-1 Recreate + Container mtusi-lab4-api-2 Recreate + Container mtusi-lab4-api-2 Recreated + Container mtusi-lab4-api-1 Recreated + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy diff --git a/evidence/reconciler-default-stderr.txt b/evidence/reconciler-default-stderr.txt new file mode 100644 index 00000000..f7f344a4 --- /dev/null +++ b/evidence/reconciler-default-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-reconciler-run-e50d556374d7 Creating + Container mtusi-lab4-reconciler-run-e50d556374d7 Created diff --git a/evidence/reconciler-default.json b/evidence/reconciler-default.json new file mode 100644 index 00000000..4d02a2e7 --- /dev/null +++ b/evidence/reconciler-default.json @@ -0,0 +1,20 @@ +{ + "apply": false, + "grace_seconds": 3600, + "elapsed_seconds": 0.785007, + "counts": { + "metadata": 638, + "objects": 628, + "head_calls": 501, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 765, + "protected_ready": 501, + "protected_foreign_prefix": 0 + }, + "findings": [] +} diff --git a/evidence/reconciliation-stderr.txt b/evidence/reconciliation-stderr.txt new file mode 100644 index 00000000..ac275373 --- /dev/null +++ b/evidence/reconciliation-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-audit-run-9e5a49008d08 Creating + Container mtusi-lab4-audit-run-9e5a49008d08 Created diff --git a/evidence/reconciliation.json b/evidence/reconciliation.json new file mode 100644 index 00000000..00d6c3cf --- /dev/null +++ b/evidence/reconciliation.json @@ -0,0 +1,332 @@ +{ + "method": "real S3 and PostgreSQL, per-run fixed cutoff, synthetic data, 2s test grace; production >=3600s", + "checks": { + "dry_run_mutates_nothing": true, + "stale_intents_and_orphans_cleaned": true, + "fresh_and_locked_retained": true, + "used_ready_retained": true, + "missing_reference_retained": true, + "apply_repeat_idempotent": true, + "restore_after_sha256_verification": true + }, + "passes": [ + { + "name": "dry_run", + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.050187, + "counts": { + "metadata": 14, + "objects": 12, + "head_calls": 13, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 1, + "protected_fresh": 2, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [ + { + "kind": "stale_intent", + "id": "a41ec1bd-b7f2-443e-914e-98356fd98a7e" + }, + { + "kind": "missing_or_mismatched", + "id": "b38ed978-f5aa-420b-aa2d-c531364e477d" + }, + { + "kind": "stale_intent", + "id": "ecc30613-bc47-4469-9171-373cc9722ca5" + }, + { + "kind": "orphan", + "id": "f210e333-3f79-4f90-917a-643f33b36f06" + } + ] + }, + { + "name": "apply", + "apply": true, + "grace_seconds": 2, + "elapsed_seconds": 0.044472, + "counts": { + "metadata": 14, + "objects": 11, + "head_calls": 13, + "get_calls": 0, + "deleted_objects": 2, + "deleted_intents": 2, + "marked_missing": 1, + "restored": 0, + "skipped_locked": 1, + "protected_fresh": 2, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [ + { + "kind": "stale_intent", + "id": "a41ec1bd-b7f2-443e-914e-98356fd98a7e" + }, + { + "kind": "missing_or_mismatched", + "id": "b38ed978-f5aa-420b-aa2d-c531364e477d" + }, + { + "kind": "stale_intent", + "id": "ecc30613-bc47-4469-9171-373cc9722ca5" + }, + { + "kind": "orphan", + "id": "f210e333-3f79-4f90-917a-643f33b36f06" + } + ] + }, + { + "name": "repeat", + "apply": true, + "grace_seconds": 2, + "elapsed_seconds": 0.020893, + "counts": { + "metadata": 12, + "objects": 10, + "head_calls": 9, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 1, + "protected_fresh": 3, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "name": "recover", + "apply": true, + "grace_seconds": 2, + "elapsed_seconds": 0.025345, + "counts": { + "metadata": 12, + "objects": 11, + "head_calls": 11, + "get_calls": 1, + "deleted_objects": 0, + "deleted_intents": 1, + "marked_missing": 0, + "restored": 1, + "skipped_locked": 0, + "protected_fresh": 3, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [ + { + "kind": "stale_intent", + "id": "5ad3dfb9-96f5-4c1b-aee7-6fcd572b21ff" + }, + { + "kind": "recoverable", + "id": "b38ed978-f5aa-420b-aa2d-c531364e477d" + } + ] + } + ], + "scan_cost": [ + { + "repetition": 1, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.020025, + "counts": { + "metadata": 10, + "objects": 10, + "head_calls": 9, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 2, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 2, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.019561, + "counts": { + "metadata": 10, + "objects": 10, + "head_calls": 9, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 2, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 3, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.019272, + "counts": { + "metadata": 10, + "objects": 10, + "head_calls": 9, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 2, + "protected_ready": 9, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 1, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.127656, + "counts": { + "metadata": 110, + "objects": 110, + "head_calls": 109, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 102, + "protected_ready": 109, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 2, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.142166, + "counts": { + "metadata": 110, + "objects": 110, + "head_calls": 109, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 102, + "protected_ready": 109, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 3, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.136284, + "counts": { + "metadata": 110, + "objects": 110, + "head_calls": 109, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 102, + "protected_ready": 109, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 1, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.609772, + "counts": { + "metadata": 510, + "objects": 510, + "head_calls": 509, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 447, + "protected_ready": 509, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 2, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.583932, + "counts": { + "metadata": 510, + "objects": 510, + "head_calls": 510, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 229, + "protected_ready": 510, + "protected_foreign_prefix": 0 + }, + "findings": [] + }, + { + "repetition": 3, + "apply": false, + "grace_seconds": 2, + "elapsed_seconds": 0.603062, + "counts": { + "metadata": 510, + "objects": 510, + "head_calls": 510, + "get_calls": 0, + "deleted_objects": 0, + "deleted_intents": 0, + "marked_missing": 0, + "restored": 0, + "skipped_locked": 0, + "protected_fresh": 229, + "protected_ready": 510, + "protected_foreign_prefix": 0 + }, + "findings": [] + } + ] +} diff --git a/evidence/research.png b/evidence/research.png new file mode 100644 index 0000000000000000000000000000000000000000..97349ae0ce427b73d3e59ccf80d6e550354cd6c2 GIT binary patch literal 88676 zcma&Oc{tSV`#(Gp?nvDg$r9~$QuZ}TLfQ9aD7S>{V;g%VBD4%D`!b9@JHsFqMYb`F zb%->~U~FSzFwZsj=ll6S&vP8VKYnu@j*Ky5=6#*ldA_!*7bZq}+?)cO5D0`@|IRHl z2!wqW0%6Jhiyi#L!>NS`{!xYL+Q7_x9>Rj}`@2H!-G});_JKWie{e3y)jz=9$NRdB zoSe)xsdJBDFy8=GSy`|D{S6r(e>d6aq*d@DkR!f#Yy%*WGa1aUL;8VF5s*U=i2f~Y zi;&FaX%2+BC5^VOk#v6%Ln}SUX|^JkfVoexsgA+m+cihKli4I6=&@Pwe5BnxaroA) zzh4o;_=^@C+vmeJFJ0Haa{c?=qHrxw*UJI+Myr_bQvoy;`lX$4%^dJ2{^u5-`0LMO z=BN6f+acxueg`?%zj%n_fA4)c@&A1&`LB-u_ci7b?&&{oWN1r?Jhx|!}sX3PmeFZyx#v;i_tVX z&}$}wR&OZN{3Tg!RXMRZOD%YU5JvIZBn|{eHTujDPD|PRji-UD_Cn5cREkb2t6I+R zX{+YH{%{O@s=ZHjx03|M&<p(1|-1i0A()6o@lb4BhyJnyC#Wk2Zv?&ov?1Yjzh4 zQXYSMb$&HuAsvDECy<1zR`48ca(;bKSx$VI>o?yuzV`F;O1DT9Y|?bis|vGSlPH7! zwqmI8STf1~TcQ5jWn?6YqV)@7Qfx1&h%AZOtN={O@A&?v24K|ia zL5U+&N-ZRB{H}j&shVrj3iZ!p@ujqGCTl^81tpf2mb9gPS?#mRum=;KRlH@oK4N&s z;4F>pF_(k(BcKRO{OTSJb*9NW3CrKZ3wT7=dg5hhjV#J|rzniZ-rDE$jec80dXrV{ z2LkI6xD2PHH?0d=2Y=?9BA)mB?o*nn3mDMAQ-jQ3-Nwgrjx9G#dRMyTRjPevdn+;7 zywtp0)BE^K{E371RIdlUY0|59cE>cfOoENz__52D$0`b3_{Odek@br`;Z^G9#YXi| z3|x2WGz221vB2ozghU(6?rjbPS3KO*N|UxfO;x;|EVSfgtlhFQ**IC86G>um*%-C= z8IPb%^QCVbZ1l=*e>r8`#G_GRSD(FeVa5|M$)SHeohc#td*E(~_=XnNNLpL2#q}gUB>OF~-BhUz{WfY_`a(|ci z!k^Ps0p{`pTy0sAyGyHR`%f}RBg|4!#?+Q<+n-LhC4(9v6&01Bk^grJz zIZ+4IGLGmbze_wvT(S@O)wkUprQywA;vZQB@BUkMq5AD4$o->J_x=9Xq$cuS&L5f`hx3 zeZymSUQ6}{0m2It_)O&0GmmM3UBQ_?{O}ezS2k!laugd z1Ir_FhV>XE`kI825}tv+)IB>GA|aX`l3zARzVh^mtAJj9(*cEXFu*ob$DNdz2XU~i z2_#$LH*BQe2+BYD+n>Mw_O@9V+{Ke;RKkI7#Pj9l?|qGXALPYhEu6^-J-IbJU50X{ zIOI^>v+bTjs}Tcj-aGE?9ypX~Ij(SBN-uYL`EjtzUD(k;DZ}|Qo@c%6rTaJC{8F~Z zZsl*>dV99*V?Y^QUSu+;4|-wv=Kji4YFP~Un9<7SLh{C zPW-NXPCbfpmTUjvBD+6|()vW4JX#^Sp;)jJd6`7J!f?Y=bnfGtu1r8Bs>4$Z>`>3O zV@M-qZs|^u&C~w9(%w-FP^Lj^pE&rIG_DwBo7&!Xw>HBeJ@8k6hKhYi4d90m`gxywP(4s0hmE;XK zFi4<8vmRTE*!V7bqE~?8|7>%gHbZYz1HJO3TGyNObYj>(6(7MN9FkiF-hnpxux4G= zgVgM7;UId<@d%2JZEHw1rKVSAvmW%X_VTtw)u5`j-BMX4C8gyszJ>us@2N%dP(hr1 z5$Gvx@2|VD_sz6Lw9r?YW}k3{pH+peS%OLA^c*zw=9%I>GU7{H{I-z9Qoz;h?!u3H zpaA9%2)cQcy|>1iOOxQbhFg|e_)*>>5WhU9nyM*dE-6$DeYK^nyj-fWeA`k!N1Ra* zv@+qkO}jGaUYvD$gm{F;P$tY2)HrV~4jl}vfkwp}y086|eFvW-=W_eUg8moEKk5VY zxdUT31w-4L>1$^kOa_xH8h2F#Nxz&lRaC|ut2EZXoN9Ay2}=D#z3G17BfZ+J;@kmI zssgWbqA_4lo1=N*gS@-hg@r*%nNrloukT;s$HA;s-iikG@^XXSjZe1kAAM7?&;teui)uG z>n|(nn?J4x&Bky%&n#ZoJ^-bDvv&gwh;g}YAr~+@X^m&Bk8{;8JJtO2LSyIGTN#a| zIFZOlpkFwB@#@P|8D>9vW)>cf64eL5!O@cjN|K;)A)+WGtw2i1g`!***rKN)f_pIS zd88_$fWW$5D13N=5`(+m;rT)&0j`)CG~s?gc(dC#xIXMgP0s2H6}tZLLvDm*;8&T8zDbNid9_rkQ$Ff;AgwG$v!)<`OCR zS~fAlk0BcB!d}LaSE&L9?P~jPbCS*qtLu$H7tqiuiT6?0y2J7r*);nl393Ht&l*~IKASyR3hA-BqUL;qzZxDDNnfGDFcD<-gOozdt;=5 z6qLzU^qS!D6f{V=4!{d}Fz>11cXQ};C^2jGw%h$B6grUnkg-pRn&uXp{$ZYQZu&3N z!5>_j60Sp?v0PvZ3c}Eziq~ym!&e-9jV>e&=L@3Ihl(-=`#vbtg5{o`9vn_GobK42 zOO$=wI2{zA^i7^Uw`<^t-lyS!ewA+IWlMoe6!tNMxOsUnh2YIUZDRi3utvyr;f!&P z=5*9k6IVR9;`ExCvv&90lnigQxc9<_na_04q}RbpSW28QTx=Q6>7D7{BfhGRYZFlm zDi5pJue$M!WqxDaEoVHI4{B3*ZRw8)z+9_BgZC!%-(KN8Gk+U4Z#jRjjd4I1tZ?Eu|0CEc`m|Uc^4Z zUFhQ3089>Fw7G3-T};?Yz4f<&HSj5S8`XRWtt^`g73%whMK!ZAKhDcwL*E*v*=Xy? zc3`WV+Ama8IlnWzln_ifZG9n1bUu*Ky`iO#7t0SY_@spFwMy{Kvj0%bc;TUrWF&H` ziz_r&9x<4M=hZP9PZOtbA>Vd2P9(AN=>xR9U$rSxYtwt4;G3n6O{|NIMIG#5?yO#19bHnLFSGGS@;?5P1}P4J?(UmEAR@tr@4N z*LDa@hJbB}_fQwkAARUZVsP%UhDhwCIju#_3}tYo4)(U zt=;UfRq>BW0C}V5^X0AZ&@itt9t0mWBrgO1@xIOsc-pBN^)l0{GoHPhDak@r|MhNqn!u}(!@UDS6 zsB@h69c?OLAogBM$YNd`dWoN(Uq-=0jF=r(8MW`zwsWO4qG&4;U=muC<_c{NtAez6 zF)=!rr(^io36aW!SXaNIyDzmKLU|;%l9`V7#unh9&CQXBC#HX3}9jQ!xSU$6C+KYl(IP*VL1 zu#iCUs)U;rxw1{OEO-e9xGsU1lLk-VS5czz!hDV}(-*DX;G<6yRp#O^E;r35OKu0X zS`A|XFZtN(7`l+UYAgK-<1))H_uyGZ(SSfDM`3aaJn|a==|bBW!~_YTVibJn@bS zG{SR}gf%8GR5 ze!Nd0ShPgPw2cil+jlkxqHIvTM^W398LeObyG9D`*h{4mCYj_|8!bKetPIi+u>@cQ z9KAuz{d$3TTzb9*BGL-;6Skkf)ovh2Lhe~uu`@T&3Rt-m^ihDIe#x;5K_IRO?4i)s zjPup;h&FwfXt6&Q)IE8m!SI+iy7|NkxApWn(~cWT z$f!R*IT~db3f~&}i$CogN-EYlrt&3;Ua<4qV|Rub|u@LrdMyyJ1=dLg_3S{n-MF!uAW48tNxjNS|J8d z@kO(oUbO=PH!NuyIqToFekHL*_Cl5pL0K<$`VMzvq-p=BH_BIrBr0fDJ<3%>(PmfD zifZ=4=BMjYtS1(2+Vox&ZoTKcTQX%(Ry^}}R)B7v->Q0bQI1qu`&(V!UdAO^?#K7( zHxAu#_iy}VDI(egK*8JA7jxM*E9Ra*CrGCREosy=o&DPyZip>|adENDkONe!`gY|gbqowEmD8V6*@2e~GzQk9f z!%52>o7VyIK7Nn-JaE4HR~EqW(I-a27860o(O3kG;z zz#|sB{AX43;5k_UtZaW5&G}OtvAa}!5ODouR_*7wu*-YSY|{y~n! zxl+nImZRuVlXDd)jd|WjA@fN}Q=Uoq)1JqBYqr$*6PcTZ-6emVBR{~HL7}$rSN(UJ zpl=@=5xc~bboq=2CM5kcId73|hGl8jp1VQ*baAAPyo^&5G#Ts-*ol9k;|Hy%wsr-t zxMOD@5D4F1U%c1=mpMk=!g32xZw~f7Gr{}?E9hy-``u?viOwhF;<&MQFgy?ZfWXo$ zcpm+|5MiL3H;|uULMq7>Y~9U7Ywt0cx+ZMWiw=TWs3hH0%>E@r-p9beh^=+M|RF zYax}El^2F!LJ}tC71p|PdIVo6InC%t_~_IpGl)4+Q#wkE#+p0-q(3xSD(5+xj5E~M z5K7kEUCf2qlJxU3;-Hixm|~?F@+Y2wh!aafSi09#(-8F;)+Y}x`0}jUXsXHwYNG$K zF8VtjU3X0IY~T7LZ{==t+kK4bjUQYcx*p0W^wrz-Ui^tFjm~}I$|;m^ke&tfCtX56 zP;&Mb%%iW2LL*%lg0ESD$-=q3FSw!^&-0h$Dgg8hCZOj(b5v@%<)m`^N9bc+>A3A$ zxR1{?dSt)icEHcY55om3rN%@p;$Rb4E%0LFKVF) zdnk>KGwok-XWc0Qe%)3awe>gK3?kY5kT4R7O?m@#tY>F^Zo^oO=Xs`K!}qqqlxZf! zqmPsY0$pY>o^tlg7u?QC&r)oM;AE`ysj=(KOCOr=Y^!X*9n^7%1JNUAB^g0|Jg75Y z89z$%HX(IAP`4v-Y?Iu_Yg}>Ur7nlrB+Q){FG>-{TTG9$LF`J`*oTxHcMw0W9ICDF-ldn%F*wA zM!NTRuZqpd?n`z^DuFaTXX<=CP1s4-(kjXa_o$1WbLQOxS-bTE;Z()_(-MCQ{VHy` z{3z8k@;h9VgJXQLNsXQP9wNTVDC{`OHo@`wFnjdpm9jag?3Y~?1M(D(IO_8h{*C>> zm~#OTlkN6oTrIk3<^)nQM?JKr-Qji5@;XZp1afXY-_V2`0x?MSe$@Z|dhs{$27?SU z>k^Keg>k_FlZK&4z1hCWTjsr<_5S$CWNJ~7rVBvw%kwE(K}ybnZ)dpnZut-C@-Idj-hfdZQsA- zN~LwRXhUseQ+?<5H6-rsIu#R~c8@|Sf5??Bl{S-D#+f9)g_Z7(c}EA=pMc;bY-)WM zJYFqEzY$aGtAS1k!~9JqiuDDU&sV(Y>fEeRxh3FY!u`7w2}?o>#b`{jbjS6}`a=E8 zhFKudZ?qvyA!(7RXJJY%5!^4%>nmloW?cR$3~+e5*5!01F96l7PA^`Wd3NNK!|Yv` z)h~%=v7sfR7pHSMTTSx?Lp=g3g;hN&q6ncxUUW8#0o2nE*`5#wu%_&CE5K=ZhU(RN zQ+-5%)L9oc(^Q#Wdid9SvV1S##vw`*umB}HCVDY66@3j)Ee%fQD^~_%^s@}U!OK2< zqpyIVWURGc7xnn2k3-R(lAQ)nk-1vDkl|9_BTiIQI=1XO@hPObUr5%s4fnyM;NMy> zstlOTVTkrwuvThKDAB65T@R_NLBnx5xFZ%o6O{IsAl~>_shMiwl4&|8<0{3~{&w9W zlcCS~Axzx<;tDI(ZnLXlI9M9%nLXnJ7U|B!b7EA9;)huZtCA zy*lb2Vtbl}q4oTvvT@QUAPkS+`1g|DUaDMeqkBh@CoiU`UmMabeFb|;`48A)c{HFw zdkJ@xRTfyfO#;+z1*#XOqwl`nD*VJlTjbp}NulXy!itaIg;hX1M4%|7ws%sDON&0;dw^7)_B&OazadLrDU^A;!R7wlqOaqptn ztpvI+4N)p6t?;FuFE37UaeJ2)8@2+3E{Hsfi~O5fBb6KynPQN*O@8l|y(FOWtsW00 zGqg+{pu7518x1PXeBP|TW=)M>_UnF5VK$u5Tq11$lZ8lswau@$LgT3ch~Ld6 z^Ch#TC?f2jdU>m*Xe9xgDv=eUkI*^vPZeJ#ygyWfi*B83X_%J4l zzP%FR`Y90aNSqOc%Rd_IXuSXVIZLm5RA?_y-mX0&FqeZVWJEBjdIV5{n<-!k)}$Sp z25F#~>@pYCX}&9I8vgzJ_qOkEMdQL5zxy)u9Dh%GxAaw*w@2=?Y+k#5J%YYmo6=sc zCmdP`)Fr}ZUxv~E4ag*aij1;I^gr_{5nzK<*{{AalsVF3Y*F<2xYiWDQ2*4-=71K3 z4)nrsrnG|rK(-C|K+UeONRor}lK-s@rcgJdH!1zXMatIJwhg@YWFKX!xz-2pKDM+W z;?N$@M(GJ8ppLFH1@Hq(IBj)$I#?9}fLp)Q=w2^snOjg1wt4N^wQwLXrTdJQ+qQj8 z#sH!bL&d4gN}7gwrxH-4 zw(1nV9!mlK&f4Y6mv>~q+xg`SMq#$75H6jmScD zz|DRcYD)iX@M&f3ct{3_Z>js;-uZwdru5N9Q=~2r~JCldPCuGbW4e_ft1n_)?QRb zyQEe5UoDRdbKeHfPjIn3gDjLc|u-;5D zQB83I0be%5EIzPQY2`<#uXV7VXbk<#t@LbKY5mpW%y*jsddQx!+2EBaMhGR3lKNyS z@5?+quaME$MV|@>CX1u9vNz>2iUT{@<^n2R#!40VCyO3aHp_Fld`%Gd)G{mQRb%*~#f2X?#FXsKKPc>;A~jCTC4 zlGPbx0ZtV-Hpf+*D9nPKU^1WV)J|iU6>;iog5JVd7zMKue=+$Z0TZSdE}P_3IIy)` zJAhsvMi`kBM-OobIa@nL)YHR%ez8k_L`&)#+e}VUsx(HeeDQw?kT|(Z85fTHP6AIK z8z^^el`Bcu&B6zzKD<}*kQ#AJZ)vQx=DD3wj=4Ar$ma{m8C};VngfdI1o6PsZEcji z^A63nkW$iv!bKJ6r8EF%s58lU96)&`%G&4SjX;c_V5;g}Tc!z85wz6EyY9_fDycG8 zrw)?wZGWk6#~{p%W(tm-RUPYyVVyqsEA_Vml<2IUqHh{0IY%W7VSS#Om5ICNcco(g z*pl&8!E~#VNzg2aW}AV_V{&srEbWOy$#iS5X$yfQue&Q!#^56;yTv@%7fvneAg4SE zoo9cZuiV=85v!Ogpg2L!&A$Woq!clt^+42p4?Pn}?~8EL{4>iMEM0(0Fc>b`yo;9^ zTBXl9$RfHJqaisN270A(hZKymuwSZdYr9CJ734@K#ApDZKhmlGLP2Bqy&0H4bBT&T ztBCqAJX&GjGLy_So>PIBj*!+*=)8>n8~Dt3GC`*$BuN)$WM5YB=7w0v3tYdew4hLF zW7;KUVe%VvWtdAWP(Ipk_5Stc6dpVqqI=45^XC6L`w-LffEfJu_P;)V!X1}(_m`qc zjdnZ&#rkCTa24jS@LGH1GhYJ0gwRbOz%}ccCJCw3-0EQkKgxZa#{p>gjmy@czv2Z8 zz(ji?0~}_tf+i?8KdC{n)HT?ob*lKfe%R{#4TVZoAM3(^qtVSSstaHRD$;xwo(YJ<;S3q2$0bqejh*C$ns3D0n_(inC0$aTJ*3&k!dV;CjBz0`mWf z;5don%%P0P$KiS@H3h^QkL$eXE!vwddH!JWKiAmjIAAVzwtd8vyNR+2kEu z8YxRZ8@~1_X!xFzEf}qzy*&Yy3<8)1?OfK7F|*o2@z!~Mv>MoxQ&=pv)0+s)fO@5w z@L`N5u+AO%B|g~D=*-N-h_zE-dz2@>Qu<>gaGF*CD)-wQ_>kiJqe7=d($l7>YgkK2}VM4UlA<$ zoWiRy>s^8e?xgj+47x{FsZ}Y-G++2U=Wo=TJ%D(v<8eoMKY6#TJOs|_7NbJg3>Xc8 z-Q=goH9rn$(tou5nd6!SK@O=Q99J)wki@M05Q{I@X(`_l+13n;dXJGBrs1Dqpeob8 zznxJ5#Y{x*Eb!4xZm2mxR)IO!z$M!{NS!;mB1!eYZAe;I!n#9Nr<#MPBi57PA?KX% zjTiOq!(9>uAC9Bg`h52!6Jth-9D<-w{s93i zV1yj=h%b&as{@eTXIEsUqL@4im<>9c#JRj=XQ@G@GMZ5uB(RT6G{K%UasS87ttI;o z=yh3i`YWAd)xf6h*xgz>RhNmqdN)&fs2-Yi-7T5?^%kV^@!-1oZp>b{VMUrx%Zef% zYt!vPXbrVO8l*l;t<5+8@Xg#XS@b!d1ZZqVql+7`5wa|Y;(>M z*`@=upF+B(__<&}51TWdLm(Cw0vt;q8w3k{(iTBScZHIqBY_fs+Fi{r|$9 z04@|Mrx;2U&GdcZxm-*Z@?o}5i^tzz0LKlHCg|PtNhI7{q6OgT)*Ea0U@rFJ4<7z_ z2ZlTMJS5dm71~IEmvIBawvP!}e}F%A?6RK5?$-WVn>E@!RqItjh5?6=Z*-ENQa?4n z)V|R>W<{(c;w!IpQ|@cK4-5d#fr?nn#x`?HqlxDqOI1PJh;H4PkR zTls1*GF_d7D!7?^%DTc%0srU5lA12&IPk$JO;I31(_;o%&5IMRd26Klooyz; zRhHw)x90Dbh{nSw4x3RCGLjBP7At`>(q~$x?@oEyQY{nTb&RYmy zdNd^#@qB9wCMnH%hAF;bM&;CgzjL{nW-h+VwDg~-8gN-41f|i<`Gi2uN+zaaOPXl( zamsC+w)esC^3<8!FrkZ@H2MAHd1=PaCvEpD*|X7mdT-u$Dc?!DDA4;CCAEZLz$p=? zk=fdjNU$bXIJFa);GriYhhUiF31Uj#7$&JfUOFmz_44ow(QKQ=NORR96CPzM)<|OE zeoN5G1#h>t*-xvPBV%r$66Yg&#EWfgg{n;G7A_@U&0Dv_Gnz>w!ezi^>UGR1Y&vxMXMWqNHvq*0gHJ zs=|)9=;zrM#9Fi`&ZQ3P8<3e&fAE{Ps_&Ld4 zpv)$aLxEtiOL-)o_>`q`cWIuotFj%G z?TWx?>Q<;7w*9(cnv`FTJx#5SLT%oPkXnwsn+s(AX4N+M2_^5Tu@UQeM?5F7vSrnu zvJc9eZ%NbG?+ZELH1(Z0D`J%o)z9aleB5iGHzD7p_~(U<48ZLK`){;c?@lMc`bpVw z0%@d1Lt37EWaxbcUqXIOPOaF}aEQr~qk@V*igVO809JeYu0a8q_Kb#34dyrn7?=i@ z2@Ha^UYV93))+XIg$yXxAnn`5ls};FaY=vP+u4k$bsN~GE{|V-B&e^i-{z96($)CS z_cu2it6@u6mQC=4S>{B~aB7c?m;q|rlizM`ZgkM+6`fLqHEymx1qi=!22>a+@oP`2 zT zCxf99$@bgRDb#gX#qI;kfMrBmNMd(nN?O|_A#5oCWCPzbdpRR&jf!`Yh3zn#r7KwU zvwqDWeA2v7YE>aH9M}VBHJLos91!E#R97cI1JoG)jR{Re2f64@iZAs8@cF;@`AjwW zeWC3D`)z*`OMdgm0cv|e1Gb_ZvAc!ROsr%8&7Z$65^%+`l;yGNi*rOvj6U_FN#GNZ zL7Argj1#%NuL5FtBOiNLC+d6g09`R^K=xE-=-O9!reT=c&NNu<^R%}lb&#`!Z5~r! zyWwPHWb}<{thOTGc-OD;lL9(EY$y$Ksz*l|r0|T%YI!K-J)X!5q+}pbqY$N^#ImXfv*>IA0x|NCa^^5gW zFF8`jPrb7GoqN*7_1syk2A@!AZWG!4NG6igh*M3! zY1+J%Jt6Axz9bv&Q85BUk*LI4=FZKIItZmT+8KEKICa*uqAbl$Bx38enh$JwP4^jX?$EHTH0T4I9A5 z)vNCV<{j^uO(T52fxDyK9JLvjap9_Ybmh3E(hxO?beYswoGtMBn(U&gq5BZ^K;Xj- z{1AHEV*fV3nFSsj7>SZA4h*;C8@!7hSMK=qDYjXTwRwj*DX-R0_liIdLmFW2ND|940=0*%kK8L>g^Qa6)1zK#qbL(zLmJ3 zF1c0gI7cnEvhV^^NuJe7;7ZTdl$83Krt5)OoLwQXbdU5lz~zwtrruzVrY<_ z+K}*mo8dIxSw5#-q^f|xb5f)F7bURPedHAr10%9^p#|;gn)3^9rPp1KxS6o#X*5L?z z^MeIuUbx6T2Nm}LKPM4o_!Ve;(<@q4$Xxiz-##x!^L4qR630Pun>S=aam5 zyeVXvwg>#ki};!-W*}^2f!{6jr`;WRMSxD$CP1Aq-K9f(EA^$VfHfVsGSk-=(m@8% z?|?eedoD^g&{YX#hD^w)lHx!^Q*LQo=%m6;_H*mt9be}ILS5Yf+WvN}6%_UX&BU1D zOD+>dko-ZQnn=kE=wEls zjK{SY@|yy>HxD(=P?Y}Vwn2If`uCAZR+((qQk2H5`csbM+Y}Bec6|@D**4HGcWC$T zV;T$!I2@#i?)Nb-=GG%J0WY8cbnQ<~-?L~pVdtl4086;`)j|l!B<}S-nkgZy49S^f zS<%#hkZD;N#{3Xn@E@=IWLIsev+L&<*zC-{lM;QDG_NFg>7d?OS42rxn;5l0E zHw=RNc2OTMIQ)HATUmD27l6RSedc!h6bvs)EFu5!?DJ!1N$n`hHAw^@Az#`toh-fv zmOodX?!hWUv@->LTIC+U2PY0@+wag=KJ56MTQI&~U@&_^Pw1z&hgjCb+1S9yQsodb zTM^~ub8Ib~+_v*OBlQY|FJ(X53B_|+Rg1?$W;OUUB)W$u+WexF&v>UmmUv%IW4Sl_ z^a$8aKS#Yr%inwA_<8`xT8K&kGf<@z@eUJ4GkOH1)10!8kj#*zaT&l(Ka$DS0CU7) zj$swjt*y=0Bwm>jJfQ&Bbh2w%Rt+C)dpGD!6nh9E@tjt_iY-2TV9M6Pgjg=AuPt|Z zRUnOClZuS`>i&ytDKJM2PzliPoSI+We9*`~)MV2?-1&rm{UA$xIUqY^?!|c`8Jh6h zQ!L(>SSfG`z93e0s0nnS5TV-Svufy2YiOuz%=2l&&hL?W#%2!Vuyn!maj1rw_N*_l z9Bc8sMK6zi$nJJ8v0Ei^LuPk~^-yjmQ-)v!s&iaCrYF9R5HLmut=KuYnVn6*Mgzg; z%5u>2qx^H(3Xod%o@2x(Tr$2(LO1z=!zH*pYG``$h1nZ1YP=bTSYRU78VC+;rxO8w zn`ZDMO69B!&gLRs5pVVcfdl7ju6?BF{M{_S1;z<7}QZ^9e6 zio6J(w#Syt8PWq{%k)B)7&;?EEYrT9x$nn8)+%#SC8XB`Y?oFr_m9^_HU(&l@5rDR zFVJ?u*!Z}-y!>%tkou0~&-5-0x%d_V*7F+xMHM<$lXmO9#I=!q3u|m7woMHXoN0yn z=deM|NT5&jT6k>FWfr?G(8UaL#04rZXd$VV9v00^pq$t*+_Tn*2Up)l6JK z2bqJz#B`<&MWol&)qQm3rm4WR40U|t%?|Ui5v6)qyMO}mla1XpXmyGo@5MAG4-(HL z9cp|ubU_K4o*j6?4`jC3rQ08Ttz3Fz3u-*eMJ=0tF@*t_INe@Bd+i+KD(d*HdCm}dO=z)Emegq=&y|Ed z%kTKadR*N(9SSeeS|nQ5TwFNM$0p2EJ{&p7meUowxjh3W^pb?5z*}Q&WlJ7(FlE($ zx%=tHOxY9B*pf^EXG!UWE}IXFW9}8(W@nWlQ{3xZ-~Fl$W7P zw>zVRqGSzzQ>%phqxUB;tKp?El!^`bUX#%V+q@nzXYC2nrRG_9WNymR;P3(jZ{?cFcnuN zMTCGKe7;uvOfgyBQA_g97C5i-%t4Fbny{dp-W3GHl&;s}UYR5bTudqu?uUl0xTdIY z#B-lj#VOpOKAItlh|0ZA2+O`Q7koR3%`$GJmJ~q30qrj(Ei7c3WVy-YgN#U0N%7Q;j@RB(i82QA zV=LgG+ew(tc|^$3SLF!^mINqLhc6sGHX?5s&mtSmqUF9*6;B};ZF)kVp zf1FVYX?~K%$+PtPhVz>IM|{z?c9XN^eY%u`ar>M6?1w+wXfYS03vT#Hfo{DWb{}@# zI;x#3zMcrdi>!IkL*5-b-m#(tGvyr`kRAzzJ?t>n&phq%~$ z&%Z}R6D|^R3^#A4>Lw9+EJ8uBcaQ((O;gs^JLCLY*+B9N;@)TJU0pFW1&DGWe2C!GEySc8hK=q>l!ZB#vRhLi+KT#4{MvgRfG{;k1 z_5B8hv3r`O#A%?`kZV}k6#pCzmSQDuc-*x|(9c#Mz79(f`>;9G+j+_W1P~K>qZ7M{ zFY$8bgLm3xRjoXtOJ>^z=14-=>Lq+t4%Y+m=)!2ss>o7gd~y}7UD|9`-}W&gR#d1; zw1Fi|*OY(27NPYos_&u)!Tl5Oy5z26 z@??xSxqL%agO<+(K?@VltjkDTsQ)(d^=(6e#T8P0aRd?}`WkDALcW%1LX6(rvQ-UU zU1pI)UmgCMp-BN0o$|ZP&8Qq^Dp(oo773+P~d4Nm@B9?8(Lr(N44Yhk05+O2jW?axrhsK6DqSt<%Qvl9?7gmm__-wBwydZW=;rUCfK-?pv;ziHDYD9l}UQiIU19Bg_N)u9ci2GZ=WN*eZbOiI9s{>m|6CsR4j5 zw`43-o3Wjrigk8QJcqpc0wz~D>R?bcR_AW6B!hSoUpf`*selq%;A|6LD~c$S=i6Cg z`O4Li+5nDkfUMCdN!B+KwO`0p1#HO~>`F0$VvKiu{1Ub(pOonGQUkDcX4f1J6>q!PfbFp6f_`fUhb9 zEkDs%eDfj&4GatSg{3V!V627}W%JVkXFy8#mJ$|cW8~h#X~?^pL@me&^T3_zt@kQq zHzU+nR2qL)_<^Oy@a39bMB4}+(nVKV_&o7Flo}Ex<0Wfhi*wOQ8bGMto41h6<(1IQ zBl*9Eb0-9Q@!0tMRZcA7&@C^av5F5CZIsI7-_Ggr&)s?&t3XN!e*ZwFAK84CVj~L@ zFjR+@T~gW8{@PCO1J9R!(@JYtvU`-|I~s7n?4?=*B%+=U%>9%6pBe}kvDDhGiCGkm zNj62nUz_E_KfS!O03skb=R7fuy_-Xr4|j+K@W_(v%H7g_WJm&N4@4gni^uNFY|{E~ zHav^D8vKy5G=QLc-R&<|o4?GpjX$DS5z(ssidWu(q9kjnk6Lf%2{B)=1X5zd zt51y`8Ao5_Zn3DI&X=pXu&$XX@iW%)pTMU#W6Ui(5Q-g!m;dxgRvnh+TY1OI`7)#= z5uDHA>)U%2Qh}9Ed(U$Fvh>IFVkN~~F-A+*?B8tHEVYxLZ(ze*IlK1 zXE@{Mqs52(x0nA)`dV0i1buM4yX62s*00OYuxce2JpwJgV-0Q`drDc;rf~hKpC^ia}<(kt&{}Ybkz168Rj&f+N_grOX;bc zAGydQe+(}shs8YUxTXhYc-MPElf!~3*qG{3I+z<7s{yFU)qQV}2FmTYhE~e^p-)h} znAC3dM%m3&_-_CU#ZhdT*44PL0d<%fT;jd?^Srb!D$b0*t!+O*<6N>T|N7~ ztg1~w5mUe-2<=r$GUV$A`Ye})x`p3RZ0!@Fokoh|526U|g2l&6rnuc6a!Fy2M-$?P zhsGZe-rEUix$t~f(rW?&yJmZz45Lr6Dn~hD?Ns;_{t9Y+|$dF2Tb;w_Q2w< z4zXiM$pl3tDp*@sLLn}gReR|H4^HZ_kaoQU&FmWrIrs7Y=kjYW@R2Fc?P#Ga!;2sk z`NT+NfrnPk7$))mod7=)f71V4W1b|(I;_=Yu=E450VSq;VtHI#p;pRp=l7>kzqrj_%ccVd|hy@*a5mu=R?iV%|4|9Vb(HbqPu&z z2(OgHGBieW9%Wk4nWmG((<+?#TC;dHmgn-b-)kn?hmzj^=61;X_sBc_sOobAE&=7W2GCEyV^`)UV-oc8!G?ASL+&<;GSFu&RBE#XHnjVuu) z*ZkHY1N#D8H0O_{+f5B8N|0F_-b}I8+W~?}*Nl_cH+X^SNP2RU^Gzsg_iaPzeb^~z zj67ABn@Q5Rgme-Pz@|K^eX9jkT2GkCv(H{lWBA@ zI6W1_bef&t!%vAP#p5~e@)ZIX`DRSrnTg}f^2J_J z#${jsXjpemgEln}|5kEsu3pTXqAsh|+)yDX=TTjht*08Ps)a+}C6-E9b6$`jwn_{Q z6*a!(IX9|;oULT(+qEa=Y8Fd8Q)Sde!HnER7DIl|D3Owi1R>1-t3oLYw*j6yYW;pB z3`DI;R6>0M`Y^9vNI#smc zP&4Qk^b97zZpmn?JmBu}715+|RN8B11i;?vkQ4RHvB%m+Avbgjoh_VIA5)@Alw@C# z*r7e&Z=f|nUL#^kAqQJawzI|Wpw~;Bqif_ARZXfcy1t741=0sW)T53Cp)M3uaq@w+ zq;J0G0T5%G8Dt4#kaK-Hil;VT(+RQt8+NZs54E1cGQ39K$F1G96@*Ry?Nc3Dd823c z#Gg=DR*9#O!avxt+xhcE_sUmh~^!6xqXmO~(G0T{K1EaHH?t5FQ z*D}h?MT||;Ld)DyA8*-nH=q1?L@AZs98V{WpX7F=@Vf2&ej{a`zmzGpY*TiuOx5T< z&tcR3J2R#u=W*oj0@d8MM$U(!m|qFkw;Ne_(0qAe+yXid?JA|ii+g6I8*IB4#a67} zIQo3RBP~yvONC}XKx-4W&4H?Po#yk zk|OF!$`UGD(iAO(?6Omm-Pp>=3|ZP2M3%B8`@ZiCT9AET#}*+B3PZBJ=QVns&+q$r zKfm{ncmDX+_vv=e?Y^)3yw3ADj`KKU_q3O)-%@uvyo%>^n{S>jX&1lYu6g8|w|w+L zk&KkTxk%akx?Z3j||L>_i<-X=RfQSY~ zB_$BiKo4AR)`!iuCgUx!{{&ap)y)G`Ucc^wkQdPzP-^bfod)`I0JhT!u&GFO??G~B z0iQqt?LYYr?KiYptvMjeUqoFzcXUC(G*Jxnq??P23qwb!lyh1-xD4#^RfD&}4S0lY zOQnFaXQxBU>pz{#AY*tKR9{{F3)TL?VPOWhx!U#Y4 zn8&iD!2Tg?a(gl%@_YP#a|OY~{f<`Dq_jUqcJZohm$U?bv?a-7=R|jU>e(d8h3g&T zyYWQNT=tFECBDr84eBP67B<^+@7$;r$t5t_kl3d0J|ga-imNAo0!xCf)zqN-{RLJb z@m#miQd10$?08~TpX?|=dhYnn`aNMrEuf9fGe?DlOUPGouz2l=yG(kBmYm*vl5?Qm zKUQUrt*et8ql{jh6F(8`|x$ryBtGKxha)g4-ISO?M~O78tyOig|xluNmr0bbYO1NQ%*Gsj8L74 z##u%>Q(vp^%3Nu4)Y#=_*T!!!7N*xY@j=Up|Go3tmbfF&zr$9C7qq3iwYkJ`-M5O3 zUyZ+e9YGDEUmbb>OtU_&Tt!P+tXFc+OT;gY{l6t^#CQ8n(#QUz5ywOgY06D)i51ZBn%JFm^M@YR{^XEq zfXinNj+5+3B7Do26B?7P-`i7}eX}my@4RNTs_P)hmV2SP2XcRHD9uHB%)&-ih z93AKUCcI?Eu5%c4wG0z;f27tQsPE!pE{q%}q_5~I(P6=HDNQK;x@1k$ zhs<`y+C0Da6PAB)O_FIQ9|)G&X7wuvrLUN38uorofG z>S>c^+lG~v^UWJ0gtA{u^H@AEUgXa4$)e+%^m(9tV^@o0GM7$|O4Lkl4eNNm6Ay_$^|)5l^f^_%W*_oIZP-*x~9hiG~PsRTlzB0^a73NVR zyEO51*WrjCx8bJ(^w~8OOelHq)?Wkpv5puWvgUdo2t9O>N0B&K`d|~1gm3#THB++B zv2;rt61_}8z!Iir!R1D-XzYJHbVvPwxQ)*908ltbMj@B45!ILb1r7KwBr{in7NoBXm9UE-l3*`LzFAfZnx0nbD~;VqT`cWNg{`nX9i+RT)2_a&aACxZ zsQ5RI&25OV(Ju)HUG)ot>g&G`>-PVt_g1lw7}@&@p)oaZO;smI$QHL zFou~RkRgMhKkB8|B~@RfZQiYf^}!g>@%;`84GsNz6n!@@eF9s+;kIG)_bVE{+?Zf# zqsDyG+8~~+1L(i)O@WQzxe#5{6)f#n6Q!O7Mg;u}?eqUrC;dN$aR2k$MYr9z-4XkF zY)%)f*odGTN0w%Me0oK&v+8hlpAC zz(!`U+jefW9aYjvYX2DWf)RRcq6dC11ubppe=pIeKy72F-;&NqIf#VTy zxN_${hs9!>D>ixTedFi%?eIS3GPlIUl~6iNz^fFBfqt3kQ(~Hb&~2*qjk}GsulPdm zEMDtbZgd1BW!nXrFaa^;17`_fx2Eu^s9uVazm!bzb?@l$sk5n_TRJJpSDkc2DO)I& zLvPVXDMe<=iUJ*GupqB*Je35NxO#o#fjS+V(0ucr%G7s3?*fOt>=-}YNcFd}mcZo? z>`%#B!}jQ;H`MPnG@$*cBzn)`6tUY%-IgYg>KB=1FU`7Y<(oZew+*nZu9go99?PyL zJ9j^+momJiT)SO{<{T*Pz{5sVLTBJTVR@o`PU&MleGKGxb7-7X zih@;^zH+TK79ELs@ep%}KLD}`U65lCzmJSuH4r~>qN>aWkH;ql?(t*A_oAZ|OiKBt zVhE*T$v45jqDDckJLQ2=ei-2kA14p(JQdIj%Bld_pddH*XDpY?cfV(~Tn=CMteFCC z{6$@W+Y9VXYQd~ZXp)?tE(TIQ$W~H7BZ-{1$UxFpOvCGYy1J$z_u`uFq`D&iR1BM| zMo)EkaTbnf7DVhAh+5LK9qtHUs2a+(b)$Xhix;N{RgJwt_hMV}tMc_+kdUDjcrG|4 z25)Dej<;4(4d#EubwZq+0+ijs7pSMl%(T|IZn(wjq9J&<3f)ltGOuUAk% zx_`_2=fhtLs}`RRJahl0->`mSZOoDLW3JjAzQFJ7dA(Z)143*;C9?obLNUhzrxZSi ze&^Y8=tbI(=X~F2UbZs(5}7l_9)cGtds6pJ2>Vhrn=>|m8T0{pzzFt$k&l7L2~B(D zu{WcZ2aoyqd;}{;O60?xSMfVNT~>2Rzsw+}vVu;GFvyX;q>vC>L zJhG**Dd@c%?U?sl-~YCB+D}f9*gcz&uQ3@Tqcs-r=3(4;qGfr>nbaaBj>!VqrY0+@VD+#tx*!SgeEFH=&ZZI z@#tY(XR>x9*Zsjw#b9p2m@)_L4rFldG6OxXb<;d**=fVm(&a_|X#%elqW|ga8FL;p zmZSa!p{Bba__)I9p`of3HT66Y%q$(wXY=1k(s6WjY+7K|=@6~21v|3Ni&};R0^y6} z4>$UJ&WY?R1mR}lICd^LJ7MO=+nLhkwDKI(>a=10<)hx_2744vEE*djZVp{cSms*bUOp#eNKzdZEb_({ODTsvB(udvhEVw-13);WGV`kO^I#*jDkM5{l)|xyyAsay^k%5fg9?^579@S!!U`_epE7J`HTbK+ zN~vg^oNt#8cd+-t$}hR#j%oPot6DSi%)G6wM%P^ZC#nDk0^rkGp)ap}-|^qt73DrNlXQHG`bGU}R( zv0fvVxTgwC-1(q0J4)5|&s9?W1cJl_%*PmqQ01{botaZ1v5 zk`b@2o$F4c7}Q={AE1T_H=2j$nd=b(H77e`A2UlO+9e(eu$bI(?Ywh}>jnMawcisu zjZ1QV^4GpS;(FX?-x$}xi3i+_Rd$l=?|;9qb-LoR?9H3Pr9G;liU3ui-*HF6ypAum zMz%xLgl>?Jm{_j)jzuqvsGNuF`bl8hDKwa6R?pr^JP|~#q4QSCwE8SAkRH;Oyk~L# z0^^Y%Ps)4}**sbc=K+r?j8Hmf$Kt_c3K>5oph?Sc>;1_ocM?-=UvscK<*Mvr;@Z1Q^@WW&h{ zwY6ZY7IWLuwT{_$x{|hTEviu%vGX+&YgZf|C9SXp+Ftq+eX_`NcfIuN<8LC5D;PWIgoDOQgau{iNzo+>bZ=QcKt$wEJ}=IZ$o}tiV$&YyIsBsFmZmV%4jICLmJJ z1f=WS&;cvqlAm5uY&*{mV%Y4NcO2*Y5!ru%@hcxMhY%+I*83~O{wL8~@6L@)mw--q zpO7X-$o-W3$CK0{%_A;Q!3kL);rr|1Z>%@uNx=l-`0x8U18w@hAn-YOR<*>KBppeX z+tdb)TE)@<>h?*_VJ)X?P*=WBt~xep@x612Qz@Tn?5&6vdH%Dyy@NWIC35m5!FnDm z1M92rnacS*0Es=CGLZF|dQVYNDf3l;DB16wDM#LPPH)LN4_$W=cdgZ(m_mnvdyjRv zx%(U)=M3nvw5(*O(H_ekH?c>X#heQqECM~RcxT>oOPo|PBq#5AnI%4%I|D z`=aJ4@6{MCfr;&+4ntQuFJHzEAg89_G~qJjr-&8M*>C$wIQF%M>SulNAgU@pace7V z6_m<7CE%(yg_`v|C1ORqugFNLVXl?@>MzlJslbqQS7pqzL5`YGtKxYfeesUNxPiv5 zV+<<+TsJlqsGH4*xQGtTu6p!Lnrj^#t=G zkUBcxQ-PeAwL&gY3p7rR17?XIYk?NTv0deS2ZY%LPHAS`7z?M@%zqL+nFKhH(3NqrUZ&81fs@6A= zhi~1<7SsBN_ng>Q^(}9%wOu=*iD}??snr?aXg{XmVC2;`Bce;!fY#!%tZKpm|LLDn!$^u={^`28$^C)!Vuo8+*{-2 zymd?Uto=zL;mFvI^8R!E*>tX+Cj3wK>NE_MXy<@9OKh=e+44PzHan;hMbOh=cuHf_ z2a8AKc+Zm61HZdV%vXb;)m`9w(Zr?`K#WiuwRK66HVICmTS6E7)UElwO#tAJ=HCbu zRcC_`jbFxC5?pDjxm($@*y2KRQ1h>5mzZHL-<(8{x#F}(cS0j=Fs?VqvRm?_x5~X6 zslNtT!W5O{^Yq%2bfzZ8r>RN32DP>I4wH8Mhc4W`RPtl}X3P%ekwA4l0jvw}2li#& z_H{KmPE)`ooI1GyvMq(CsCq*VaO7K@?lI`>S9r?hOQj`9F#+RJq2#<(sknD^DoHux zPtsd1Ogu>^^d~sKYWZXtK`^<`dNy57IVQloi0`D)u40`I0j7|)RAb+pE^h<1`Q7DJ ze=3e&8E_fn46f9apn{C0_fqGa*B!=_ta$EYnyNA;wU$MdyIPm@4xP(yA66Q3j?r{S z>>|@M%k$#Y>$ExtTHJ2h=FU^q3rE=woUd(UimR9;Ma9l!R=R00Uh6w1iMn9f8!YJU zGZ{PQR7yN>gOcA!3togJ%OZ*N0*Ut9F1}G*!}m$SLDe-ij+?$CpODeU_jbS&-`0H3 zhVi)HydEw2vXdO<-mx_{ETBlNHC30y9^~(DQ!PJQ>7ux{zD^dm8!)}ms-9{UsF|!k zH!$hW;=XcAIQfdGs9B1a1tcEDN)hL~4OL#IF*kzH&)Lu&tQHDR6NWRn(7n{G!Gy6< zT7DZAeOm{TfPpF+0-AuyP4SFDdg26=ESJggQM%Q;xWf3jWkE0_a+CX41N>$C{B>eaw zwCzKLHvofrUB6c8Tx95|zh^7m1sEAPzA!Pe1?v6epGvB9YyEBKcYlP5HQ_f<2Yos( zeyNAd@F0s3qSl`y-pe|Akl{DewFcBU)V)1Dohz{xdS509bQtIF*j@Afc0!@=Ab-8O zsKPK-VTg3Em!-VJF84DZ7kfFU_IY00OE14+$GgXfeB4(%B%HGznjl)&H@If*U z&xu0KfIg{5DoVaTc!{43o%=>AACf>E-798NkkiKKllbie`{E%$5Ew?2FLXuA?ON2c zXJib9ABRbUV6@f=!W;dsrX}MF5<_Y|dzPS1{v7vW1v1%(LEfg=0NjK_oJo@A z{m0CkXPOI4CPwCA&nonIM9|^KM4wI!P?&+bAX^~4BujecY^)OM4maFY03;O|?EfhJ z5JVT5Ol-d! zGO9g8ix}efUVK(XOk50cju41VuatpoSz;>M z_kts8>fe8N2@GZLR1~G+$%;3-nTFkS>r&PkTqZheGFrFZ%bn&7Ezk^jsb!nxtuF%3 z5TO#njnLRUavaL}>>zI|{dF!Jz*bG9@4HFR=R!8rhCp|B0m-IJPc;`5v0plJ(a`Q$ z+ULN+qw1W^C>=l#BC;heQj;KZBcRPn{o}6KIoR@9o6l-;rM808EL(s^hysHQoh4Br zI_}=FiHbdwgoOJ6d6kL|Nul%~YKW@k{#tfhenLpU8@0*?xo zs`=&DHmt&p#3N5_k0<2cJQCZfrYG%vf^FBY2g0$$8zCy#Pm=9U~GMjRI zAFR~&|8dPT9a$lFRf}XeOL4f~aZWbMeW4VZ|Ecs{W|#M`JcmhT{alC8;yi8cz21vj z>3FeD1g#l`Tb|vUj0!&4>qW9{540X8H%H4A-|M8`y7QiQ2e|X8tj{XY3w==cBx>Gd zDlzjhC0&~}zqvg|Ru;;22j*OK8#Rqe*Dr=C7g9EjG4Oqd6k9-5SN-u6tmDOjn8k$n zSI*IkAY7tqe=XryL%uKW;pJQhD-3s;KCQg2(Nx1>o_b0_BYGrRGg1AP5tv;VDqM}_ z)-1olO7QNCHwsorTm-V|$qw=H^DW{`#P7n=KHbVrXL2W38g8`??;U{7L%Me4)Awg;0NJjxQ9w0^0ltr=jp<+ z9^eayYyKg_y#zGQcj(OY$YQ7eMi~Wlf0ciV&Rh@V zi**eJ@4iXHnDR$-|A6i`UXdvQ%(M>18?1TCJcny7>#Merva$;as-%nY57b>s#~cx$bWIVqICn z$H`HrQ0FeYIqh!B)nwdUo}q1DTfm)UL+-w!TpXoE^Y~HfogAl!F9sns!<$Sb?Rm|v zOYlx*iR3tcKO=cP(LtNH<_P#Kt90=M8{CEr-}U+=tG4MmEwnaY+?3#L<>S5jQP{gI zM*6Ln&60p3d-OHdv=!2GmK2SN91Ug3ir88W;p4t^_fb35*CTokoruj)J-235`tgZx z_)w)IQ(KCR5(i%WFquRx8{aawe3P-}y2NN`<#R=4g{<@M8#{2yxy<@_v}gKxWj*4w zFW;257FQX7a!%jI(b(4pnYXWYIgwn96smPP(CIz=HAm^~quR^|ps{_@CGII>2^}~~0 zQ4hMD0`cnD0T$O}|B&1D6P?@GdG%!qio}wOL<0oCY`NBQNnB~-*fCDitMT(|QS)ms zKV?>>a}qN99su!hASA)VkdZB^@u#iU|4a&Z5^c*pHl0qV)iOMMXYIH#n(9T{;{)=c zJoLyi(bn=ItyJ$QURo0N^0MQrmd96mU@6T4TV!{XKCc8wUY_b&d)SpJCgL7k&Ar-0 zxv&sGXRFG! z0>TubRCj<34GCXiC=JkEKK0P}dH7Ik&dmo)70eR5c|}B8LFj!r_>B6;!p}RU>Hu>$ zgMW)ft{31GWsTfts8RP*w7Y#-X=^c=BOsI|0RD>Mf6Do;TimMUdZ_5gfUHGCPTL(R z6KM%4%O; zQv$b(P?E=AK8%ZxkL{9tI%6tFogAbyxf(m~2eSBW-~{-Re_L>9uz5{ey2Gdf_v)Ri zv6)V+EBv;Y>)4JLt=6dplZeKXKEGih^;-2}qpn$_yJ)>PX}?7y-kb75#w|sj#9q;l zU)5_*(Q6k?*5hf18i%$0fW3a6dEtmiOKa~R^h7W$teXlPrN`~ggKlYq%jsvzHwOsoyLZ{89~W*f24e+s zkyG+T3{awdZ;9u@OC}DMUC*;!@!bK1wZNNvO}Nc=I6I%SjpWq3zLocFXPF$w@@(hT zynFR#`Zd=O$@84A)rJNyLV0HcBuI=!qvy1^JrqMy_WV)xzB#0w(}{zXg+H zGGK&?K8fIjSYw_zXuH))H^MHQ!f2LnVPo_loiTobYpmNDlCRRaU;KMPgb=TmI>E_y zG@=Y8R}AsD$VuL0gA5hZ*0**FQi&0EjHNCpKEt;h+B{hE97uQT_M|}c@=^D0KXhU{ zrbeo71$e`d$tivCNg$NL&CNC&#uLqYS(b4!_Lt8v6hArCS-UXVk)>JlAyO~&xRb%T znv46>bAd?|lxQ@Eh!*4*F-nVMgi;z|d>fI~4Bx=aI3{ef z$BPrN1?)Q^oY8q9+!BCInYlmCb~rY!?ELm5t)gKLfdV|NOZ;F&DsDB1Gbk|J`bLX- z%7av^4VvBK{t=A!&hg_;R{T#9uWQ=*!bu%}{1dI=;q8-f2zM*$; z$h<=}BU;tn2`#g^+%zxiluQmhv5-ePCu*!hyRG~T8!m0P{ojO|o0NrHN0&tm8xj5e z>wKm=s~fnw>d&;acGaRM;GqEAF#7ZMGI42(L>ZIQ1SndN1XKpF}#0j46?OX5VwQi*tk6UVjlMZ|W_ley`e_lOfSA7mH zltn)euLy*lw>7!x%^#hr{nYMg^zR-D-Wy+@K&h$8l(_I7H|!o_Ub^l?$PV~^Oe)r$ zO{&nH4G>aeyN!#$Zd?ST*yK$6LR|r0p4I1t$nmMr&RpY%fAoO&?s|#Ni&=5s2v(1B zxkbp3iCeyjJGm_+A}GYhe8&&76O@5&b#?#lhqPliBin9B7GlQTQ*+EKm2x55+N?B? zdV`+hT!L?|h2~2vEdWSw+c@Xu?jn<5wC3H!+#H(b=W|%3#(t702@g0=^1L&$qn-27 zWmMqh@?2gF z#^Dp&1rE~=9@!Wt80oK9W*V<3A!#~28#yj^CaLRc)qKWUAGv7~WIpJz|2^p^BfFbx zMPH5Ho%)<-Znm|a9B<9U;b@?etS`cCJVK9-jVwX(9w=vyT8rD}!=1_mS;Ni{BcRWj zCDVeZI#W$Q)j;PeMD?r4G#V0~&9k$v;tsQP#Hm)KiP$Ha?!MT8|8|(|)pI?P^ou#K=5rne?LpVz@xq7p$W&!c8e}$82@p*+n&!=z zt$LbZ$5IS^vR!}!9ft|rP$xx()z&mPVf!l|NOZ#8SR1dFmxmlQd}@9ZQHQ`Dyhb}X zV87Kj_=~ip`2X#V`R^VGY}^CM907r<{@<>7-s_eTOKY9u^}yIu0@e&E-`5OJ3X1Xg zUEPdM9oe#R8v;bK+0ydjtCVwZaO5#4*$t$mO;SNO()S1nzTgAxIq<(V-v6aHrZP?Q zkljc$ke@HlnDOyd|NFpxvtb9}P!Gpje{Kp)y>rmO^^J1Mv^qlqf`W!1d%J>8Hyi-J z@zJ(!0x4?+p&veloc$nkMYK77MBBD%Xe3TH_Dv}1V{H^eliZU5&>6X$H*`8DlO zTNRLTei=x124DR1?~!$*xOWSmevdDFrtrwLinM4At~}j%xLbgnX^jmHJLBxqG|ztU z1|`abptC-W`$64U;F22){F;4wzOcLDo%HK(jmaX}4@ak3b05RY5D)vSU&idl)zVaP zz)^e+@g}3YMagGKhAlcR!uGPyr0KGgb#*4~z{o>?pHx=K`S(g8S>-5j(RQMp6a76x z=LmMmH?ZNuv`8o|d6l+gd9Mm|;Bw)qW&%9CMJZid;Iq)h55d&_RLLsZOY7nIceye; z&du7azy=da+hBSlU7k{{%pdwvY#}PxpQJs41kA>b<*})rs26vouq<1T|A`#;p3uq{ zP^RGeq(r~b6mA$*&yCfha821Z!h02;=DkMnA#(os;r8$ABDw$rMd8D`gSKZyZbolQ z1iMK0{?uL;QIhUz2!158zatmNy&$!5APeolBR*Mo9dYy8cWvsN?*hy?_C6vk;LeFd zP2I|eGoHrAI~+uT1tN@WJqmQs-nb}sbZeNzb!Fx1d&T8jX_B9W_g0mJ*mjO z8v*W%1uA)#Dr}a2RJn{U-;9?~lvmDrv+SIy@y9tcr;TMngmxK$n>Q|F%m3?bVa6u< z=Hm2%S84XF(mWd>p7#l7an@9>ya;aSJA+JXtNNS1vvwz@%R*ygj%C}?o@>IRjk|hP zbTvH?vULsIR2;l6Qu#@CpSMtR_j_Aqs%#wirN$LA4PZ*vVeL7u3&tH3^j> zCZXW?dS+e3vV91yH{_rwNxxVYp4)|v7$fh0<3@m>=>^Q#I7pPh-SYSyj8FiKJCp+7 zLkb|tIk0_^!TKu|%x1)$CN!6U-=m=oM!U z+ussoKCu1efBzoy@4Fd*9XXD1MtBo=Y8|G|P9(-1(L>4*gz^F_S2(UO-(tiU#)IFk!0wm@{$+I$@j| zkpb$(XU9=5kc_y?RBopYC?OqmQG{Tv`XeYcC#c=jUrxqL0!a0U_aZ>@3qkzCknB&wf4`P~j zdW52UA@YB%U4zM9;+$gWF)sI!c> zG&5)J>a)^H9!k6D=}&Q!Z&W~Eop83sbytDgLU$F&jEQ7mau8ny*sB>L0zu|@BdH54 zyZi3rJWGrmJY@=3N?bsNNi{@#Dr@ z;KA5VKMh2P@F^wFd=MRUkV=7SvibhiZvl*!m-Ju7#l{wnT^s&l3LPH#BfzACm?tl# zso!Cu58cF;H~(cFxARO*w=NQ-a6?R= zaGRysk+pFco9WO9PT0GO(+F)NyB@I8XrF+?G|=U459f)z7`z~FRH)7jS6GjnwggXv37Ny&L+-8cn9iT{2VpVW!Y{@*r1hKm&Q zJd2Uz>+|1Zj!FIU=%YM0eyZP@=e<`?_DFM9R#qL7hSN49)epy(YV4H|dV)vOHY2-$ zUc|wQbPW6~d3Ur6=`t!!6crV-A+q*>8mBDMX$;6!0B?h*K&i~|1JJ75Fo;>4FmJ&D zR(>4cT|YhAj{mYp?u#jD$N~g&M58<#-8wSdMCl9{#ZI!uj7pxV^+S$(x&Oz0BXs$l zn!3|N7|hoL8q^=`Y)N~JLEy7G2Nq|Ra4X7I0q{>dJ%l8hfRyJ$A9`i(4pwF=P1oDp z#$0m`V+(4q4)97P!$j7vOCaCM2dkwD7=>N%dala%upP{h4Z`&miCGj?;?ZPeTvvv% zgJF6?LIG?bUj{(lN}PMyExuM&c2amyU4lolT6ub)LV0|kqD5V!i^{8^5M^s$#p&Dy zlD^#m-NUf%bHOa}D^dy+gFJvKCHjf)n-eNKCAq{ErghM+#Oq$;gi%IwQ}>-v&VgO1 z8|eyb>+0?elThT{u+=9YMK;$YX#KaPh~GF50HF=^6y`ZW7sHhf0Y-p*AQn)UKZ&8N zLO<77ziEDjMV6|hQ zR|cZTNV^k_F)*|{tE9nQ1&{Wk&`83w1X_KZuzOR%i&rlo`<8a4bOqwlA$@{^exvS- z-_`Y5xz`I4?{^0WXQ`0@4GQci&X&|pXtsbZqyjWfL})kkY0VVM zdPn>#{*%~T4ru0(y3dPe1X{^#F|nohG5liEh2pTYi&LtfGkDpB>$m4urJwIrzL3E5 zQ9_e{qGADX`Jzt53Nn?-L&qKknXG=4@W#1;m;(^o^3WA9(b&!_2B*Sko;&=0fC2YR}@yM+wP)K#|3s!zbBcOHH_Cu-eM zhebZiEQ+)=g>@e-oe7Y7v#w*-=ytYV+_rv4NPns!ck$4dei{%F$b3zdVIT%j9fnaC z5UjQ^*3HFP?yx~uBW*)VT_StCP<^^`zsT*PwS3e3t9eZh*DKB(UBx6G002^4(Xrrf zrHnT702frBhA#OUE`9kz_c1>rJf!DmvQG~8AN-NuTLO%cRS4+ts<3Gw(tTEoM67u5 zJ=X(1H3b~UX;N`8lWk>3E0f60Q^Ih@yWGM|z&b}`KC&6v=x5F_sB7l!xm?UE{x z<4{zH>g0Gm0Kql|co&)0l6il>(ZENLMWM+H2*I-dcnbS;4t!<1ye0Y%u!tf{c{gBK zX&5V(ki+xzz(D3-NK!r$br^V@k(X#LTXFip>~{N-=b(O>kRW9|**E9%g013D=53cZ z<-#yvsUMjzQU20c;1}m5@GQ#!Uu3DNoYtd4QZBQ1+;>mC5J<~_->Dmvuv1`DY(EJ< zv%CN~P$g~yROLfcfHBhVXz13$!$fhDwWP@L+~j!j>tRAqDKY2R&n=JCpz(o2!qy;j zQ3V3&aFHFf@Yo_+gajf(A=UlW;Kw-TWf33d_yWx;UmMppbP6PbbAIv^m~2DWT zpxSBOr*$DRMzgKxBWyf2fJA2@;cKN2=Omr+#B=dGUw3nyzI~-B3rS+ZyM_$Lx6n=f)@ zTNnl=+G{e%4U*+)QnjfK2b)>4NN3`^PNY#$u)@v~eM`&n$BzpqhIp zp({Xa6eY!&$YZpkPx{ix=t%d|iz}9&BSP-n-Ji}J;%t$Q7hg%@PHmh9Sm6SuB09qM z;>yp5irc&4<-Qbr6RiA+w&c82o)b^glE9tmhXrQYNpyQ8*p|t+{&tlR`;X#O&$!_xJzQa00=nv$^ig{739WxbJMfL{B51ECT0*_8gw=)ZFhN~qlr52N zl4{>d{Iby)!n=X#t0wJF>#|VserE#;0>+U3oCnMZ`f|dNVcY}D8-gctE}qV%yR{U5 z^|O9@0mzF_FN0YPCEU3`Q`S3b{Q?;NMmHIOZ9E{=l_f^UzHL$4Md4&itm1?l=J7&U zom-kV1H(hzFZ|6Up(Jrk-!`K0A{H8Jdli*a@l`8Fs2F7Kh8aFZ|!y(O`e zxZiK90q`y}5p&TKOTt}NnJ@|0>low3uv#42{9II44~ib*eNtx(cqW6EYN%rwc4!=! z+Hcc40c`;^d;cOZl=zoM8y~px8_q4gy(=)#1eN>}zQ2$1{togFyA#jZ(#}=vc;ur* zq!;A#b~cdQ|8p2Q(CN#Xd>NlXg3Gc9^>bW6d`>$hoSy7|u#7`Ft|E7Lnfp2|{|CzR z%0{GnHo%sa`OPH0D~Xv5MXj|~=acZZsvhyw+K8wtc?Vi;xDdv3VOH~+)sr$wfadW;Szr~DTX zT$W^1CrUmRb2I-0EJ(Hge|OIie?U!P_dI(`Vv$|Cx{VncdQ71t6sajaM@{`=OSz``hSUj^0;iZp_fT+=EK{KH5Tf2z0|8Xy-tolvh%7EFp+LHi(w%L% zI~53eWa!WhFWp}8XIiwf7$QIuy^yDc;I-z15%~EFoPr3i zm9}hShFJ>MsJjC4QBV_ypf?nHNo<}Ta(o6ut9a&*ppG_3fkxD2_}fj5!J549GM~W9 zsDjC>4{5&$hBlW4qtN#KVo>+pzj$xlOmm)}p=3h7SnAE3ehJKNq?%cVKt_Q{XxT8k zr5g-Jzj47j&O&V+48x+zdPjMG>{lABj~4YOu-w?-**)%m*lIx6?6?=gw0UU0Y0t~U zD)}?!g4;tP{35I#`u?y5rU2^XFAI5_YF_%MaI8SX&%#7wpDM{R22}hT*q3Xl;Qo zavNok%v|TYl+S+KoD3ehQ7>T;0(7vGmWr{U&5lR_Y|X8?Cvx1$S}V6!`3KI;*{;Ee zz9wU7ZmjrMpJ)RaQii@97`Q>u<);X`8u=c16)5DPGskjzPb%1xcEJo6g@k+P1gerW05IV@ zLd<&??@omakD;1#Vx#6v1|>GYpS-0IsaagYp(4z#MNjC6EbV*wGan0x_h^_L|I0R| z-uz5so5TlQ=3S?qGh6*m%1g9q(f4Cp|~(8bZkyV1q3F; z3o|z6JG(x&B-oM%)3( z(fz=EaAgi>?8XTk7tFE*4YPSH2+6X!xF7OvDH^qZDAwo~#&{e!#^&_ck6TU;q zA>&uV!|G~U_fLd9_-i~PJ%V#Q>z-li?7H#!MEBNDD6wN9`xtgF^u)XqcW|^f^8($F zlu0=k(!dmU!au|3nkq)wq?m78XNs&g^PSJG|rcHDZD`k=d^o7K)+`3WPtrFO)RhsCDjre0eXVtTuL~ zOL6YHa6thj2ycw_jVC*2uhm}}tiGm_JLmJm_UWRnU3InmA(WoN6VE!}fi_MXSc(gv z_UQqWD-#$mkOk8l5MByfV7>Rg6VOrl6q?tgkJapepIL{V_3mBS!97feSFI4u&?8KA zwz3L--&D)qk=U>X7y^8FYd00ZsHo7h4pDtDAbQ~wlW-RV!VxYm>WgBiJEvgg<8xUvY8Dty1HDgQn@#M# zcN*T@>IcxDreEk7%Txz#U;Td;rmVqC=6US(Kw0QF>YW1>_g+Jw7gc?J%ua!zgEFFG z*0oSaiF9!*7R^`moiwFdnI3}zujXdoEhs)Zzyxuz2y}mGY9r8WWKvyym0i3<{n~sc z`o&Z)Ol-M2y@@RWgd;Mv2p)MS6`~jjdfLtEZnBD!@YH+7-@796ZsrHg6X;b@n>x&? zx?5Xaor!>x!H*B8&l{R3d0`aa)SdT-LWLuwv z@7_*Iotduh_LG*B>x*0&q+uSQxkk0=m=+iF2!(F1Z{jevYOSNZxz$Q2y`<}Jw7 zbZpC1@MU6N?ws$0vDlddL^f$x01x%`^w^7$ZU}Q*H3nk5OH2%lJFG zBLqNnQU5O)hB}zQYqhi309JDNy%u?dQe+V~f^_v&-2!;V;?^A}i@`Q`3gn~rWD2JH ziphu^0B|P<1XEkEEv>C4g22K^$7b4B$QG}4H&VPC&9*WOgmk1EFWz*0229uq0AH^L z`S}^M|5<)(;**J(4TIwV<}{og-ho^P;njIaf;R_)i9P7?2Kk@Du&lu?fyCe6p5j33 ziU<5%4{bv*&PUv3_Pb6kl#Hs0cgDW=03PZ1G5@N7#J@5GikZqz?5CnUTtL$qI^Hs? z&NLiqdPc$$pYyZV^kG7b=Z7t5L*8^vy7t)RBo`t5^r!=x+p8skM`%VnQA@`ME z3ouu-8znXXwT0E*Iy!OC`Bw!SYJeXhlq$i<7Au%m$%XKIk1Gs=bXAX~K%O}T)szFD z;+4R2@ZTTh1G_7$Oe--y1*LV*mRTo*B7!=Yw$K$wpNh7nOBjc{s8Jg74?8F)-=K-+ zX=CK350|H4YsrMns0Sc8Gl zJ*j8%8Mws2$$VglJ5$n_IoqnR^wRQ8r|3Yb=|q+A9)sa&Hj`bM(Qfb?bwtY0UAPY_wYxE5_5^ za3JF;$Ba@sOruDcj{88bm7-$|GayL0@Y*2>J`B~|E`_Q{47d^k>X-u>!$za~-x$!a zKjbg~rU0@WFuws!n~4~KD3qWIp~kZDRVX+wOl=G4FEn*SDog9`1U&Hlkwaa1xKK0# z@39bIW)f^3WT1PdAY$wa(attgDjtsuRmh+z{ZxPPEQwCw1NI9W*#Nrjgzfb?H^D0f zb&epoO;BdrleP1~)k)mQG}Zn;IORtaJk-5e0N!y-PQARz#-NlA#uMg;^Wr(P?c@~y zA!1a=E0S$9tlIezVtt)#(OZy7-g)W2oLnXoylOM1k&OZNsM@)Gh(M3b3OVmwh+X`s zNiGl*vG~J<<$7nDqB-4@oMTbv?$bs(^&%H z1_st6y1b>Tb*fkwST=3&ra<)X-%St5eUBH4y4P%hkGjGJ%K-`}(ou#%BmqN``%lhD zNQqvhn;$g=8Z9c*TM1Wj^}49sk{*e!>;Iqf!3g3R&itFFYIM@Ho?cOPZPaG8XEweTL`JSMV;r5ZhvK6b-OKD4Sj&|BaIZ0s=!> zXnYg0kRU0msN}=Wi|{`nJlqz*jjapBtHzLMdXv$2I`)5Av-$(w*>4W!VMU}Yj0S}2 zr&@*yx}Zlv0*L$2%U}yzbUJ#ENjuprfQ`+B_vi&DDylGnv0HmVlxqXgssrXPNC6(l zEwqe~N^5&pAyLNcY1m50-q|P0Yr(;+M;`%W^@ivYoNCJt@dm zBCIsM&#_|-5}bhT(EgikL?Uzp>}`jH(_05LXST;|c_u1O3ndEsJ&H$y^le}WI4b{& z+FtvehxY&JB{WeIb)Fd6IM$u6LOurHjFL|ng87F5l3EAK8AUY03wcZX!s%wSx?R&e zj-Ya*ft{<1Hm1lSMSvT#xd@!$4Ab*~x$FU(*%fRbfHIaM<_$T}u-vaii<;mzL7_c| z1aK(_s;9xb7KQH~7$CCGp&Q;8G9vIMVP;TD`HD#6`Bq?sO##`=IsD-FKM1*#9?3*d z;P41QflpgciU?F)r0vp^l3Tl+;%BULtzRdxOLs@T;+h5j0)40xR$BqGwgF5z2fP8w zx9m7@YkjHFodSLr^0(yCZWei)mu*u4i~}XZJ8p-@E}QIUkrg~E#-#`_TMD3W(1GON zUF>YPo|TDuKlw|0OU^I?sp+d0+U5mEYwce0o`huG*(2)RSALc2Ml zVclnXS4_$i7eh+`C!<_P%ivja*%E7QK`6}Dij`4I&$g)KZhE_=1=!4t+8hXNSYh3) zW5AeNf|FgB3mO*ru@sOkxm-a=SPs;7;;^pB69mjQa`oThub zp*bTWp;J+LO(NUO$jGS6w8W(!cxhS-A>$Nhe2p-?xcUf!guTb1(Gz74m$y*%IQ8oTmG^Eq4F#@;xhnfvD8pu#*|EP3DHVT5U!62KEd>gn5#T=AnVN?%( z!D4~~15LqPEDP3|Zm^Ky?Vb+gzNmM{2YR@~ag54KId{j(_#*I>u5Wr)adB~aK+kv) z60a&u{ixTNR8gTJFsyQTAe^T7 zI?dlaO13gaL*K|~oMxqi06!IlGY6QscFKS>*w(6y?F7SuAMf)!jfjO@4ptwRfIiiQ zv3gz|@a`p}3WzifIs?Qlc+BFE#D+RDv(UV2?quU)nGvnAphc#boVe9FH>WKCN$76- z`T40G$-u)1qT(vhGz$XKicn)^RJnMZeQtXs`zR<30y%n#AizL2vK;5`C6rw?bs&`U zk?tf^!D^ZeDug$!Zva^Yx?{Z)#c!2k7WD+r&Jof0hVmhPYv?M!Z3O#Zz|Fh+Q!|sF zsATObT^E6_qBLDY;`doke1|b{uPeRkhyRDJw~op>{no}GL~#r-P!t5kAe53$5gm(0 zEE<)Tl#~z{h0i{F<=|)WAA(USye{~?L7+H2uxZWTsC zJ@#tDl{3;$C3E)g#7v9#0#i`MCmd*p$E5Ay+L1B6mAe#*;6rPS_Tm3=Pu_{WUZ+@bE@xqF%)wtdVdn$TID}jMt=xrmC!oWD-Kly@Lgt83Dj>A15 zcJsGAFf6j;h5k_EUr*`oubg{2S{oJ~OYEsSM#^iy>MrI@3Oka{A8fOdz*gDfnX%6H zDEHrHE9RN=sNt_DsV6tXIrj+ zadN|4kwvSJBS=aPqIH?=fGn!F$-^3ZoK4%m%($3qz2z*U$lXoZ<&9?*;FBS4XosYk z?U$v$$K2LIqm#sU&GV&T5phNkDjXGWJI#M{>5m#{t)RQW!SFmDLanKBtYs-*AY)7# zu8T*L#QJnS-DrW|_4;DkY-cjmgRd{jXA`;g2Lh%Ves)}=mQH%Av3Z1iu57u3=3V!4 z*pb;5BqtnM$-`n7JGs8};2U@Vie83X`eF%ohestJzxwJ6m8GLKKvTV!S1b!6X{XCL zVPq>#fkJo0h_YSpzu<#mf?VI#O~te|=8W}{@fs`b^vh7v$6)FqnN|;5&ZT-RFQo_K z!-{DUvuHJ|^_YJBXY#bode8rf2#CL+i4h{*t$AR~MCR)YuLlzD{-!@4!{4!TC(9jN z-2?b5g*ej&aV7)BIxjD|e7@>B)H{@OWU4!`C90%Q95eqonfYxlQ_deEHNI6Is*bB5 zXEV^Q+|jYZg|$Q%f;ai{Mq9_C{ENqG2YKCpem}8;#^{9Y%IfmXn~0mj_0&xh)smx< z+`lfq)+*5T_+8>iZCzl*Gj}jPF0KVFL9d=KtE7;H04XljM*^xz6GORr`jRLrTxx&+ z{78M0NBe6=59o#>W1QEHwN}PQChNvtdtbShV)Rb5S;v5c8v`P>gHm-n``fsCR_x&M8>~FS&lBox@bJ{ z_ORxsr7X>0_SZxK8zJN^*@veLc;vb-6mG59Rc*SK&3Qr#*rtq#eDxxbAfLMcbvn-T5 z*U9Zwh6E_xBPy6n4A3lJjV8Ahc`I0)&Oc))0aQ-RaYUmKQAQAx)@Y|NU!#9pC0$lJL32GYvf&xC3^59Yp-d&y{GGZbx(UN+YxaJ zyB3T-VoqZ^%^OtcIkdB{8KJ#1SxY|n2^%NNSp)e(SG)cVrOsr{D1p;%Cy?Nu16?=` z=Pv_5CKG%Mv0HY?M0>c{s{6C$>T^j?eX$DGi$B3{f;kmY$}5v55?>^>-^xlnv4#Y2 z6hg@?G}0O*gSYZ8@>aH=eAtoxO1Chl6N}wB(#>#mUY2vt#1?qz=`VjnTf35;zx)BV_nWS?@gq^Y%p_?gwFDa3= zgP8y?KNB!dyha<_QWb@X(y=tuW9663`9_g~K-`|`TzTbBa9^CMKCOFcNYfn}o zfu8{x%bPOO#1x&Sm0#1{pfL&PDEr;*wT06G88;Uk#Ygcj(}^v-aR_(8;)xyk$Kw|p zetoA@=JfR$&c*CF44BxIVg(q?u}RaUm28=)AzKbdfmG%Eo-qrvI3yLd^&xihLJt#|YX*6P267WIGJ%8#solTmK|`FOJWEFh`IaB=G~uj}|S;+Z_p{n)yu+hzLu6@Zi_yTsYQ zkX`Ls=T*=P5v5m%j;(z9hLt&@dmt@?*d#$0fa)4}b?6YM=$XbRZlYRbFW@fxXRWF`UVrXK1<8i07CiN{#+PSr2Lhjn zX?b*hgId>b(q|UCW6D~;y8A?NuZUwQIE@h$iWS}S@isv31V_?e{4erz;fw0{k(5u*vbiPk4I{XOwmdtF~& zdGcBwK8(%sPsgq$&D3I`O(rT21mxV&CBD1U%`bhK#NW>#7x+Xkq>EJ^UXEuP4<(8HdQd#_Q-Qv4)B1?=2spzt0tY zrA>szo{x5W0ux(U^x!%2?c+nn&izN9^s*`7>|)ndy{nl_7C`TU2CG;54~6cq%m-Po z+Dys91C{Jcl3T54fQZUf1sODA(Ys-2D`S2{Bp}gRLTS2b@TzrMX#C>j#q^&i- z$7YgA6YvGIi>KeV!9}qlT(9_A_=-x(a6D(D)g}tZ8Je`UyKk^3kOndK4AlcR*wm6= z$iv9<4CRuPZ1s`&_LnPU;5w>BYEZw7dEEqSpRw!qK^w9%vIcbK_Ga{|IDZEY%r_h; z{;MfX-x?Fyl<%$>rn_Xk-X2NB4$uhnCm_AA9C2Fb_T5e87Af&UG@jevTfxrCh3wR? z)vyq_Dbtx_LYJ5}hI(B%{Y&p++Qm~Vx4Znw*SomC&){7bqzB@L9(DPWqF3Ce?~8do zXO|NRDnB;dKQK_>@D{1Ub56L!lkw6>N8B8%d@QRXfJ+@$bZ;^02i4arsG<+HfcHBG?aig6vx=VqZC!f$1L3O| z(rR|{K@5G+lU^8t)GiruV=Jq| zggb{pHsqGN;L&q>_l&SZ2(~oidS{R<8>)a_Q43g! z6to)0>T#(KyPQEw(QMSSPDih|#yotWKiB4;;}~7+<_=;wx{y3KQO9tq?&os4)1IHo zL7lW^s`a{BS}|&~U9S6A9m?2p)~1JPJJ*L|y}4P3eK8U?|4@v0*xCQ`!iaQ)x6qi3M%UqspI?gu}WE z7v1P0`O-L|?KQUji5gzSXQFuda!mc49940*%#FiTI~<8^_hf=%8s24Qr@C61Rcyia zR}dechz-V0)X?i2GFErHIK1Ry+|7IjBA-W+#XX&1eC?pWc70yb1h%ZY_+~;lRx?C^ z8GfX9#wh{U>cA2%Ier`IybwCQK!yuXcm0+u#9{;11l^(=XQk}FUskwCO#hm}%|lYG zr;N3_GLGG*GhV_O2_0{)NWGIVN@V5zznmmDtTty?Y&YS*`|8475$@z$xriJM-Kd!w~o?=g+vP(JxcmK-DaLH99V zqa}zb7wQrDm9SRE#j_r%Lm^bNO|OvXp+-<$7jB>!bwfWZb}}Eedp06#|M`kb{~LDa z?`#JA<1O$97T&k6D4~10t)G3Kp`-CHjq9H~55>a*CHxgYqoxlD? z>n2+(?Hu;e3h=9>)kHm3J{gjr0#zP&_y_IJy<_L7%oJttb1taeo^oRi|S6AN+| z8%-=m@}4aoXm7``kmjC1V?nF#65VhF0tnv3f)LJF*lg^K+Xv?yL>`#D?ZSoO>#Rqw z*7`kfuGQ$Sr0bFP?(vav7a(p`d3VE|U(rYuk7ZbON)2@u<}wQl7S?7T`Qtv+^7wat zLv5n*J2Sr_+$DjK-xvq#HL|zt_m%OV$o;O|PVg}|eNhRM+6$av#$l4Kz9Ju#?%uy| zVmLV@QvJv-$4&1j?JRY;#f0v* zOe6f%P<^8B^mw=aJ?LclLx(@vR%_I;M`lt~)YSZ_NY$=aiC$wlC~qd1?vK>iRO_U# zT#>TADz((9@54+?d?qhft(p{XdQE39`>ThRlwMu_dZ_5k086={Ph?1200bXVd+C_| z>D!-j+Chd^Wub+YRH|kDa*%tpuLwnk@%vYXDwY!sGmjrXw)-V*_u`m*^^N+=;qJb^ zY|_SFO22;rqZXu^+NfqWiM&j_rPS%D<#YD70UTc<-L+L=PC%v|el5%d>vAUXF{SCTEk-+B?kj($q-ju9b3E;Q^NL+#!2$-72 zWW$rwij1b0l{4G3LYjz=Y$=V)T)~yqtaX+0R5NW9>CE3WK{tP+{FcF>Qd2}%`)e&h zjZ>ROdHIHyU3E~#WwVreK^gYrMi%2(Bt1tyBAHpGw2eb%g7jUUzrB>E#~FKqmR5_w z8sj5p=9$cMKd`hjJvqfq$FcQf&IrrG#H-16e(CUaebD3Ovm`3&5BF7gC<>VmcxR4; zZSYv!n=WjuU^U;AytlE-j<+mP`D!Je|aH7bfI#|IwvU3 zK8G(W_l;$^&-TRLheYToWL9^ezNU4OWwR?XER^ZugDoRvoHK?NrR~3~&4Vvcz8oKu zW}vO#{(DfARw3=7VJe;BuKfa2Q^lBQ2lS{%uQ|qd;eu}zLSE*%D52amHda<%2(2wY z52|;s?G*dXrUQ8}LHP${_wGH0JLjz3%`$g3hr+7pqun}#Ut=ONEknXGEe&`Kl5T0V z7mef?xy#bB$gm>ywe`L-R@PJJwAxMmlrx9yElMGaiWqWvqA+N7Psg#nu_-|JsG?NV zWve-S6*1rM3|z13c<+BM`>~8s>A}$}amCxSo5OY&4fRxFE05^VAD69(2Y&;_5Iw?X z$MSOr5FOToNoJED#@qWIsxnEg?&j6gv$dS#6tYtC@Uo_;Z9IA6M5tXZxh+@U7tuZ; z@W%XNWN<7`)g)h=pdGlI3?Om=>tg>kM@62yHt>k=u(8KhOKP0$XR+FgDrxMaJ+drs zS_+-g4w}5Y6;o6<(K}P{9u&HX*R|HK8%5}=P@*>tIEX4LNv|8^R0|4eKKNI46jf#^ z%%^(bnr5|5{mj>xP)p~|sGJyMtM9*_gbAE5V?J|}zuM~@i?=$Yw;I7S_3A+iwEb(? zRMq<(76xXETl4J_H!=%-lxb3uxzv8cW-?`ZMMnYx*tvza{@kcZL-e=<8;yx%wPJme^^xU9eKM|cpG+CDeu zUr3k3=yCLO8O8J48CRog%x9`)>NiYS7taJsI@KpjRCCr%Jj?2fFmB%_R{i;_-D&zd z(+U~3;mXg!@h^@^X)`Gs7dnlL&9NNjHkHovWjOh~|A;n@$zYCFhFaQQo$H^rbG|;v zrKNKIXqwi=>rfU92;anFv4TGi>yMWy*NS6@lX3@gBoMdCa*0nF~ zUW8fJm_EFyO&5|f^1^-7vy|wr^=(hJXh(B47IjooU1Qsoq&Hi%ym@C4T{u=i9s0@r zTl`1pjqvR+sc5AXf&T2@gxOAZ=@8JZ#fgLEG)4d$s*b%#X}G_e*UT6c5jUH^x_W=v zCl9KC$7ry9;p1IQk3)1koQ3n#O*Tx-T-iuZ;g)Z;L13i!!IMcw9rinEQuJQLw@xQ$ zUJsXi5v02Qb@Jor=;(F7M{yf_tA+BKki+$xPf)YuS6QHKex6T3U7eQ2S}G z$v6CZB{iy@AEIyQBp*ywv^E~ll|8Zk0}(2WxlVt_hbA=;LZ|AmnBIYq!~074@_esf zaa?fF*r@FoBCXH$mcMvfJz(BUH7&xB-{_^LJ_~!j11JuqQ=UrH3t|>6hgH+ec3D@; z)?0xs9k9;1$z{=OzJvv#c^k1`rczYg-a@PqV3SU61wKHw_XJK4;7A;`_lOTGY71i* zuJ&#)Lah0IeYc6btqG9G)b#2T#mPrz8<(rG{fcD9UuEO+Gl9!isx)QxL|wrWo>s&5 zrCNDs!;a%&#aBcA{+Xheeb{;Vd5NugbQ9_SSDPTY_B%08c{>riZKz6cc!eSLQGL}` z?VdvdB8&>wsl3>py1Luqo$xMtmG2UIIoH#L)~+JPtbmFaJ9@H(O^>xZ`#YJw{MGm> zY&atAvBkb;Q6Yn0n;p3LI9%YIikPUJZYxFS{WNhFG@x+wk>VV`5YHz*Y%kMZi`ZPy zNITbJ(JzHn+!i_U#&@qVe+HFXi2;0ST8Aa8-ZA{jtL|Rdazkw&ODR2IV17Z!)JmOe zIjwMcnSXX|PdB)~PR9dXH|OoM%^XMV^{-=aDF=k593zlxw0r7z7g`|k*c0)_ve!-~ zAS&_Derh9eW>lG+oTK?Ie(j53;&m-d2{Pd{iEA%tZIyH78d~Qa^~Ov*<~llOF6kKfa4s*<6j1iG=f)%ygx zWCp#d?}_kv9cS=qXcTF0erYAY4>cRO4((J%Ga&LzOtdH7XhuXL_QMX-3OV}2-1JN> zYkL|XGrGL$FD-_1JjtdB0l}=;7JbhR5wpf&xK@R!<^#EG>rc_LXsuTh&gE1J^&9L7 z4nEd6B1#`QfL87CH;45B#t9%LlsQ$SEUd)Z(ls-`PYUhUo*V1r;OFPx-(tMXB4rgd zIX5RBZ#q*r2j^i#MCQiE#V^B07v{#r)1HT!ot)`1HIATL6k&h;_lp+aH80+7aLKR-AOB%Jc-03~s)hl^U`On$aW09~sE*A=a9?X|P+*g(_|b!7GTc@e9QJhwFR zG;2l7ouZayHdRzi?O;m~PrV>dF7fRB>l0a-aiy4^@(wPkpA69AR}nVhWqQ03^Ld~} z{f1fCN`(epfmEdCUry5$Ms?Cljnc2(GBa;MBV!_BP~BPtqfJ@k7Zk*fa#F4C+QeNyM|jBdc=tIPa+?Lrd$(xBa2}vWaw8B4Nat)7gkYCLN3JD z#*Yexs8kOR8%F6P@~vXSrQxJ^>^EiU#ahq5Ixmm*S72(sp-Q-VprPt~icnnp`P--R z<8KL%ZvOM_u|W7>Lk?EDKam0WI3RtwKHtuSq=G89eg=V30U}a)^Ba(ooUv6UcZ~Fw zm@KH%m1PySae{na4^3NGO)6YSg}`JlCXXy)~Ho{OOj{rT^lcFLJuoD1_P`*Gr* zN9*(MD(kf9`5a_wmq@g`jh-S%Kwl36`UpXrG#;`uVyHO&NE)uni5#oJvtbe@#gR|` zH+DYoe#9*+}h<2#%?*^9>(O;4d5z3K(_7R$-%E>{$5lIR7Zz z!%=$YGOdZL)@uH_rZ_yxZcnpm)m*;2tK3)qx}qV>7=l`PV5kw8y^&hfDjO|+_T2s* zhc2aB&|SrZ@add|o4=c&Q!H<{?UySqp3%M=$GcDccCg}R+CbX~L0V$(af0C4zbrzg ztWuA%aGUM^0|!nJ1f!mbqOgk1Is*+k^RfqkQKeQ$GCZ%c5vLtT^u=hs{LwdVOT%;z zAFA4hHs_qZ!0c?vm8AYWvaD$nrAP1Bv17(E;?-U}m-ww0oajrjq%}BqHTP#*mOW|7 z6B84Hn$~UC<;vWtR)1|Lzn9rl@yr=Nj2sc~QetCct7&|Em}72_{dm%K5~I%1!Eamp z>Lqs>Rx^~fHT17?>mPgFCZ9Gt^NW71r@DsEXkxUxKSYt0fL~>br6?UrW#S)}&!=dK z8nB-F_t)14{z2mO$BnWJC?V2S+8KrqhmJS{PvjjX^T|*{Qo!QkRINHdu=kqgAPfeR zbK@gk^iiE7`i>vZ_)x@OeXaaYvwfgKKW*zPikbqkw{G3C<)5KOoOc)3aQjeUwR=~ z<7-E>q}}H$zey?fvWC?;{@}4BF1uNhPDe)8kpFUd%ez*|F~>aPMkzZ4S6DDEHy0NN@&;oXGV{W(?26PD zZWrlE$YSjWreU>P75jV?e=0)&Q)O6_1lz@qCKvK2SdYRp?4Es|vqp1#v)k*NXHbWRNYjhbTM;L67c$#GY_?TjU!T!l`o_-~5zDrQK;EyO zkqoBV2ull5MV9*|JC9#K|5^Etdd_RP6jW&!Unhnnix?iLj<+Ji^v zC`J`{MMl7A)n8xaV{{|l?CQg-pjh4E;jzq1H9t+o(AKtpc!l4LNB+j>C?CGHKoq|) znnF>E$u1stSy??`VV|yEkCNN5V~5Guvu9PnH{M_WW$5i4ov-4!$Ch^w+isQv`|sQP z&pvsbK136ZlLOVXpTwkZaX{9efwTKXxd0V#LCMQ6=G^x-9s>rd>BuaFY16K z?88(ziIJibd~9!4Cy*c0F?7FYCCq{T3B|?7QAk(en?9%O$x^C-n%zP{9=%VtkiqAT%z{_%ko74-hy;a=IM@X#I`BU;ma? zr2f9p;zm*XY-Z#Sh|1Z8T&TgU}4lvuDrxBa)@3wpO$g*`bURYIsH}An5?6>CVEvulob`J z6WIVwxp5u&BW2*j#^~px79R}AOOEn;ojT!icoCq&8GHMzWZ>@|!!&Pv=wteore%3~ zl}J!#Mjr~u0!MsYWgupV$hVG)1v4JE#2D_|90{MMg zVWIIx)oWR;EW$Gert&E;S1PpD0^s_^pYhx(3Kcf-0TcRpwe8FlJnZ_C^rfxe4>JLe z@%Qt)?Bx(nlZ%Z)^XOXofSS}yeQs8gVk^{3(~Ok3ejIC#4tB}mMcz1HviAPh-KJ9f z9U$M8Pelehy*!ub13dP$R*uC`SN3X3jJLuWg{R|8Rzc3@vrqYrXB`vPwU zz8D&B0L|1#+j&}El?OodOC`WEmIb{zc+7dO(<$N#ul>e(BrR}Us(nr)85Cg~K>mEv z#!@p)e^{C-^X&#d*4EZ~Z;FXm%yPR!H$VVe1=a7At~OWoO^8a78`C{=COFNbo%eC` z9-Gjy!Ct=H8B31a{Fc03Oc`j%^6i!uorQygeO|{iE2Or3o|1L=u`Pnhc@kf0#6%O> zdo;BSF}%XQefyNkIwDuuK-iSwJZ0>Rn1-rl@c&e@G)k^@V<|zKIH4uel@61=S<(p0zTHhgn2k-NGKUxklS4Ao*D=&B{!-g6B8@z37f z+Ts+siu2ekGBG*n9~!#z;p4}`9LtfCwtWxsH*MI!^G)=Gd;+t%%UCxep9u6AmJrWj z4d8F324a?cik95S%!Efd>6Wy1lQu4+{CoJ&Jk>w=H`j3xjL4TuH7b(hSAvLi%|+Ln zvm>ozTB_(wvs->0a9&Pqoc!#>!|G*C<1TBv+`};W`vK27e2|kTU;AglD->C4B%afb z?%H~+(0=HB@Tt_5zG3_Jpr?O~W0!N+3`TyfjqN&*Rby2Wcc4NI{`u1;M;5Q3Qjao* z18n=x`!(ls;?SA%hYUBq4#{u64M9UsxL&AM{-~cfi?n{QwdD81X_m3CgqHrH5&G$W z>T3FpbY>dLe1d^p5(GRItEIB8%3D|@OP19i;>1cG!Cr?U2_pWws<^*7t16BK4{3#4 zW+zdsBzPdtfs@K7%=Y;>RrT6t($W~`qoii`qX?^sDXO*6oou?@H-cr$k^~veklGQ# z9$nMQZtl3KJxNVK(0s?u3(`Z4B~b^N=_qo?h^AvJGxI@cwZ$8irh~(x-jZ}&=39*H zTv9Th5b#uGpe?3tj5vm11U$(vFa61M>k?5T#BJb3=Soimw)-Dk`~3C|0qvZsep}qE zf^)c``F^n=?|gk`*HX(bUvHKL?cg5mL1R(F#r?GLPWOKkGnoL8PKgOtSLpVkFK3EsN{kv>rP zVG*QY7@THBFs2VIW4O(fZr%=)!g#~{4d>DJ{>iV?*|vN4N*cJ#vwJwZX`h5^8jU=thpVmP>adwZ3cDIO&!Le zvvccQ%%P=}hE3bQ=G7=vJ>@YSVbNmOaYfI!Dck7%e%ihaF^6(=dfUSM{5G#!zy9ju z{rX^q4Xs!jQsDij_LGzuc@>plBJWVk&GHdWG&($EyvwepRRpg*=H(4dyGNHLuTS>L zGb1=;-5Dd<2aN#|Y&{FuX0%OTzk5~MWr%r=;6vjd+_dv>=|-mAn>YIz5vLIswjKo5 zvp=Jo=2Q;vo0gnv=TE!Vlx#L88oG|cUF;M6Fu>R3WMvL57hA%mzt;J!O&#J65m}y_ z!x!tgy)pk*)LX!8r&S^){;{>N_VDmPLvi|f>S;6gH2cD__S{;Trkcnx2?2heX*W$; zZK+^w6@#X0*{qL0UA1aE)jnXnm|MhauD$1yZDy35=5GdC?g(7Fwe_#nI8UtU_%?Kb z`;VT_Q-2#kC|i7Sc=r)s(YlXA5;QK832Q&XO7s-oM+t8(HMLu}ZY6ZY8@J<}wiL}` z6IMTr?P3Ny7crwuVwph&QqRAAvBIsxd@Ku!gHpo*Ge4DP?T|*6X-H%qdcz38H%g(D z^YR_h9Z2iq>E>%YB!AFn$DLFQebur2SI<_j{hj^~=J9XH85bHFhGyui&|b+83Nf0>AazIXEcchMNpv6J-RHe4j~XB#>8f>4-MqrwMn#p~ z@|~co)L&V2U+t@;*g|Rsp!*PCJK&N=H9Zw^Qy_2U)O0-!NEfmi&}zN;-TTAz&oAuE zl5qi9)*}Vy;SnAqmelfk)uYd@?{1EhXAJ4*;<=by4zM0SQQXHgb-uG#O3muvJu8bv zDJv;?-vJsTv;zi*K!gWKW`PM3wnUPLA;fcac!pMGQw0>%R>=(o1PAkHUL5gH@>t|M znB-#=zSiE1+iBW-|1p|CWJ7N2#5X@__GJ|3dlJJ?_+{3xvuIBKp^WGYG z@;3hI>GwbDRVSF8e5bY8H8PDu8?2o(QbV74%a}GagjLMOV1aMxG@>s>MQp_e8%}7ne-}-3^^BD+l+(C z^U+uz5_#fBaWOcn&;>NfdAy8CQ}#vT?HOy#)?|%Y@Ix*p{WHM@ke$ zZ56Ez#(i5dXmcg4-L$(Z!@}kFTcfwQO}GF1m+>B5F}xWL=)3mkmV1137rF2_Bt*wS z{$~C_9o@IV-qDH+t10XsAe!gvEOZJ%jSIWB59h;RnEVB~y@v#8^6u|;IkUp&or*B_ z;nOS9AVWQ~q;1^LCrJQ%-bS}?->&|1cY{ysU$fu7ePa#gJ5xAfDE^>;Pny5bTN-#x z=Kddn-qGnQKF{Xo#NQ?SpSIR(1+Nj>esMP92{x0Me%;3mryTR&uFzH%)CyfA^!gRAk<@~wjI@X zZ^M@6x77+D_^eeV9Ha$`hWP+qT6}n3nQLVDyayR)^Y^)F-O$A^w_EYs;#R zA3y%c?GJg6F-mKDpz-IxTvszQiL{+y6feaqU)2L$&H*^H|DK-2Y&+bP67Z0bs}ka^ z;V)wtM*~}0G}dj{Al%EoEer2<`jM(qaN3}Ca5w2SQ+o?L+Y*CxI#bebm$iH67#K5;!Dmk4>H*sioiE=R zYq_*IkAXMs44yNRdE#w$c^vUke#i?s3l+rQW-V`?cR(HUt+#hS`n+oHL}G^`anuy5nIkq2~YYHL+VQ?HsvWIOPd>0KOU z82X6o#0Zgcb?9svZ(M`;jaAl7!eB(0Pjhl4vsNMBwqHj!KUic4Gf>GTl2q!CAQlOk z=ucM#X(V=`5qWd@R)r2#iPkA6Dr#wIiPJBEE9wOyz#MSU5qSEJ%F5wksY96ru!Yg%z2FY6f=vXSS;)(-EXAzZM%e@_6LP=0NLxss(3%}4`2ksA9 z?mAWe_uo?TC+=SMO~H*Z|NPlbGG__XbwEA9DkZsUEUtMiF}#`)Rm1yt%zU>BuxQwO z+1xV&eSX5!7*Za_ZAvV)9RX8F()l=)Z@j5ITK>BCi2onW@eSvvNI~Xiy?fItrq&gkZNgve%QUGChPjQY zayW2}RfVUkPw%Whmw+1&_hIfim zQhH+3Qm94=gOgHs*7WA>&&6L?3x7&;Nu=D9XO%$6&y5lh=sQ9eFoK&X)u+>DE%w1| zVO)6*Z|@v*e;lMZRcx*gD0m;U_MiD`S?ajd_NMMt6z<(wdn)Uc&rbFiDWk=0y`qSL%@Qyr6Ze2~#hmHi=~>Sx0xfSW@6_}iONb;Bc;lnKkU+PU-adm7 zpbcN7C>^C#nE}rRf~>u7pgf9*mm!Ma~iLDx385^hs2z2t9N3dy{pF0{L!RQyNp(X0;V6xfbBa zJJh}P@DRQi9^8RqacI1d@4~?5q&u{8#JT^KO#H>HbR!pdwQygi zuPfEfvpI%Ac0bCu*li|A{MxrL(*ihxGS7W9>BB|)@vbBLr$FLgnS>xTeE2WKb5zt3 zPK}vEebXLUUSw!-x>|jFu*Ir%2?(SR;`Z`!Hz+$GcmXfr35|*Brh5)*y&bP~@@?Il z5So!IWh->?bFB_nA;UAr(B1!y@GkQ?ucpXpzPO+YSU4U9F-mWH;N@_m9EahH!mM|X z@ck_b1GI^K@Aq-HJiuMrY%q+4EN6)~r4M~{5d1|7 z$B%oRIB|j?j7fSgIA}%L#j*A##NZlDO-6SEW(t0i&R0KQv&sgFSg~7zY!M_&dMgXd zkKiW@xS7jAt@Xn-5dh6+*w$oxD4tw47uQg@gI@SVhg=tKu3U;8WQE&ZL**3xC~wM- z(>E+S@j8(|e1WY04R{hK@*Rmfd+#%_`GfEa%eW~OCGN*a|4k)Kh$FTyRv1Q}K+pu` z*q-(6a4`mbt4-6_C~9TT>ucY*Hwk;x0ABEXVR*3a5-lahFFr1z!uN4*g~1*H`@c$@ zogb2pSC&QCvu772rfMF}bY;2Y!d)WejyBNNyi{JTtTT85^tjUir%<*@ zP(ngOxj_(D%}ioetf1CojkJLL!6YH0iEg36v`&AkY-6QdjUv((8fw%seH$FEZdyYL zl;0ts7l;Nq!qx-NqisGR7OMUb1Q?}EeYE?n{t~3|`N-mlr0Wj+boHt0SpI1e0ztg1 z{fJXN6zBk(dc}<*Xx|d4NJrUGX-+y?t97Mj*6HQtL0|or*Z(Y`O;lT(ChPL#&yYVhYs-n5D$<|%FBGOscw^Dste{-C@aU&10VR$x^ z{6}rW*_=QtD!{H+y_~)68X)%6&mPp#49pIK90HjJ8xE7d{Yi*2I5@}!PfL|!7{dAL zsq}aOfqsGe$f!HjJhHtoPJH4&V6Hd;bGe7#zkh##o4b6q5F!KyGB!pVS0v%`-#)?e z;=p=(dePiS)7A>Cv@%{iDiEF4!M|(g&hI~2@VDfy_vG*dkSaekpKzoeg3*2G3muxz z(NcOYPnDQ53tQc)(jMLEW_~{Ng`32y=#Cpk8wi82c(mG)U~|sNy*8~0MzeS_!x!K* zOUG!A^1?0Lqx1Dvk-p1N{K{(i)AtzI4uOsKCoYX%JnnwKJ{r!uEE&(={6lj;4n44Eu}DiVy6AY#M@~bfO+TleF06I(6F;SMu<^s+G6g>9lvk7N z2iOoUlC9B6*vRd~2$3`(%It z6rPRnX&?1=d^z$pc5S;vG$bP*X6e*2T0E?Wrb)Ho#fMt_2fQ&uJ!YyPL2J0$80vxQ zq?{YtJRf!rz-n! zvtt+?9yYh4Jn|Tbpz9_L|594Fflv<~si$jtyl&s_2bNX6GAGv5O3_d%AE2e^>kS(= z9KPkDYHNAu&g$|%x1|-DnM9A|SmAgi)bJA-jq~CPP7~?7DR#>?PI}wVpFV{NMFT|O zZD?NA`Y%2R0B3K`eEHF z`2IeC99I6cmG8+fjF(&8cvgi^T#P+MpV^tck;Bvfn4-$Hfe5|jeRkVXl(%*6rlFkt zQC^|3B5IeiZ@c1h!_?24wNAcGJoU3QX2M-if^_MeZ%Qv;wTgRK=lt}(l8aBjW;i0C ztH>;0zVnj0BmA9t8<1Zp_wzj6R8gJstAyoinMhfch5W)g@^5P+}&lVX4mXBsA^ zR=_up>BtY$ueMVw$k=+}&tEGOlhM2XfudA^c#FfG6s3n6gomPWa+_y;+YDAdn52df zfemSd)D+@y7f%~W6Aa2f*s3~ote?YnCSJL(NjBv`sjs00%7Qj&X!M>E zbNyr`pSvV5Tzz%IAADcHl_S~@q6K^i|Mc7^hp;v^cJ0}d)I0whoRPZlq2)d3PBl~0m(0=H!a#QMbMm%5Zr+lQT%6Q9 zVZ(i&(t;sdUUVxm`sJB3*}dQjyEr9g=Nk=PTQIDO*x3b_S7rU{m3L@#9PWWEiwd9+ zKMs!qnqr30xYCCTStUgnEl;PO-@7dUgN)O8X~wagW>2TVQZb z5x74Ln-GEg1^#P6|HMH_e3wHsh(krdkW9iZVFw*znA9G9)Kl`8w-X0iu}Ny>c98y^ z{gg(R<&N^i@ZEeBMZwSR`a0Y$y9{S$_cPQ}*ZivN zq-~c^W0?6!eVmLV`G}|5lqwS_=eTOss*cHm)4scdV6jrb34~@|{m{i^ZZs}|=mZ|7 zrCH}#GH%`42aVWMqA&5h0t2H_x!+!OfFZ{L^x|hXJ@Q&B8D2LYqKciah|B+v@<>Jb zHT8D0qd$~KM{?X@33Xch^T+5*%(0U>`KSWbYO8mNSsX5748y_^Z5%~jqC=~nWsxD7 zAL}|=T@ei5_gJ9Bj|AzY{M(ZpCXbEbb*X_DZbH)PVk5F+hmRa6R~;)XxIJyE8$sS& zk8M>xa6?73k#Q!qQN|WOf!b6cGe)^*6yt)>`Cx-qWh_!a^4)DmheGog z*DfU*SBDQQm2WA#9z-c+d{L|C=ZhylWvz;oQnP0}Ed*exd`q1sm{#U^6bI>8cZ>G=>}l+G9PhFOO?;+s@sQ#;?h|g^{|mALiNY2YZAkR4=TlljWrfQ;& zn2!;H*I+i%vehe#s(g^y#D1{=aGrva(ge3F`SpRKf}i>{bWY!xnw+$ij7F$%py1iE zT54XHxzY016?qab7xY)bP4xR@NdFKsjb`d?A7*f-qeqVtc}yVg<}2wNS%g-GlcQD#WQCX7^7#&x|4gu-U(XwhV{Jxd zjqE>WTdzNV{yg7>AO80*yW^(Lm%Y*>4^|9Cf@*itbUn zDLuxnzL7x_n8V+hmQ9cRpNVui*a(#?z8l}A5uT5Pz!$l>xeWkxR)0kK(cn(YwI5Pv z#tQ3-^DBS>jC~FTgj}mGP7UF~wM|vG68f)WG2bWp>K8$(%94_jK(SrT_(vsLmJd4X z5J>ndK_3w@UBZ0~&r2E_P$rh&jSD;_HxLKar%vlFVyggH&J{aYnvl1 zF?KM=XzEIhk+`!ejWx5RbODE%1jDIlWAmn6?_8NHsZ)|34OQZo-igO=zt8{xy`Nk~ zw?tjFb1VesAd?iZ0UU$}M~o&9H$W((&?S+bb9mm%@qioCHJ2b(Egp!~n1(7vZuh9n z4TaCu8eC;6Ws*v1+M-IIb(%F<8n`3%#&9@(eeKZ(K24h-GsUL%VAU-<-DJG=&ZyDGt7 z#<7|=eqdezpjq_SL;afrF6vAHj$KL8GHllpJR6n|EvWFqs#W4{lDG@Eut@ zUSs8HO)0Nma?;OI3f+E?tM-$$k~L;q@7-;hKhDz|KuxIjxrQECl#*t}$B!rG@l=tR zf0dB>jS_4sF_?}Kwc6(PwhO4W(mh=l7i8UMKygt60DyYud zXK$s(O(W6Az-2}0;E)y6EQxxn>8|fldZi%RH@BlMF=%5+R{9OsLeU{+e2{*2xADs4 z(PS!@^6=NBi)0^RYt~kjgw#lzVYduSwc15D(m?}rmG#4TZTOk7b_7a3qKwpp<>&*^ z6_jJt<|0P?-){4kA(tY{P`&&gQFOjyLG5ZierB3H83G#iI!cL^qV?ba&jtiTY4qli zuh$U6vedkj;>R`Y#1^1xe7Z8MW%d1`>fwvQXaAO#fAVwE;w-w^!_`Z5f}JA!m1}#m zc;kfe#0`|h7_mf%nsZ|MuDUK+ecmnsH7El%NMZhG#`OsN39ZTogrBD~=au#EIqNy+ zw>5y|=146gpYSQ!4KQuGnzGHO<+1!^f5(#yi%f$}GnAuy@?-UPF@K4FuEDEgv5FFt zoTXjo8;TBBa=k=heyq>iJ5E4CWYW%}hLK z@C|SbE#1b$#p@kpXy~i>^h-J-^Vp8KoF|ukNlA&7?iGq>`TuOf22AS@A3wz~bYRPN zFQOu6|A5NIj?O`lgM}myF#2WIx-B4Vu@B-8k>0mRhcd8`TzTAoP&!CXqn9-H+%rF7n#&0kshL%bNudl+j?bjqgF2|-P&>T zGAMU9dt&uf0OWJwWmxP~v(@cfINKavaV4H`Lm=s^dnl#!_C>)*#rFiFw^h&~it&H@ zHYmoQAG08e^1By#(t|>>B;9_!$u~+ccqz%aWy`x@Os_z+)|Wn$y{W~olgCcTRG?D@ zI;p?}2V@T&IwYXFiDHv8{v@9}y2>YzzR*`%Y16vNfQ8gC_9I%&oT$T;ytr2Lw@bgx zZ{?`T$x*a-k{7+6hX|T2$Ihv%XXvU>Lt6wkQOfnPo4EYXWVN++-&Ox!_gwAQTAaU_ zn5sS<#tS?GG0ZtKrJ`gg(Gn@PKh$Ue>vCafT)_?bO?a&9NO*NJ2kO;V6M^zF;pU1a zqO>tD9R)4$-y=Nv9Td+I%>$c?ii(8j`bz%Hw)dk`a|(?P4?lzh9*C~2esLM2cMT9u zxv-EZt04VLqfZ6*`~g1YXtYou^v_coqT*Wli4id>O+sI*$;7L{4oE$qn5?y>m=Hq zVf&JjCSqzq`)jYcehS;$0yga6mr!a>AL_a}r%QT)&NiZKTFkxE0jjrS(v6~!kd3Ff+=7D@`-g?xpcqfDQS85zo46P{NY^Hxfx?lmCkq}c)_^19n+yZW z;(3SYutBJk7+S2tm3sUmLZ)+WCJ|qf*T3EQ_3PIels~1? zWDJ7e7+5GoC=Ll}xHg`|cw`G?IF0KuV~+)^%#AOG??g?%$RWv~=ZhnjwI%2zz>x?w zY(kldoSdhobuoFOIbT}>t0&#}{Fd!E^>9D)Eciy*h3aKrm4rYjGs|l8yL$x6QGb4X z4Y^$Fok0U7^!DOgH#_7bGgT#Q3z{@zWz%;lBeTs(GC8A5QTkG?$-JGufFRsNe>#un zl6AoRvK4ww&IdWrqy!WjPhnkVISi7VfR~@5pD2vib^l_wmAjCrYhaES`cwPNKYaFW zvbcc2(_~CXG=k5(I?6cBj9I=t)M56RNV^CmWS^bd;kUZgP9vu` za>VxH?q$QLN`DLr;dV6cpPX#ES#0$2!YEOYdhXlK%4$*Lloi_x%%+;CW+|Sl4mHu_ zMgCe?h*+pPCN-MwJn`)$V=0NB2zDee5TxwkWm?BGV@4_L;>48NzDBz*>ET=IyYI_0 zb#F9^`hHU*D(=Vrmd}*>#{#vuXA8z^p*c{=gDb7D=c>w_k}!s%l-}q4A#};W0uJ5X zd*p~NN%@&V{MtK)DGTQ7^t;4F4C*Cgd-Jei;;T2;r7w;Yzkp|JRV(tChx=NFB<>G6 zyuMcv3ew~@OHC#uyp%_ejNi6ao+V>8WxXIJk%JlX-&(h$&ZMP!Qv}PJF{>DxuO+-4 zDypoyL1|ghP7rXc%B3pPs}W(6l~-U+^BT3^2RIWer~6&|=IGacaxYc3hSS#0>~Cg! z`J12t-hj`G@veWKUu=I^W7Ag#A7tWQId%U8`Kxk!>3T+PtE3f8DnvyWVLDXbNpPFb zTrV+yC`HR2QOqIX!ud`?pAy_|-eiMBNSb0}?Al;9Z~Ly>R*Li86_-EPo(o--<4Mw# zyhD#KWWBDAj!w>?U0c2lTIvjhXHPA|+T!a(iaO=dKFW@*TlYa|>;LWl6$9yRkq{~AT!2Byq7jfrX({OvxbL^nea`=n zd+!?K>@oH}ley-abAItY^$x=l6$1a$fXZSN&4ax{+`v)vPdi&?e}j#k)j^AhxC=M^ zMizoa z`|{v@sW#fq)?fTGb4!41I&!6}LcYYWEIa?}VUF0w`)De^7~)4KHaO-EYea{@dl&>y z?$#g-25`BPWZodpi$yufFnFJs`&dE!c6WRSPy$pkP7$+_PkkZ7g7S#iu137QDvuDM#OMbk7J-A{~*5=lGzfEgT5Cz9PJ;62ieaBX% zyY@fJ$uSmiXn{CdD&AvNEnTxTj}`V5AchUea73!vQgAd!GJ-?FV7#$Vd`B%&Wlf#{ zbIgeoC)#1)+ci5ZPu{))87lDjo=2!mUo1O*>J$UW&QW$*WSyj3KI+D1NXz%&>n(`h zpS2etZbo7nUG#36mP6p}k{5CR)HgyThy+b?x=vLk5Gh@;_GNFhJSX{yR!15Lm?`ax zHzvk?xJ>76JeRrU7}=d}>frJ#+Fyxw&d`;7*ON} zZ-V_tEoh|oH;=%BkWyKTa*5Y|!gcQA@M(pdYg22sexZ-;eDp*J`{7;Urkw|KS$}Lg zX!62MVe`jjH7IQNNC5iwp>XFY%iMbgFlQ{ncL)(DfyB)_@4J1YWE8&}Hgm|MQP=fW zU_!spTdZbRs=u=fCrZy?)-H&x^iR)X|Brj*9!l`J+b=Xjlkb}4|4IEcNm)0D3>fC;i3w*fi~G(RWf{MLdP5XkX=Lb+%Q+f2b=#p~l9Q*Yhw+FcgNAOxnaO{;02)sNs_j}O zgjcXGg#-tWqU0BN*P|ASp_bNQp1w_Gjru}KU*sBoK?rNq@R(94K}2BSS+u`idTBP* z@TgTbALeVBtEMccXu1>M_7X5_$;V>Pk{L-hbZs+H<9AXYXp+rv>Uh}OJJ!Va8qJV; zhAWbu%SW{3Z6S(MU^g#p1g!~}QJo@uQCtt*c*EzJ!1aK0(g4{8Xtx`6@>Ll*`p<(T zSU+XiTvZiGf*&8)a5LYAKeQbI6`PUpH26u3G&&{f2JV9GPwx|&ImC4dhp$OUi~%zY zNGLd1>_+co%0`0UjaGQ}GFHhX(XXN?#NI?{Kz|yTrNHh6TEQC@AlCg{QzHdln2s@X zL*>a^_yK&-*yfWRxKM&Xp9>BzmH&|o46AB7$A-OOh6_K zci3SuYThPPOr3>g9fTv74!zL_w%{;n5TpwP8998!Q(KrhxFCLz{^cC`2zme|fSa0@ z0auj*wfh)SCjw`k*M@#|H8J#RVh({t-0)}cESFmiyHv3{cd7=fHdpdZ<naU@F zWrw&F-B&O0?-|AkWoKyuNe!;nb8T&HlpPY+u6_H4{$BAGx)$z}AX6B*7UjC7eU5~WsjdciP{3FKp_Wjstlz(Fbuj#L}eh8)8|sJI>B7i ze<6nLwU6ZRd`(9{Lj-eq7*Jhb;0C9_?)5JzwZa;&4J?c7!t*bSUovmGEMyo0b|OY+ zg@DD4`E4wIucR~&0mIyD5Mvyjrix#D8x8>f7^?oiO7O?Hx3DLH5`1rHB?Ut<0$&<- z#13x3wi=jIiN(S{K5L|fzCrbWslZ`BoZh>=;l~tMaQbg46gTv%xEUF&~pVA;P5K3_cXgJGyqRuEwT30F9nH>FS$5MLLNTZf|9*#pDNJ4g-&O8ENz zy3e2M;r5LIc*55MVg-xm_)h7g z!GwdrWf?#j%^pz-eJ~oag#G5U?!T_uRRN7q#EVWfN6Na@xwuO2BKU0z{8D zsrW2s>`OsW7U@R(wP*4l8-P32B@2qs1u6pBr4*td|9jxvI>MzLBKSgb z-=Oqt&{i;k$s9=ln?_*6t^hE1oM99KxuKzkJ}ru0*WU!ZyU7EP;GP|H(!fIo)dauZ zq9O-;(W8@piyWj315?D_?4DACb}7iE03oj^yUVKf5sE(rF%=FPv>3wsfTkSe0fNW^ z!8ZdyWm?S5myD2*W{aS22^EH+<$PVw_MLl`Nk*d<_GaeY^0?q)Nz0O?(l72yD+}WT z@P=CZK&96JseR|b1|DZD(O;*Hgi#?no`nKtZNVKk(ob_&5$6 zT$f%F3`+rj;z?Sn(L4`#y*&4=zSC;|ln&{Z)HYwh zYikKa4qW#~30R=BVYet7Zt)^Ga{amhMTCj;tp~s4e5{e@@~3-+`5*+^kF^)W4m7Em&KQ-UZapwn47ARa_v^) zNs!qSB*<>qQqZF zxn~I}{!@_AhBCX<#;2XFXawy8!5s7j^nWSfgRj6(`ihvZuKNS^d!&ftO@B2(%*js` zqL5bGo#4>m2h!3%u{%|kg_jqWawEs=T`Y=!&VS;1Fk=N_eOYI&*Ozc^kG;!&h1NL? z`9mmz6w$1LXw{Al!9nmnd>Vxf2})ZVgqje3;8|Gt3}ouuJMu!JH}0mpd&lxqSaA)u zw|CS$=(M?hy@KsD+2f2mqKf`;m5>-=5AGeLCVNo} z*mjc#gz-RjAH)**8shKb#!U=~x6Bt7)4Y*k7G?GJzP|Ij)B!;-AxO!~g%sGE)!Gi8 z=8%$>4+Fn$l5#*?>HXTL`S==SINBdxgU_d7=gNEymd6N~9nuwFKRR&eP`PF{fOLR( zHsrbzU=l~uw;F|^{f|u^vZ5&1dqEh4E>NAW&D3-S1Eey()S- zq{gUF9kjAqCzX{L1kMfk9WmyhZ&kRVIiSsMrHtR6U`P)i-7MZ8O86aMqt77wnzB{* zR62Oz0QnTKlZtv+>Yr1*$i26)Oa+1pPC{yLbME1j!OC)uplhmN)mCD!R{iacrFmo; zO&k#lkobC6k?vzzXM;q|wjy@Cw~G|=IYNLAk%oNn#yFwwF@!op&DGjc$N4Kf+%j*3ot{3Xk>BmaDO^-=i0--NL>OMb3c#G+*&jg3 zeFOP=3;`5b8iQ88+yP)?E;_FxhJWT3%vX<8rv!?7m9Fm^GO?Y`@f(f_kCT;^4ESEY>8~eP8Gf3HKph; zp&fn#I&>UIme)AZ?NO{izHLzA!{G^c?ajPyE2-G_jbT?OaD zsO*en2VM#V7wDb$7(=aCYcb$;O*FnxKB=V^n8tf=l2AyVTA6?}QkjEM^vIFr5Ic&K zAm*GJIvKhvMi?8Oc#Lg=31mYL*ctqK5|zym89QASuQM|^wOcry|+WN zK08z$3yQSco_O8^PONwu+TJ%L0?+cLF8{)qtSs2SpnSupH|5*9EbU5<`G5>xb>DAy z*)EMIff#NrAB%@OmGTKfS44YQbCwE4{(U6G1pJA|V3g47H22ZV@JR~DuC1+vBf-R;?ChCp^Ia`KlkZz>Q|cfNHw{43~OXvd~M`n@%Jjl)u5(kPRYER z0l8C(Xz%=koe?&(t;x}#D@oHl^hNMWy5(Gi#S1R|52o+gHv12fm2`BsaSj#CBpX%2 z`dB0km)MXE+bA3~5ool4JwHs?=MgZ6VnYRrv`TGLWFs$MMV9kkFi+o`gB?^Uz#~L^ z8l#dkaNo%V$?R@J01-z}xxWA~X3HTbe0rgfnKp=78TTXX{CSs{dyLuXdR#E9B2qHN z_`$O6(G+Q=?ikz%)hq^uV*{ocV9;|MupVKqUi4yawlm(OG zRIWL4tU)!a3p=gQ^U<%0oIl|qrRNH=Hj&+hl5#Hht#2M7I#R#3WM-WO5e2qt(}K2) zt4Z`}$kk@9nl!cDFHBi^hj+FY`AV5M`>$8ocgu5 zKx1XP3q5b^0Zs-+)IJD6xY9owtJC|e#aV2A^dH=CzME{`kRYhm`<|-o?fS(%Q7H3PYpY@e?%Q||-dhan#g4HG z2)&SjDwBbckx{h;Dx$@JAq`B8%h3Ho^%@*5WAxB4P=S}g859P>HYqu|FoX<30*0M= z1cW5j?*9fTswuL`QvzH4`0-NKSHZ4bgYJp&VM@>gcCqS`8*0Bc_W7mj#46DMPVftZbWB`H>#PK0US+>cO zIB;wfN>xU23V0Pzd5$!r2WCNr#{igJ2o%x{z$PREN(0;>`P0!*o{g!1D#P&?TEX$4 zMuw?8*vyS5D8{vr+-pd0mlstew#gGZK%6&>aRH#ivjz?+h+l{Q4meV_nLRm~`l<~e zJhp(-L_y{_^7O0DB1ag*u>jkx2Zw($=!zg(h8e|9^z{np!_e{}c;=`b=@oo{GUQm31a->3E2%|#vTkZv%d)Bi&U{#k&0O2}liJv76 z1u;;m0hwr^(IS=)==%mVmE+p?Kv!3natXL6Bt#oufVmF^gTddFC@VohC}0&Lywsy7 zVEfa{=d+te#c1j!jry$%Thec$!@fQ@fWi_;p_hY#LnT{=g zq)A*f-nam;Ma1|CfHn@i>G#7o;KFz*EkVZ&#h&)Cqj$NvkoC1%KaI0hnf+{Cy9`(< zpP#2>hiAtJvp#p%#Sp{5_g0z$wDR&=g8SlgG!rW+DK+j=uxl~{hs6`HqV0Rc`sUsp zXHDV46vG3sgWKqFvoOFNF#3au^O8kQxQxyzNKFAz)xYZya~J`?#>TDLg~=Ce-;9QcK zOse<~eqycGk~YL-+m1DiiVNf%g}37)EOYQrCfId`Hk8*Bm>>?NSs~$7eldYY8W**J1=;NK2{VgKV z&)+{B33x|#L7&98hAbSPKY#X{0;N*ffddB&e($EI#Rns*4K%ta*vS@}T0X0)s)8^! z9hf`MgFtiqtSK#7an_ouN7xefr%Z( zL&8o#47ZFxv6v=+{^pN+cZm0(j4&jD&eSNDM=~5g7f^La05N_QFtpe)m!&=u#i^;g zYVR!*5Gn}#x}bR|uCFWTglb^~?Dcx?;eQ3{K#rXd_w?hS`A+6~=>D^l`Oc9GrX`W~fb;u=RtmY0yAc<6wdlxYtn@u(&r$2W^6qCY9xuip91nRJ^}b&LXf1V5`@LM}xb$sM zC4wioZ>tcMaz$)^&WSNRPQZUQfP`9z2{BGy{s~Tz+F45Pku*~MYsSYbc#;7QPCN!a zk5MJ*2hm@gV{+ZoAY2njnu?0i=s7uPU&2;16p;fQ$KJ<~rfuI@3P(E83#df<#H3G? z+MRv!;+vdl)>mdKdSD~^^YOt@2fj)ky@P%^x&wO-mcAbh-CBd1mNPw??j99APyZf$ zc0U1s^^+>n*icXiOD*>hk#tO}=A6~fo@QNnr*a>^Ek(zAv%-cyN>f%k=nGL%y_MOT zfTvmoo2^|=xUEG%>Nw}$^LcYaEiM$ZN*jTKl+qyuxkx5NJ*@GbRpGnGdDy>)i5J1? z&O;)AT4i*VRJi$iovz#ZhAVE?Lj$Lxb>?~T6T}JaeFC}nSu-zn&GDZ{F>X+ykn|-P zVlQ$%Fkwo+%aYt^b>ZJ5-3|;Z8r5jt3gsBL3_Ybm3VZkEwIDD@g?YwNJ^O1C**lzp zjDSB>09LM@UdMZ=i~(~A#L;F$94%xH-66TOH);Pl7^;F$7Gr&~(GHz1TrbT{n$!rQ zUtv%c$h0^*)a+S0KT!y5cP%MOya+cS>$l_>#ssk*NMiTVu@M3@h8=l6jEK)@u{wVes2ksh@t9e z{QUzqybZA7!T4(|&vdX*w&mjCug0_Wd&f#M!95H^`Bq0jbvLvdYa6M3j$WL$BK3{E zsdfjRiT!gV9~`LyfS=*qJM7~2yas~+LzAxdJju#v_QP3v{??ZNJ}s~@%)BHCenEg& ze3|=X3_foXqF}VcEx*zE-{W-VEXMaF2QQnJQPwvWmgxEVTP-@_Ww+Nei?xRn*V&x79Xzu@CGXN z%ziX{9DF{1W)f&L>wEhS(ffDO6`TL-Jqc7jFg(opZxXZrUq_T`VTktZ^z#S`wSO1Z zL<6850}QZi1LO<{`Jgx%3_%u_cO_^`Qr|$GTQBr?$;3$0!w*wRYp%n=KG5(}{0R80 z4sGDDs>Kn1(mDQIt27TBcoGd1;t{hdBLRH87!6aF_`iBs)K`ivvkn0LG7Bp8tbZkJAAm-S@U3al@yvyA^DO}R z;#0HwtAu9TBsE2w%4cmEwni==vd?spr@AsEKFAI zcX8u}+5gqOkwt{gi9QX~`>gBpX>O;_->0HA60W5j}!(n^BE%82$qPt5SZaKrl!P71@NL6e>e zF?IQba_01@sUP~ex167xHSJ4$hrq3m5N65>6_n4M+U*Md>SdwA(9xc0yK^lmHb5cNVmFrOgMb@I{IBR^ zRY*w}s`q$ZbP94S_-*%lzF6&wt%vXIs#!sA5NC~w&p4b63_PgiUp_BYlWdkiqtX>|e=rI{)Q~nrp;`Zy+VJMKkLjt`0kg7#h94b5b%2e7nV@l>!Ep0_AuhB=)~~4h7{}u_qJgE*}C6bHO9<4-E_WEboUHJv!?(MbmuRIE>)}l=gnb-yBnBV z_=%x)=q67^xFz3k3}Ju4*W@hL6cH`?G{W_bTOsP-B`a$GeglSql%Rx*p1)o}y(>d6 zuZQP%cyM$$Cx^_JFS`jjxT(dgpQ?BwVDAqCMfO=6HWrFROP6BAlx>wlhJ%$msPn#u z?hMHmKe#X7xY%uP1v||D+MDQ6%8<<F1>Lo9wr}QFOBTElp z5{#FQ>Fu1n<@x?VJiLkjZg1i9n2FDj5;6vcoiLOvbm5XL(x_K%jGr;>EwG1VAVY=~ z$M1T1)cl@ze~>9BLyFbcoV&Ll`TQ2fq6nY855q}<5?RndBtzCJlGY<0>{52gH6$kh zAwC#|)RBSY`i&dXaG5Cut@kyi2~|X}D0yS=lk%vh(AB_;SdjUtIAz)R={Q}`Z z{xf~W3hD(WNSCzO$lc=X%KPstqbPgA8#juS4tV{~r${SE7zqN^#aCfa0w#>%!fMrMD=Q2cRjq9nVQMj%pDca9g=6b9{FVz$62GX7HlY%Wu_gGkhb z0(Z#BLM9y0*i&@9mvt91iy<}%e}ewUnAkX!E4JnBjfptJinPIXb#>*_dt5?^l%8G^ z+=G^5G>Q;}4|)pyK2$6qCC&_zgS&~`wqsyn`J&qsOD$kaUpG_I+S=NW#bf=_40%IG z*`d`IKI{b|=tiN$HHr=h2slP7e6KcuF5&i`XZyuI0)C+)n<1cK$83GTkkYHGhOW-r zI3+C&AKQ%@Z0uS=+)cW69-w9jJu@GyP(yw-@U4tzp=(m4XO&#yVvvB=p^e*M7iNHl zs8d{!|6_meG&`<5A>t~CPsky8WZc>S@pl#J{CI}4bIzrZ_34_O`QUY4Qq z?nc$dgfqs9{lEGi=dDMzg%o|{7h@(uaz+mnaLBG7gw; z!F{r_6*2NdC25U=L2_aJMi}kPnWJ=DzDXY{KVZM{k&CVp8=4Ib$9;04Cg7()T6xfd zMkMjywrA2%Xaa1pWV`gX;6fBZ0_FRJt2Upk$krRra=5$ih-Ash$vKqDA_0GkMPSGh z%deONXPKz;_uU`zj=fBXXGs%3w{uo>0{W-LZU=VTy#bu~9!LUoV?*zO)17N=uR+0# zJtd7^wI}}dXEr}Dj1Iss+=UqlWqtv$M~}3?9W%v%+b@*>fDg%2=n%sbduyNg^D*$r zL9}$1%Wv@9A|>3X-_Q#dXHd&){!I~iWnLGe*B4J=qQ49^;i&la2mqpb(Bq;>lnZ*m zpfI<7$UI&Ii(9mzw%4|ULJk@n-dvySI>3JdAMvXuj(wpl$|yi03^fe~cD#K;$3_K!c339#Fd{Cnw(mIDQP=xW+U1VqMZ7=-Mw{qx)K|8MmxDE|K*hp`08@85s1z5tB--;ej1KereEANuR-;uHv51Ob)4E-5Jm z6c!KM4M4nIWE!5qw9o+jN6mvmX`qDYd3bnKXOTf%l$(HnV4QnoVnX#TWHkPf*EMWun?^@RhgOjov`BFPef?c4t9d_1fDOA~8mx_ujGSDBy8u|9#KiOkz zm2!Rzn4<5tJ-}jVSb^9u15zw%7s2Vmolv;G?p$0fv0x3@rNM?)UR9w<+egDbmG$?6 zNUK`(TN4l#?*9qNOc|JdV1F`!KeJnJgalWPky6Xb4Wr^BiG|Cazt{cNF2X9=3xJ~; zB+X_)wjyQ6%~w=o1+y?QW&)GMg)>o11`kFT1P&S(AWcihu$E`v>-AQm_M&-r6eP7) z305}B`PY?c5D?YaJ_kZ7%B@4hl|>+NJK$j>S%dht*m1gx4L1ve%Bgn_dd+qL>s97H4@mXY zGX9NWegJ~KB7yGR(GZ1i>lkW5sh^csg|6-QZbDskR*5g%@Vozp98fsvFOoz}585!# zn_z@svBtPVPsRF^?x1+veOe|ZCZKHXYQTD4EuiHCbp0s&=06ijsG zSu`&W4Ui&xRTta|0a-C%r5wOxMM*&m9{8A)S!go*V0`Le13neBd#x{A=}46XeuKx=_jXL6&^&GGy2o8+3SpqtepM_lJ~M z!m^z$YDOV04L1vni_sfY1u!zoKS@1yq-u3RZAbcgU$4{P;G%L|9;8wF_sR!&&3N@e zm|1{d^L}VEZw=v(b19Vb>P-?tOF$0Rxqe}YxHsfH^2*(i-3od&YFanUrB)GTbLp)( zBd0lUU1nLPqkSVs)@ggtmJnJqRY`zOFsyy^X^!8b zc+JAU_;;JT_c;kxE%Vo{@=bEq1CFoBoqafyWYlv#Yji`0=Ae^AOE@i_Ctx?R{3tkTOL77M{$9~m%!-g=(eq0=(*w$ znZ`s|gqeoTe^|;aXoxMwr)nIW;FyJ03DT%FMHxRMlApa?wlGF!3YNT^r(0k9X!O2z zZV%ayefy4rbQ#;kj!9$UEHc+Lj(a-jI?a}rau{d@1~CXD`)dAy%Q((!S+B6UgY1q| z!Tw=w#ETft#D&Fm&L zmeFpIc2mbsL->7z$_f?M^w3s6Y~bJ&G4WO?rDbDB2&2$CEdv#a-DXtRyrJi#n*tuH z%Y5G`RqHr_o!Z#!aJu!=Jh6}3L)jcc+Ao?FW6<9L0awe$=NNln|M zgG7xUKv*}i<%5ABW(!tg2@tJ7UbdpJ826@7N1ma~)VJI>xRyrfI>G|ClGR!$^cZXx z+o6_N;$PY-hGcOmaG66vA`M;0G%MF&6%2+QH$@4Jv_G`qFn0HxVN=Vy4tACl*blbD zgN&HphQpEm^Pr{gk0}&R4D$~TrtNdE9Q3efRM8wcazv@r)^LyMMSX2b(O{&8$c<`P zOwq<3R#4PR2+{VldEf+%fVn;WUTzXn0Ll-!L8opM>P`(Y=GC*95(TJ4j?oByrQU_@ zZ%=H-4pd%T9-H>y0eiZ!13m1tHe4+|Dq+nK!a{5e6K)Zr4IrnDoo~?j4Ql+n2weAz zavo?I){ku1>}xJVUc1;kZxyei#0vW;w1#DFNsx+++It4jics&|81O|?Lwn-Y{iW95 zANAYR(TqD&GamtY%IM-&M%7S5ceg6|b5=FQ(%8LF`-}NCs6Y)sM=`GX1H9LvP*e4k zIb`&EtPRLlC4dM$0y=i$Zqj+s4|%M`_3g-uh;^;l7FsC^w$#<#ghYT*`%2oJNk&N`tN3brVCp)g7zg*o$OIS{y#D1XAASaaL4Z}kj3IVVA#00;X42d|&9 zus8ZN^manX6}tw(tlV2zP*KL(C0_fXd{4peV=+`;c0-UZpSIKoZ-!?9>@g7K++OBj zHW<#up4Q^$d1JZm`u-xpV;@X8RaXW6Okc=tYqy4ojSQZ-EpV<9113tGsrpBZ{$W zy&oie=%Rq4mxCgg0Xn%{na8rN#|*(%0)@QU-L6lC`e+eWy9UUX2prok#IAu(`uIx* zGp}m{s*|D;3e786D8xJ;_7YgrVV#rX&fC*rq4r1nw~m$aG>VuNYRAC}7YH+|6;4o| zl(bO&z^ZyR4`y3~8GWOcc(%DDiaHcE2sI@)jqmcuXpz}WNAOybX?9d~Q(ENksY1#c z^^UejX5BaBdFI>R?!fq1DkqlY&l1Jv9;_=|k{zE{a$NpsJKp?AdW{WAHqN?t&6RI@ zU1l1ixGC$7!n{S{)J8P3x&75Uk~U&e->uh1*5@(R?+yoO=WG=|aroFV9zNA&tL6%; z)@?IbV4D`Kq4Ca<2ZGan2gvz))0e=^&N*}wOQ^dg*D*r4?GJ*5vFLL)zD6&SFR^fbf2$_0Cr89JA837%BGW~CdBc_I+!hGIUGdZV?nIbxVs`Q z$Kl^M!mV6nijmAeb>aN^Q6Ey#dX+Px-0!t+b;+M~T}-QcPj#Rp#6)BDQUYb7eIy_R zrQOH;8Y%hbayXj;6dVm=82p^_+?V^4kViTwj|*uLu`c ztt@ptqW-BTj*0Xg^R4hu&1lW4Svg`44w<7(pdF3GrEQbT{qb`Yu?}BtDEih0)u1Fd z>){w_gxRpE@+$Yn;(VJaZAUQ5V>V(+$-jvCcw%eHo*GEB+Pruqh*_@_}ogSq~B0K**UHC9CePJ1ya#CtG~>cGJMPD1aogN(YoCg?Nvq?dx?hmPmDEwdGY5`A&7yAI3-&Ri zE7wZHF+~T@^zbDnGdGS94jj_7YvN6{&o>I25;cNcS=uw|H}m@nxPxgg7|_0x^s(*L zT?p5({IV*Pz4a7ScwP&kG?QE8q-sfk+ zT4%qbSMPL||ImE}4&fVSo8O3I{8_6|Tjx`sfZ@34|kY*FI{yPp=r< zQ_IypFXKpQ;gUkpV?i;$@;jAz%Er%A0o|h3{5IQ2D)DZjnYqfRX;WHh! zq2f~ag=p)iN!J7f6%$NDESy4rH za}~`c8#3L5j@Uce0~AWpMbB~flg2>i)o5lA1Qo0R$vRt>o*ezu^+Z~6fXQCDl9oR|$uOEF(r((}q+Y3(FC zvt31!)Z|GhkgFBjWp3da;~m^0s`06FBi+JPn@fntM>4>yYV+qMH9$FQC3&#VvX+De z;-BZ+DsA4|f24T8QR88}8~EH(zlP49CMP!#ltaP~O_N=yM~NzSyqQdk!acoK+}xoM zDV=p$xwi~z(aR%mq4n!s;SZ~827`QW9}!MC(3wBLir|#qGy6Bj=rQlV-h&%ITRSvUQqfv zE@);Affwd{l z$Ak^`u2)TJ6Y@QL=D2yBDWSF(T&&)nTvUA2)3EXz-iq;$21dOZlAU1UtT$7CIgxQ4iNl8fL=1{00Oj8dyaTu-Kp-0Oo(qhUu( zzeN6JbIn(`vq$0$wXZPs7}`$eX49Jelw=aIVt-I1%eNgD4RY^3*W623#kW_l^vTs( zwJq^($Eo!t=-}GSA;M8K#Ak9@JA>Nv#`}0W)0^sf6GZq$iSn(XR1f$eL_ugJA@LEEG^%k2UpIO00iwQkH1cwRje>+m2vz2^C1ql zDN}hf$s)5GZ?3vypKT*K+SJuGFvv#Nxq9pAgopvy;coIYIext?X@t9n>&|jvI_$Jz#n&F!4! zXKPGiQdcGm?+4R`=GaRvlj2$OosLXbG@skOkz#R9sWM9>@&^5;TtIw zUsv2RqnusyUlFS+x_Vl`9JZZKIJ&Z~Au}_@OD)29NP3?J({8N)nPvBuowDm!c}&|s zzo{d3-FfHr(rA7<*T?pzwRTQ-@BEqN=r}^^-KB)xddO|DY7Rmv#PId?(1DC0a!1jO z@)ZC&Um)|{EU7!%{~dt)ot)pR1|rpv84nOvAyc&Y? zNUoS;AMyNpEw<{KlVVY5;2Yl>aE7nhFaI@#((<7`Z9HHWBJPocqT21gOXEUKtYr`V zf<#2u8&>aq={)&m<~e7trpy+64W46KPF9c%eDnEBzc7N29 zcN22))D&8EQEgmN04l!R*oA_U9-1jV*^G|1$gZ=cvd2iDZ&Fki4A;LZq-1LJR=cWo z$b-1DbmUr*kO}WUS~+U@Mp3(F_qByYDOM}5PUVI>K1$+deNhfGY_u7vt9xQMttJhh ziLZjwD~rjg5ke8Vwqsf%b)By@dW)@eCQoUZZ#@OZB2xMsRie)swc9^lQ41(3ljrp# zj29S#mF1tW1JOv7A9C8#fW{fIMKsWfvtvv`LLv*OK*tRv7jAKbk{OIQlNNyI%lC<` zd{vf@AU$#*>cg$_mUMc@!LC@*FQ=xG<=5_rrRZ5=y-d>r#kg(#XXD3{C4 zcLB^4@|fShZ(pWzs;muwx(D&I!Mp}&c{71EC0T;x3ZvBl!CjPYgd6W#((Q9{2X3K< zUC6A9HDB}PX||*8{;0{lfb1@m^W0tf;Oik-g0Or2)SP>{A9Xj8AwIPVa5{ONHxPcu z-aUde<~RiX4tWs`v%p$j1W5u>Ex!QKC0f=jC^VoxrN z@Nd@#U4Y4vKMAC{Mmykx84vFk^88&=2*H5k0%8CKQAZSdc#Zt{vAE(29b2!=ntgry z{jvuzWt1%L1Re>it4**4xd=HAoMtYQ73dW{}A#pA$sh0NK-VMq%GXxqDtw zL=HLr5@wSM>6lx^JED{8^xz(?5WGq{(M$>yCs5#cOA+FDaF@rXLQm(~oMaW~kk#ed0{v|_bI=tc{G6vze+)>-n2md*7 z;r(D*pNvP_NXhKk6|+_v11Wpy*BK{t-RE@Q~#Bo2`csZx-3Ww0S12ZdzS_6#`s? z5Oi{_Wp`~t|Ct11nD9ad4`8vE*Qck9d*4^iyh)NX3~x8VngPS-<3gW*NoRPJ-8 zHA>eeN$$1+*~)C_jkoAXhN&Z(mzpZ8MuTP0PKt|o9Kb*;#*n){R<5@->a;s@ zuL5g-yF+L&3UVe553HIm>y>ZwG&NlE=aFDuDx0nmYdY3yk%?VjInSUNJo!%$>U=&z z|KP5<hM+>zp_e05Ngy&e$(~#K+jM{=k)@? zCfE_k6tadg15U_CX-#!G27o#N%hn+mIg-|+g6aYnxVHz*e5sl&JBF^ixn{nO(iYrF zA{;UQPz>O4Oy-^KZF?1un>u|xYcgvRS5`k%{GeYFZM>0jXmUq+jrh)f7HVyE-@L&1 zGkB`DnuXZWv|5%U-dL4n7FxG+j_LD;%%Z%`)sOkw>+OTn(GKBcvor|;Z`iqK2@kBV<_ta)9hYV}%kz&SR8yLTxmiAQ zGWH32LoDKJcALGcaYW{DTEdI=VtTvO;!-KP$A}+I8H<$wfqy3Q>vJhY$--1z77Dist*;94hXdHC^9 zb4?7nhM?se?XwzHo+2Zs3F;Bm2HO`I+0#S>jF>Bkucah)fGIfjH}A02-*+lEC5kor zNvcR8LG_LOOVI~I`pn5NhCW8mO)1rztPcvm`X5C5X(v(2znnRU0{#bS{_KAo`RDxj zgZV&k`R`x-FMi&=e*;hdLuZ>)WMmh>;dmS}Pf*uO5E+m=Uw}lNyTEL-fO;Nk5Jm_N zYEUT$*d7OLQB7Z;1`60ioiw|rVJmb4IQ6CxlxqO~K%5~|r(h(yFm8id;zByCPR@}~ zMo7`MUH_%Kqy-2X>bS`<)KUo82Q(XY6c~b5nGo2oNk5%u+QIldAb>9b&{Ph<&7DeT zfUj+=ybUc=r66u^X}e5Ztr%KREz#TsS1V^7bx)f=P6~2P?0a75=?JYU9B;h8$j^TdHL7rd zXnuO|t|+t=gD4GYm{`^FK5KrKR#4#UNRx|UgiscbA`9cj$a7#LODYvNenZjcx;%Lf zh^Bb^pI=!NrjsaUc;3H->5dI{ICiC(+JPP3aX@25uxeL;;O1cHX;+)ysO18}=;rfh zg))KLxCpz!P%ytX9B5nwIP1hBq@0w+(w#UF#2~JEZ#szp63~ic>Li{X5dwqPU+?yB3fWEEUBi8N^(5?ZU+ra^71?H@*gHG-M^6k!EIief#9t$sl&6?5O19 z*X}5@MNrqetX+p)%-ZYe~` z7_LY_BRO;zNRV#r!{;6!zcy10w#QGg8n8WGYKY*N=Y$Z79Ad=XIXt9UzPjjtBpz*JKThz>EjMK!HPRij$ea2^9^8U_Y%rHM8@=-1yV5Y@CpFpeOyC`hM5P# z?2AbPP#TMKRlA<^z~Ay-psZcBzdv*SS`({GJKn!UlB4E<2_Yp>^ClOvySc7?8DnKz>M z>vI#b`)LJ;tVU9P`Vsu<4w?TI<6{z*>;hy5579mqO6vTg^ybCQRNni~BAuGAkB&dO zy#e|Ro`bJ|)YM>XtWE9#Tf83DLK*J_d2U=YZN>XTr z<|%x@`n{ns%z#pnAvB}@mV{@ptn*wcY^*ijE2JX+v`fKdp@sqrY1OCi-54f)cx#y~ z$-u&9hwLr6FG?@y7ro6>Pr!u#>F|$ddZ>+3c+bPDCD6L`|8#ZTfmHWxpX{7$IzqNX zAuEm@*+ouN2zORg)-j4~+1XSwG81lzLuTAob|)*@J9`vS65i`~-_P^D&-4Cgoca5G zzu)V+KG)~^d_te{vxbswK!dfa!~Jd!3oCjcs9+l|A&Z$MuxDByK3A9uMAd@IWg02) zb=E0j%b;cE0wlHD5$@{|i7U}$Zk`(Jw7Bc?P&R&8X4N*6MWtUumll5PsQ2oTY+oy{ zA<;$k8Pyk(iz2Ow!)9-szCu86U+F7wHr^f5=qvii2eG$ zL3=8gK|PrS3*CBxqovzKn#(HD(Vr zt@^X=!`!J$0aPs>IE85l0rq|(lkMoKU^>z0Ibv^y72F>2)cLGv&c@R%S^X+F z;%C8XsIibA6>pb<5`AE5!bl)9~I=r>J>leRioMiq310CBG|Xj*iCMLK~KKpy)BOsyJxJm`8I9Q z-&Db{{bP9BO4}P`dzRM^rGEy?66e1*p+$ljD!l$W zH2`R9KfzlhQt30?$X7}R(AS`KO8W3EF@W9*8r+>v-A_c>)Y<;rFo7IlG}0S1_jaMo zx7r3pyPFOi{sZH3z%u^^q=T`FcI4dBJ5>_bYR@8OR#EB+Uf9oz;|3sZkF^8pVpM`a zC1z7m{+ptl8wtYE(i(#lVPGQxG)4D- z&$GtL7(?LK(5mji z5Uy^~BoJ(ALe-3E&9?K(>S`G}ell=f(xAincI;{dy*NLj=^WMdFn9XQYTW56KXB}_v{At=z&IdBH!BYczCX#idA#Su~mDY*L`Z600iht_6jw0!@9aD#>yuze+29{kuNoTdn`&? zEPGf>-EiRmnZ?efi`Lcv4^c=R!n(gb@?fwhTQ&w(4Q_N8_KBaC-2&zuLZ(ZoT7+0z zt}+06ESnPmhp`34lCu(=25;vcC7ZtzH|mJ`>uWRZ>$uflHG_fK=a}VMG$Rh!+eSbg z&&6OzpxN;8!6 zSbM^!K>97vo=GVWvR<)L0#TgZ{xqj!V6Fcrv)WIXIcs(tCL5Qh|tu5sD_U zx)suh3nN>8a>KH$!jF<1A^pf#NpMbIcF%a?VcWj}%o#QZ{x46> zpQ!4TzurObDvn5N>N2ssuc137(=mFnp5C7s;vY#1J)L^GQ`08B4FiE1ClxQMT^TYr zn>QmDX%uxOJs35QDEGR1DG=(1bVO%m?0C9&@Kb_YCm`3B(_R=Zv$;67yux~GO>a^) zG%mN}i`PpU&Q?T4Se^~`qTBu?-g0~2rgsVm9%lu_AVw!^r8^p(*}Hc#L@TqbU=P~Q zq8Df|oTMf&I8b@0ytVY5%lqU&dk2NRiF=r=O2WIj3GtAKFg)tOH*YE w$=ImUz zCRaf@QQ^-(TV${VXDvkko7^?-ETo(j?mO-uTgY-CVIcdho+twN9AVB2S# z(Kb&vjnEz_vy3$>EYzj}nDK@~yB-Mwf(kw?`&g5Q#TN}KULzaF41bVo?Ag6^1M}vl zaqlyp-l)LWz%?B`vMt2jh*3FdDhx$HrHcRf!x}4(rQW8MQ5;q5DDK_Vgc7d?mi)yG zHdl*p&PI;di6QpASI)fKV`=PUwT3hTT|i-vEu!O98YZLzHTfkv7uP1gR@x%kJFNfi z0at;5#(jO?KUAA&-T?F%v~7PZuXKx@*N z02zG?BV+329(x5?Jj@wH-@x1i9mCj2TMXSL|81BosTEt~YC z2R(-qL_N%2_1#Zr9``ppFbkbKj(tdCPai9Cw+3Hwd810lUV5)8o)&1q#hNZix8iAE zm=Wp~-c78kKEAKs>D(w$zJqU3dWoLb(wdi$|-5twpH z;Z7P*6OE0cy7m?}v~T&c%4&(nR#rcBsd)b>lv&+&(NJ4o*wM|8p66n$ z_r?L8vwV@XRkcD)vV142EncKB$mlNsEZKGKqWU#`>oJhimNo9Rv1oxVvbA00&gRrq z^fu%bUU^Afy1Rn+4BJD4$XXNX@a@?)452sKw6!E)g+5BcUVhv2R^+q0pdQDa3wdWc z12y!L6r7<4+g*EredgNC$QsI$c$O#qj8E^U^S4NzZ72%)h@Nz%p7Cmv*RYU{O2It1 z)meF&m0e4~TPR_@=5!pX=djyu+L<&l?y+xanAro^INOMX&VDpMsUC}XVm{s7^IHe@ zd}AWWm6Pdc!<0tqcplR?WN`frS?DCN;n!S(!-CZu#d9+si2A)T$H!KI9?QqcPfd+H zE3>r)4^B2d5hEY&tID0yQI&KKe8I5KT#(O0d+Do?)S43i?1gv%q7=bIglc-Y-luBf zo|VnLyoG`~6sL5Jmbti7Crk7u7+{q?oG(ZLm9KK0Sbhyts%xCVPg{7#GNw;@0crI3 zFjE!)>DNnazz1O=Bv-tS>`4X~Q4deSjFl;IJ?~s#KVxMKs8xP>r;2k+_@V`S485cp z0PVm@kg&1(wS|k8Z)H{p}z zY0d`EQrD^plGYTN14wt9xX+MN+2c2VH%wQ5UEzjuiwl&aAWu?rIOGlp~iSM0J{o_G7TVEXFDRe!5U zQVM9#|6Qt{h`vdVw)*}8ES2=>@}|A$74|kIG=ob?F!R}fYT_JJ=ql$M68|Jcy|XW@ z>AH^|p`<@;?%;E*a_KQ^ct|b7zBG`+<-$)g_;#>R&T+LR<6oTUXq=nJFo>VoM?K#M zRCwXF7~FZ9qKn@dg}rA8HeADxJJQc~@z~>bC${DBvvr0(U=V~c< znKB-p`%M{CS@X+>F5QSYqqX>K`!-L;hi|)r;~$BQ?JiU!$8(nOY_+4F9k;`pMtod4;R|7Y<@yH`Yceu{}Vy{E20M}d$AkQYQ3oso}?BWk5lxEk@KF3?B z;cF@cS)rDbF8oz}Mx^+XVzaRcdM>|6Okb8~>~0TF>(MhN{12JUA&K}>O@AguA~23F z`#<%GCoyqEKi{P&#k}+hr^;SX9P6c`@ez9YaG}%}W~oExJ>ILdo`l5A5gZ^>zjFnE z5ER7%J-5pEf^>#QzwRNV`xdwMpipu$g)unV_3chf=68VE;)hkRuK^?+QiOfOybXfH ztD7(;-rE41)3#gDW698A1bBpK?`R0&djNrUFPXJEJ&1?&eQXOF(yzw=uPYvZFh>x7-`C1I2LRllTeksib>~$#i(rJ; z{|31D^EDD5QY87^rtojh?+XJ`k?kiPcdvQMoU9$_-$s=pj;EBoT!-2pyX(sVa@UQp zC_b-W)dvc?sA~qC_8G&65HY_Z)T3>{6uI^|zE~bYNHX$QMSx9HYrqZ;0Xms}us-Vf zhnyxgE;#FoIml`K1pwH7tqAgTKZkiHbOt zgYRwH1cUSCNqhD2`&8Kz6_RKg778~tTbcweO)*A7OvEt(w@6fxgp~4)9|xhIqAzm) zZldC@pjpm^3ge}dd(i=fnpEVb*xJ58c`>vfCXpeb#Ri!2zxm8vO(=AVEECsSKQw5> z^Dr4@u=1xXikP;UvUE1jvY?P5B)HvbJBgPpHr2Q0^tp-iUtZ8Eem1Y7wY*>GWBtaV~ + + + + + + + 2026-10-27T13:12:03.290731 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/restart-after.json b/evidence/restart-after.json new file mode 100644 index 00000000..b8f15252 --- /dev/null +++ b/evidence/restart-after.json @@ -0,0 +1,284 @@ +[ + { + "container": "9d2246f08a86e587ed23ea8ee7bcb63e82fd39dff9cf585e6989ab1820640262", + "name": "/mtusi-lab4-api-1", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:05:36.193553009Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "fdc4019ab865a997e437ed6fb3affefd6f7004fa09287907bd2d142674ffd15d", + "name": "/mtusi-lab4-api-2", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:05:36.14388102Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "c2faba3e2c41441c9a6c17abb0b6b0a2b643dae3c04272965e26ae6014d2da58", + "name": "/mtusi-lab4-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T10:05:35.580074761Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "0acb2abc71272c11cdf6677f4b6deacc61c9fb0bb1b413c56a9e1b19ef5390cc", + "name": "/mtusi-lab4-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T10:05:41.760433323Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18089" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "5f6ad41d3c31bba6b9aad4f317144db9beadd9ddac68fafb91bbe6515a53a89b", + "name": "/mtusi-lab4-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T10:05:35.080716141Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "659282c0c330639947545a49529899f3df40bfa59c6760487efe827dd4e5382c", + "name": "/mtusi-lab4-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.807991429Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab4_postgres_data" + } + ] + }, + { + "container": "0c92c3135de8112ca4f9d39f49c72bcf010bd29317b40a78b75ccf05283c53fa", + "name": "/mtusi-lab4-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.855523719Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "a871486fd792dc6774ac326c0f8acedca47e3a441e34042bed85d2241be43ba7" + } + ] + }, + { + "container": "8dd6d84d8b8c86b9101ba7c40a547ae5422b365543a33bcae493ed3e6740ed8e", + "name": "/mtusi-lab4-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.820013219Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "14783007132b79e82933ac71a5d25b4559f3af285106fb42539e7967bef98ffa" + } + ] + }, + { + "container": "4f266847c0bb7c5c863260d12d7bd838accad03738de7333d366bb8739d2cf19", + "name": "/mtusi-lab4-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:29.140086899Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab4_s3_data" + } + ] + } +] diff --git a/evidence/restart-before.json b/evidence/restart-before.json new file mode 100644 index 00000000..7dd01206 --- /dev/null +++ b/evidence/restart-before.json @@ -0,0 +1,284 @@ +[ + { + "container": "9d2246f08a86e587ed23ea8ee7bcb63e82fd39dff9cf585e6989ab1820640262", + "name": "/mtusi-lab4-api-1", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:04:17.903587814Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "fdc4019ab865a997e437ed6fb3affefd6f7004fa09287907bd2d142674ffd15d", + "name": "/mtusi-lab4-api-2", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:05:21.182107229Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "ac813039081b26304bffbeaf125695bddc4fa32ce4dcc67cfb068bfc2ba39fdd", + "name": "/mtusi-lab4-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T09:58:48.19724977Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "0acb2abc71272c11cdf6677f4b6deacc61c9fb0bb1b413c56a9e1b19ef5390cc", + "name": "/mtusi-lab4-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T09:58:54.973653506Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18089" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "bafb9b9c7acb36922df0ae5172331cef431a8625b52ef566a7192df0e6a44e40", + "name": "/mtusi-lab4-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T09:58:47.706874791Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "659282c0c330639947545a49529899f3df40bfa59c6760487efe827dd4e5382c", + "name": "/mtusi-lab4-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T09:58:40.73770571Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab4_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "0c92c3135de8112ca4f9d39f49c72bcf010bd29317b40a78b75ccf05283c53fa", + "name": "/mtusi-lab4-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T09:59:46.354988447Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "a871486fd792dc6774ac326c0f8acedca47e3a441e34042bed85d2241be43ba7" + } + ] + }, + { + "container": "8dd6d84d8b8c86b9101ba7c40a547ae5422b365543a33bcae493ed3e6740ed8e", + "name": "/mtusi-lab4-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T10:01:37.331049666Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "14783007132b79e82933ac71a5d25b4559f3af285106fb42539e7967bef98ffa" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "4f266847c0bb7c5c863260d12d7bd838accad03738de7333d366bb8739d2cf19", + "name": "/mtusi-lab4-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T09:58:41.842064577Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab4_s3_data" + } + ] + } +] diff --git a/evidence/runtime.json b/evidence/runtime.json new file mode 100644 index 00000000..2dd76c90 --- /dev/null +++ b/evidence/runtime.json @@ -0,0 +1,284 @@ +[ + { + "container": "bf045a5f2da07f277b60fa4218fad624b82fa3f05c3d4870d379eef6ceb5608a", + "name": "/mtusi-lab4-api-1", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:15:42.778343664Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "15c86e71f1c1c806a7b585fc3fc5c499f886408f75bc90defe3355c8377e49e8", + "name": "/mtusi-lab4-api-2", + "service": "api", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T10:15:42.719072973Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "c2faba3e2c41441c9a6c17abb0b6b0a2b643dae3c04272965e26ae6014d2da58", + "name": "/mtusi-lab4-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T10:05:35.580074761Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "0acb2abc71272c11cdf6677f4b6deacc61c9fb0bb1b413c56a9e1b19ef5390cc", + "name": "/mtusi-lab4-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T10:05:41.760433323Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18089" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "5f6ad41d3c31bba6b9aad4f317144db9beadd9ddac68fafb91bbe6515a53a89b", + "name": "/mtusi-lab4-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab4", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T10:05:35.080716141Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "659282c0c330639947545a49529899f3df40bfa59c6760487efe827dd4e5382c", + "name": "/mtusi-lab4-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.807991429Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab4_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "0c92c3135de8112ca4f9d39f49c72bcf010bd29317b40a78b75ccf05283c53fa", + "name": "/mtusi-lab4-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.855523719Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "a871486fd792dc6774ac326c0f8acedca47e3a441e34042bed85d2241be43ba7" + } + ] + }, + { + "container": "8dd6d84d8b8c86b9101ba7c40a547ae5422b365543a33bcae493ed3e6740ed8e", + "name": "/mtusi-lab4-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:27.820013219Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "14783007132b79e82933ac71a5d25b4559f3af285106fb42539e7967bef98ffa" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "4f266847c0bb7c5c863260d12d7bd838accad03738de7333d366bb8739d2cf19", + "name": "/mtusi-lab4-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T10:05:29.140086899Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab4_s3_data" + } + ] + } +] diff --git a/evidence/slow-upload-initial-stderr.txt b/evidence/slow-upload-initial-stderr.txt new file mode 100644 index 00000000..d4282e18 --- /dev/null +++ b/evidence/slow-upload-initial-stderr.txt @@ -0,0 +1,4 @@ +Traceback (most recent call last): + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/slow_upload.py", line 31, in + assert b'X-Instance-Id:' in response and b'X-Request-Id:' in response +AssertionError diff --git a/evidence/slow-upload.json b/evidence/slow-upload.json new file mode 100644 index 00000000..74c82992 --- /dev/null +++ b/evidence/slow-upload.json @@ -0,0 +1 @@ +{"trickle_seconds": 14.862, "http_status": 400, "absolute_deadline_enforced": true} diff --git a/evidence/startup.txt b/evidence/startup.txt new file mode 100644 index 00000000..985af07d --- /dev/null +++ b/evidence/startup.txt @@ -0,0 +1,64 @@ + Container mtusi-lab4-migrate-1 Creating + Container mtusi-lab4-s3-1 Creating + Container mtusi-lab4-migrate-1 Created + Container mtusi-lab4-s3-1 Created + Container mtusi-lab4-bucket-bootstrap-1 Creating + Container mtusi-lab4-bucket-bootstrap-1 Created + Container mtusi-lab4-api-2 Creating + Container mtusi-lab4-api-1 Creating + Container mtusi-lab4-api-2 Created + Container mtusi-lab4-api-1 Created + Container mtusi-lab4-lb-1 Creating + Container mtusi-lab4-lb-1 Created + Container mtusi-lab4-postgres-1 Starting + Container mtusi-lab4-redis-1 Starting + Container mtusi-lab4-s3-store-1 Starting + Container mtusi-lab4-postgres-1 Started + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-s3-store-1 Started + Container mtusi-lab4-s3-store-1 Waiting + Container mtusi-lab4-redis-1 Started + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-s3-store-1 Healthy + Container mtusi-lab4-s3-1 Starting + Container mtusi-lab4-s3-1 Started + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-bucket-bootstrap-1 Starting + Container mtusi-lab4-bucket-bootstrap-1 Started + Container mtusi-lab4-redis-1 Waiting + Container mtusi-lab4-bucket-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-redis-1 Healthy + Container mtusi-lab4-bucket-bootstrap-1 Exited + Container mtusi-lab4-api-2 Starting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-lb-1 Starting + Container mtusi-lab4-lb-1 Started + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-redis-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-s3-1 Waiting + Container mtusi-lab4-lb-1 Waiting + Container mtusi-lab4-bucket-bootstrap-1 Waiting + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-s3-store-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-s3-store-1 Healthy + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-bucket-bootstrap-1 Exited + Container mtusi-lab4-redis-1 Healthy + Container mtusi-lab4-s3-1 Healthy + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-lb-1 Healthy diff --git a/evidence/stopped-instances.jsonl b/evidence/stopped-instances.jsonl new file mode 100644 index 00000000..4e0801af --- /dev/null +++ b/evidence/stopped-instances.jsonl @@ -0,0 +1 @@ +{"instance": "9f224351e7f146f9be39061b5e3250d9", "container": "fdc4019ab865a997e437ed6fb3affefd6f7004fa09287907bd2d142674ffd15d", "running": false, "exit_code": 0, "seconds": 0.634} diff --git a/evidence/storage-security-stderr.txt b/evidence/storage-security-stderr.txt new file mode 100644 index 00000000..64fe620c --- /dev/null +++ b/evidence/storage-security-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab4-audit-run-0bd7cfd77571 Creating + Container mtusi-lab4-audit-run-0bd7cfd77571 Created diff --git a/evidence/storage-security.json b/evidence/storage-security.json new file mode 100644 index 00000000..0905e7b6 --- /dev/null +++ b/evidence/storage-security.json @@ -0,0 +1,14 @@ +{ + "anonymous_statuses": [ + 403, + 403 + ], + "api_credentials_denied": [ + 403, + 403, + 403 + ], + "internal_filer_and_admin_inaccessible": true, + "jpeg_byte_roundtrip": true, + "damaged_jpeg_415": true +} diff --git a/evidence/student-api.txt b/evidence/student-api.txt new file mode 100644 index 00000000..1afc0d90 --- /dev/null +++ b/evidence/student-api.txt @@ -0,0 +1,20 @@ +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok + +---------------------------------------------------------------------- +Ran 15 tests in 3.870s + +OK diff --git a/evidence/student-attachments.txt b/evidence/student-attachments.txt new file mode 100644 index 00000000..2e259411 --- /dev/null +++ b/evidence/student-attachments.txt @@ -0,0 +1,11 @@ +test_access_and_cross_chat (test_attachments.Attachments) ... ok +test_attachment_message_replay (test_attachments.Attachments) ... ok +test_bad_images_and_mime (test_attachments.Attachments) ... ok +test_binary_roundtrip_and_metadata (test_attachments.Attachments) ... ok +test_dangerous_names_and_query (test_attachments.Attachments) ... ok +test_valid_pixel_boundary (test_attachments.Attachments) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 1.497s + +OK diff --git a/evidence/uploads-initial-s3-state.json b/evidence/uploads-initial-s3-state.json new file mode 100644 index 00000000..074d2ec0 --- /dev/null +++ b/evidence/uploads-initial-s3-state.json @@ -0,0 +1 @@ +{"Status":"exited","Running":false,"Paused":false,"Restarting":false,"OOMKilled":true,"Dead":false,"Pid":0,"ExitCode":137,"Error":"","StartedAt":"2026-10-27T09:59:43.487064787Z","FinishedAt":"2026-10-27T10:00:07.995173442Z","Health":{"Status":"unhealthy","FailingStreak":0,"Log":[{"Start":"2026-10-27T12:59:57.792402761+03:00","End":"2026-10-27T12:59:57.867972258+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T12:59:59.869289789+03:00","End":"2026-10-27T12:59:59.912601342+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:00:01.913345235+03:00","End":"2026-10-27T13:00:01.947306663+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:00:03.948017117+03:00","End":"2026-10-27T13:00:03.977837726+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:00:05.979365523+03:00","End":"2026-10-27T13:00:06.022467377+03:00","ExitCode":0,"Output":""}]}} diff --git a/evidence/uploads-initial-stderr.txt b/evidence/uploads-initial-stderr.txt new file mode 100644 index 00000000..214bb3d2 --- /dev/null +++ b/evidence/uploads-initial-stderr.txt @@ -0,0 +1,41 @@ + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting +container mtusi-lab4-api-2 is unhealthy +Traceback (most recent call last): + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 75, in + main() + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 36, in main + docker('compose', 'up', '-d', '--wait', '--no-deps', 'api') + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 18, in docker + return subprocess.check_output(['docker', *args], text=True).strip() + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 424, in check_output + return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 528, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'api']' returned non-zero exit status 1. diff --git a/evidence/uploads-second-api-state.json b/evidence/uploads-second-api-state.json new file mode 100644 index 00000000..873f3ac2 --- /dev/null +++ b/evidence/uploads-second-api-state.json @@ -0,0 +1 @@ +{"Status":"exited","Running":false,"Paused":false,"Restarting":false,"OOMKilled":true,"Dead":false,"Pid":0,"ExitCode":137,"Error":"","StartedAt":"2026-10-27T10:02:01.511397321Z","FinishedAt":"2026-10-27T10:02:10.139299032Z","Health":{"Status":"unhealthy","FailingStreak":0,"Log":[{"Start":"2026-10-27T13:01:37.254367338+03:00","End":"2026-10-27T13:01:37.434073622+03:00","ExitCode":1,"Output":"Traceback (most recent call last):\n File \"\", line 1, in \n File \"/usr/local/lib/python3.12/urllib/request.py\", line 215, in urlopen\n return opener.open(url, data, timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 521, in open\n response = meth(req, response)\n ^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 630, in http_response\n response = self.parent.error(\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 559, in error\n return self._call_chain(*args)\n ^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 492, in _call_chain\n result = func(*args)\n ^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/urllib/request.py\", line 639, in http_error_default\n raise HTTPError(req.full_url, code, msg, hdrs, fp)\nurllib.error.HTTPError: HTTP Error 503: Service Unavailable\n"},{"Start":"2026-10-27T13:01:46.337329806+03:00","End":"2026-10-27T13:01:46.548565827+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:01:55.696703643+03:00","End":"2026-10-27T13:01:55.854363477+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:02:00.855578299+03:00","End":"2026-10-27T13:02:01.017461147+03:00","ExitCode":0,"Output":""},{"Start":"2026-10-27T13:02:06.61049249+03:00","End":"2026-10-27T13:02:06.793434431+03:00","ExitCode":0,"Output":""}]}} diff --git a/evidence/uploads-second-stderr.txt b/evidence/uploads-second-stderr.txt new file mode 100644 index 00000000..7b88cddb --- /dev/null +++ b/evidence/uploads-second-stderr.txt @@ -0,0 +1,70 @@ + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-1 Healthy +Error response from daemon: container f723dc1c71bdaa44acae07bc4e2692a6eff5c1699db47883ac50d4fd85f164df is not running +Exception in thread Thread-3: +Traceback (most recent call last): + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner + self.run() + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run + self._target(*self._args, **self._kwargs) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 45, in sample + samples.append({'seconds': round(time.monotonic() - started, 4), 'memory': {v: memory(v) for v in containers}}) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 45, in + samples.append({'seconds': round(time.monotonic() - started, 4), 'memory': {v: memory(v) for v in containers}}) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 23, in memory + return json.loads(docker('exec', container, 'python', '-c', code)) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 18, in docker + return subprocess.check_output(['docker', *args], text=True).strip() + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 424, in check_output + return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/subprocess.py", line 528, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['docker', 'exec', 'f723dc1c71bdaa44acae07bc4e2692a6eff5c1699db47883ac50d4fd85f164df', 'python', '-c', "import json; from pathlib import Path; p=Path('/sys/fs/cgroup'); print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events']}))"]' returned non-zero exit status 1. +Traceback (most recent call last): + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 75, in + main() + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 63, in main + results = list(executor.map(upload, range(10))) + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 608, in result_iterator + yield fs.pop().result() + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 445, in result + return self.__get_result() + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result + raise self._exception + File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 52, in run + result = self.fn(*self.args, **self.kwargs) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/student-tests/uploads.py", line 53, in upload + response = Client(timeout=30).call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent.png', [201, 503], actor=alice, raw=content, headers={'Content-Type': 'image/png'}) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/tests/client.py", line 132, in call + check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status)) + File "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab4/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: POST /api/v1/chats/d5f3faf9-30c1-4c21-9c11-c90c8f661698/attachments: expected [201, 503], received 502 diff --git a/evidence/uploads-stderr.txt b/evidence/uploads-stderr.txt new file mode 100644 index 00000000..a435ba09 --- /dev/null +++ b/evidence/uploads-stderr.txt @@ -0,0 +1,30 @@ + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy + Container mtusi-lab4-api-2 Restarting + Container mtusi-lab4-api-1 Restarting + Container mtusi-lab4-api-2 Started + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy diff --git a/evidence/uploads.json b/evidence/uploads.json new file mode 100644 index 00000000..5617f61b --- /dev/null +++ b/evidence/uploads.json @@ -0,0 +1,475 @@ +{ + "memory_source": "Linux cgroup v2: memory.current, memory.peak (lifetime peak; API restarted before each case), memory.events; samples include tmpfs and kernel charges, not Python heap alone", + "experiments": [ + { + "width": 420, + "height": 420, + "bytes": 529848, + "pixels": 176400, + "clients": 10, + "before": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "62935040", + "memory.peak": "70307840", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 821121\nuser_usec 772582\nsystem_usec 48539\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "62951424", + "memory.peak": "70447104", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 824208\nuser_usec 761585\nsystem_usec 62623\ncore_sched.force_idle_usec 0\nnr_periods 31\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + } + }, + "after": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "64696320", + "memory.peak": "70307840", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 997847\nuser_usec 923635\nsystem_usec 74211\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "65744896", + "memory.peak": "70447104", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 993358\nuser_usec 904791\nsystem_usec 88566\ncore_sched.force_idle_usec 0\nnr_periods 35\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + } + }, + "samples": [ + { + "seconds": 0.0001, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "66539520", + "memory.peak": "70307840", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 926703\nuser_usec 862307\nsystem_usec 64396\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "65880064", + "memory.peak": "70447104", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 935083\nuser_usec 852368\nsystem_usec 82715\ncore_sched.force_idle_usec 0\nnr_periods 33\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + } + } + } + ], + "results": [ + { + "index": 0, + "status": 201, + "instance": "ec1521f25fc94654a795aa7079670d75", + "elapsed_seconds": 0.141092 + }, + { + "index": 1, + "status": 201, + "instance": "b2ebf9eabec541fabee2a014a635254c", + "elapsed_seconds": 0.147615 + }, + { + "index": 2, + "status": 201, + "instance": "b2ebf9eabec541fabee2a014a635254c", + "elapsed_seconds": 0.137885 + }, + { + "index": 3, + "status": 201, + "instance": "b2ebf9eabec541fabee2a014a635254c", + "elapsed_seconds": 0.135644 + }, + { + "index": 4, + "status": 201, + "instance": "ec1521f25fc94654a795aa7079670d75", + "elapsed_seconds": 0.14879 + }, + { + "index": 5, + "status": 201, + "instance": "b2ebf9eabec541fabee2a014a635254c", + "elapsed_seconds": 0.135081 + }, + { + "index": 6, + "status": 201, + "instance": "b2ebf9eabec541fabee2a014a635254c", + "elapsed_seconds": 0.146117 + }, + { + "index": 7, + "status": 201, + "instance": "ec1521f25fc94654a795aa7079670d75", + "elapsed_seconds": 0.149378 + }, + { + "index": 8, + "status": 201, + "instance": "ec1521f25fc94654a795aa7079670d75", + "elapsed_seconds": 0.137382 + }, + { + "index": 9, + "status": 201, + "instance": "ec1521f25fc94654a795aa7079670d75", + "elapsed_seconds": 0.141443 + } + ], + "seconds": 0.386745 + }, + { + "width": 1800, + "height": 1400, + "bytes": 7563763, + "pixels": 2520000, + "clients": 10, + "before": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "63131648", + "memory.peak": "70373376", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 734874\nuser_usec 692651\nsystem_usec 42222\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "63029248", + "memory.peak": "70451200", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 760554\nuser_usec 710452\nsystem_usec 50102\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + } + }, + "after": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "95686656", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1473238\nuser_usec 1280945\nsystem_usec 192292\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "93851648", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1478013\nuser_usec 1302398\nsystem_usec 175615\ncore_sched.force_idle_usec 0\nnr_periods 55\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + }, + "samples": [ + { + "seconds": 0.0001, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "78561280", + "memory.peak": "79040512", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 843102\nuser_usec 785823\nsystem_usec 57278\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "132190208", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 983686\nuser_usec 860092\nsystem_usec 123593\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 0.3939, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "133455872", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1038539\nuser_usec 903599\nsystem_usec 134940\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "131489792", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1050472\nuser_usec 923848\nsystem_usec 126623\ncore_sched.force_idle_usec 0\nnr_periods 40\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 0.6506, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "118374400", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1111506\nuser_usec 964571\nsystem_usec 146934\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "123994112", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1127383\nuser_usec 991773\nsystem_usec 135609\ncore_sched.force_idle_usec 0\nnr_periods 43\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 0.8914, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "110870528", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1184065\nuser_usec 1026924\nsystem_usec 157141\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "108843008", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1201834\nuser_usec 1054525\nsystem_usec 147308\ncore_sched.force_idle_usec 0\nnr_periods 45\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 1.151, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "103288832", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1261279\nuser_usec 1093241\nsystem_usec 168037\ncore_sched.force_idle_usec 0\nnr_periods 44\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "93851648", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1283831\nuser_usec 1127662\nsystem_usec 156169\ncore_sched.force_idle_usec 0\nnr_periods 48\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 1.4121, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "103329792", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1332057\nuser_usec 1152319\nsystem_usec 179737\ncore_sched.force_idle_usec 0\nnr_periods 46\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "93888512", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1353665\nuser_usec 1189153\nsystem_usec 164511\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 1.6763, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "95698944", + "memory.peak": "133599232", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1412922\nuser_usec 1223799\nsystem_usec 189123\ncore_sched.force_idle_usec 0\nnr_periods 49\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "93876224", + "memory.peak": "133210112", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1419903\nuser_usec 1248255\nsystem_usec 171647\ncore_sched.force_idle_usec 0\nnr_periods 53\nnr_throttled 1\nthrottled_usec 90676\nnr_bursts 0\nburst_usec 0" + } + } + } + ], + "results": [ + { + "index": 0, + "status": 201, + "instance": "c6e48dd0a12748459220da96a8f39c1b", + "elapsed_seconds": 1.604444 + }, + { + "index": 1, + "status": 201, + "instance": "0e06ab6fa85d49b18abd412444127bd4", + "elapsed_seconds": 1.6491 + }, + { + "index": 2, + "status": 201, + "instance": "0e06ab6fa85d49b18abd412444127bd4", + "elapsed_seconds": 1.710992 + }, + { + "index": 3, + "status": 201, + "instance": "c6e48dd0a12748459220da96a8f39c1b", + "elapsed_seconds": 1.733797 + }, + { + "index": 4, + "status": 201, + "instance": "0e06ab6fa85d49b18abd412444127bd4", + "elapsed_seconds": 1.731711 + }, + { + "index": 5, + "status": 201, + "instance": "c6e48dd0a12748459220da96a8f39c1b", + "elapsed_seconds": 1.734976 + }, + { + "index": 6, + "status": 201, + "instance": "c6e48dd0a12748459220da96a8f39c1b", + "elapsed_seconds": 1.743231 + }, + { + "index": 7, + "status": 201, + "instance": "c6e48dd0a12748459220da96a8f39c1b", + "elapsed_seconds": 1.635536 + }, + { + "index": 8, + "status": 201, + "instance": "0e06ab6fa85d49b18abd412444127bd4", + "elapsed_seconds": 1.696447 + }, + { + "index": 9, + "status": 201, + "instance": "0e06ab6fa85d49b18abd412444127bd4", + "elapsed_seconds": 1.712033 + } + ], + "seconds": 2.042275 + }, + { + "width": 10000, + "height": 4000, + "bytes": 533859, + "pixels": 40000000, + "clients": 10, + "before": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "63111168", + "memory.peak": "70426624", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 728470\nuser_usec 673510\nsystem_usec 54960\ncore_sched.force_idle_usec 0\nnr_periods 35\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "64720896", + "memory.peak": "71909376", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 750816\nuser_usec 701763\nsystem_usec 49053\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0" + } + }, + "after": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "182140928", + "memory.peak": "342212608", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1374606\nuser_usec 1178092\nsystem_usec 196514\ncore_sched.force_idle_usec 0\nnr_periods 44\nnr_throttled 5\nthrottled_usec 158446\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "182050816", + "memory.peak": "335065088", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1411374\nuser_usec 1208889\nsystem_usec 202484\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 4\nthrottled_usec 226190\nnr_bursts 0\nburst_usec 0" + } + }, + "samples": [ + { + "seconds": 0.0001, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "151093248", + "memory.peak": "151236608", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 841782\nuser_usec 762614\nsystem_usec 79167\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 1\nthrottled_usec 52104\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "297021440", + "memory.peak": "297283584", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1030597\nuser_usec 900395\nsystem_usec 130201\ncore_sched.force_idle_usec 0\nnr_periods 35\nnr_throttled 1\nthrottled_usec 108977\nnr_bursts 0\nburst_usec 0" + } + } + }, + { + "seconds": 0.3807, + "memory": { + "e9964da9119aeac107c2a641d7bdbbfc948f888369ceb25a03e8628011999365": { + "memory.current": "225230848", + "memory.peak": "342212608", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1266794\nuser_usec 1077023\nsystem_usec 189770\ncore_sched.force_idle_usec 0\nnr_periods 42\nnr_throttled 5\nthrottled_usec 158446\nnr_bursts 0\nburst_usec 0" + }, + "79f0e5e5984ea9aa66a933bfea7ff32784a9ee2b6ff50efaf9d862ddc01162d4": { + "memory.current": "182067200", + "memory.peak": "335065088", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "cpu.stat": "usage_usec 1351217\nuser_usec 1151890\nsystem_usec 199327\ncore_sched.force_idle_usec 0\nnr_periods 39\nnr_throttled 4\nthrottled_usec 226190\nnr_bursts 0\nburst_usec 0" + } + } + } + ], + "results": [ + { + "index": 0, + "status": 201, + "instance": "759bb767d4ad47d186953da4bd2edb23", + "elapsed_seconds": 0.370876 + }, + { + "index": 1, + "status": 201, + "instance": "53156426e9ba49a6a2919f02c96969ac", + "elapsed_seconds": 0.53464 + }, + { + "index": 2, + "status": 201, + "instance": "53156426e9ba49a6a2919f02c96969ac", + "elapsed_seconds": 0.351175 + }, + { + "index": 3, + "status": 201, + "instance": "53156426e9ba49a6a2919f02c96969ac", + "elapsed_seconds": 0.398946 + }, + { + "index": 4, + "status": 201, + "instance": "759bb767d4ad47d186953da4bd2edb23", + "elapsed_seconds": 0.598276 + }, + { + "index": 5, + "status": 201, + "instance": "53156426e9ba49a6a2919f02c96969ac", + "elapsed_seconds": 0.230657 + }, + { + "index": 6, + "status": 201, + "instance": "759bb767d4ad47d186953da4bd2edb23", + "elapsed_seconds": 0.536612 + }, + { + "index": 7, + "status": 201, + "instance": "759bb767d4ad47d186953da4bd2edb23", + "elapsed_seconds": 0.43354 + }, + { + "index": 8, + "status": 201, + "instance": "53156426e9ba49a6a2919f02c96969ac", + "elapsed_seconds": 0.602782 + }, + { + "index": 9, + "status": 201, + "instance": "759bb767d4ad47d186953da4bd2edb23", + "elapsed_seconds": 0.231471 + } + ], + "seconds": 0.831182 + } + ] +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..5495e1e8 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,46 @@ +global + maxconn 1024 + nbthread 1 + hard-stop-after 10s + +defaults + mode http + timeout connect 500ms + timeout check 500ms + timeout queue 1s + timeout client 30s + timeout server 30s + timeout http-request 3s + retries 2 + retry-on conn-failure + option http-server-close + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 2 + server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +listen metrics + bind :8404 + stats enable + stats uri /stats diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..eeac3c14 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,42 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY + , username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$') + , display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100) + , password_hash TEXT NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; +CREATE TABLE chats ( + id UUID PRIMARY KEY + , title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , last_message_at TIMESTAMPTZ +) +; +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) + , user_id UUID NOT NULL REFERENCES users(id) + , PRIMARY KEY (chat_id, user_id) +) +; +CREATE INDEX chat_members_user ON chat_members (user_id, chat_id) +; +CREATE TABLE messages ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , sender_id UUID NOT NULL REFERENCES users(id) + , text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000) + , created_at TIMESTAMPTZ NOT NULL + , FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX messages_history ON messages (chat_id, created_at, id) +; +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY + , user_id UUID NOT NULL REFERENCES users(id) + , expires_at TIMESTAMPTZ NOT NULL +) +; +CREATE INDEX sessions_expiry ON sessions (expires_at) +; diff --git a/migrations/002_unicode_nul.sql b/migrations/002_unicode_nul.sql new file mode 100644 index 00000000..224cda0d --- /dev/null +++ b/migrations/002_unicode_nul.sql @@ -0,0 +1,17 @@ +-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA. +-- API validation keeps the contract's code-point limits; these bounds cap bytes. +ALTER TABLE users + DROP CONSTRAINT users_display_name_check + , ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8') + , ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +; +ALTER TABLE chats + DROP CONSTRAINT chats_title_check + , ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8') + , ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +; +ALTER TABLE messages + DROP CONSTRAINT messages_text_check + , ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8') + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..3ca53a7e --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) + , chat_id UUID NOT NULL REFERENCES chats(id) + , key VARCHAR(128) NOT NULL + , request_hash CHAR(64) NOT NULL + , message_id UUID REFERENCES messages(id) + , expires_at TIMESTAMPTZ NOT NULL + , PRIMARY KEY (user_id, chat_id, key) +) +; +CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at) +; diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..bfd51898 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,30 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_bytes + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000) + , ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}' +; +CREATE TABLE attachments ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , owner_id UUID NOT NULL REFERENCES users(id) + , filename TEXT NOT NULL + , object_key TEXT NOT NULL UNIQUE + , content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')) + , size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760) + , sha256 CHAR(64) NOT NULL + , state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing')) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX attachments_chat ON attachments (chat_id, id) +; +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , PRIMARY KEY (message_id, attachment_id) +) +; +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id) +; diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..5a9a4de1 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Optional: only for rendering figures (application and tests need no matplotlib). +matplotlib==3.11.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..30a631ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +typing-extensions==4.16.0 +boto3==1.40.61 +Pillow==13.0.0 +botocore==1.40.76 +jmespath==1.1.0 +s3transfer==0.14.0 +python-dateutil==2.9.0.post0 +urllib3==2.8.0 +six==1.17.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..c18a2ee1 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(mode=0o700, exist_ok=True) + +def save(name, value): + path = directory / name + if not path.exists(): + path.write_text(value) + path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700. + return path.read_text().strip() + +password = save('postgres_password', secrets.token_hex(24)) +save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger') +save('cache_url', 'redis://redis:6379/0') +identities = [] +for role, actions in { + 'admin': ['Admin', 'Read', 'Write', 'List'], + 'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], + 'reconciler': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], +}.items(): + key = save('s3_' + role + '_key', secrets.token_hex(12)) + secret = save('s3_' + role + '_secret', secrets.token_hex(24)) + identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions}) +save('s3_config', json.dumps({'identities': identities})) +env = root / '.env' +if not env.exists(): + env.write_text((root / '.env.example').read_text()) + env.chmod(0o600) +print('Local configuration ready; no secrets printed.') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..efa8b618 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/snapshot.py > evidence/restart-before.json +docker compose stop api lb +docker compose restart postgres redis s3-store s3 +docker compose up -d --wait postgres redis s3-store s3 +docker compose up -d --wait api lb +python3 scripts/snapshot.py > evidence/restart-after.json diff --git a/scripts/snapshot.py b/scripts/snapshot.py new file mode 100644 index 00000000..3d0354f3 --- /dev/null +++ b/scripts/snapshot.py @@ -0,0 +1,11 @@ +"""Sanitized process and resource evidence; never dump container environments.""" +import json +import subprocess + +ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() +records = [] +for container in ids: + raw = subprocess.check_output(['docker', 'inspect', container], text=True) + data = json.loads(raw)[0] + records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]}) +print(json.dumps(records, indent=2)) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..ce9279f2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/stop_one.py diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..bc495aa5 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,32 @@ +"""Resolve the requested process alias using each actual container, never a fake header.""" +import json +import os +from pathlib import Path +import subprocess +import time + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def instance(container): + code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])" + return docker('exec', container, 'python', '-c', code) + + +if __name__ == '__main__': + target = os.environ['TARGET_INSTANCE_ID'] + started = time.monotonic() + for container in docker('compose', 'ps', '-q', 'api').splitlines(): + if instance(container) == target: + docker('stop', '--time', '8', container) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container)) + assert not state['Running'], 'target must remain stopped' + record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)} + with Path('evidence/stopped-instances.jsonl').open('a') as out: + out.write(json.dumps(record) + '\n') + print(json.dumps(record)) + break + else: + raise SystemExit('Requested live API instance not found') diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..e635caef --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,16 @@ +import os +import time +import urllib.request + +base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087') +for attempt in range(120): + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as response: + if response.status == 200: + print('Ready through load balancer.') + break + except OSError: + pass + time.sleep(0.5) +else: + raise SystemExit('Readiness timeout') diff --git a/src/bucket_bootstrap.py b/src/bucket_bootstrap.py new file mode 100644 index 00000000..a0a2cbd3 --- /dev/null +++ b/src/bucket_bootstrap.py @@ -0,0 +1,13 @@ +"""Idempotent private bucket bootstrap, executed only with administrative S3 keys.""" +from botocore.exceptions import ClientError +from objects import Objects +from postgres import setting + +objects = Objects(setting) +try: + objects.client.create_bucket(Bucket=objects.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'): + raise RuntimeError('Bucket bootstrap failed') from None +objects.ready() +print('Private bucket ready; static authenticated identities loaded from local secret config.') diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..29c87c58 --- /dev/null +++ b/src/common.py @@ -0,0 +1,98 @@ +"""Shared HTTP contract; storage is supplied by each laboratory.""" +import base64 +import datetime as dt +import json +import re +import uuid + + +class APIError(Exception): + def __init__(self, status, code, message): + self.status, self.code, self.message = status, code, message + + +def invalid(): + raise APIError(400, 'invalid_request', 'Неверные параметры запроса') + + +def unauthorized(): + raise APIError(401, 'unauthorized', 'Требуется действительная идентификация') + + +def missing(): + raise APIError(404, 'not_found', 'Объект не найден') + + +def identifier(value): + try: + if not isinstance(value, str): + invalid() + return str(uuid.UUID(value)) + except ValueError: + invalid() + + +def text(value, maximum, minimum=1): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +def fields(value, required): + if not isinstance(value, dict) or set(value) != set(required): + invalid() + + +def username(value): + if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value): + invalid() + return value + + +def now(): + return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def page_args(query, scope): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + invalid() + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]+', limit): + invalid() + try: + limit = int(limit) + except ValueError: + invalid() + if not 1 <= limit <= 100: + invalid() + cursor = None + if 'cursor' in query: + raw = query['cursor'][0] + try: + if not raw or len(raw) > 1024: + invalid() + decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True)) + if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope: + invalid() + stamp = decoded[1] + if not isinstance(stamp, str) or not stamp.endswith('Z'): + invalid() + stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z') + cursor = (stamp, identifier(decoded[2])) + except (ValueError, TypeError, UnicodeError): + invalid() + return limit, cursor + + +def page(items, limit, scope): + more = len(items) > limit + items = items[:limit] + cursor = None + if more: + last = items[-1] + cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=') + return {'items': items, 'next_cursor': cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..ebd85f70 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,13 @@ +"""One-shot migration job; replicas never execute DDL.""" +from pathlib import Path +import psycopg +from postgres import setting + +with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2301003)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)') + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,)) + print('Applied', path.name) diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 00000000..33c6c552 --- /dev/null +++ b/src/objects.py @@ -0,0 +1,156 @@ +"""Bounded S3 operations. No root credentials or filename-derived object keys.""" +import contextlib +import hashlib +import os +import tempfile +import threading +import time +import unicodedata +import warnings + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, PngImagePlugin, UnidentifiedImageError + +from common import APIError, invalid + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 +PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) +UPLOAD_SLOTS = threading.BoundedSemaphore(8) +DECODE_SLOTS = threading.BoundedSemaphore(1) + + +def unavailable(): + raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно') + + +def filename(value): + if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip(): + invalid() + if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +@contextlib.contextmanager +def bounded_upload(): + if not UPLOAD_SLOTS.acquire(blocking=False): + raise APIError(503, 'unavailable', 'Все слоты загрузки заняты') + try: + yield + finally: + UPLOAD_SLOTS.release() + + +@contextlib.contextmanager +def receive(handler): + lengths = handler.headers.get_all('Content-Length', []) + if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Максимум 10 MiB') + if length <= 0: + invalid() + mime = handler.headers.get_content_type() + if mime not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG') + deadline = time.monotonic() + 15 + digest = hashlib.sha256() + # Ephemeral bounded spool is never a durable source of a confirmed attachment. + with tempfile.TemporaryFile(mode='w+b') as source: + remaining = length + while remaining: + handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic()))) + if time.monotonic() >= deadline: + invalid() + try: + chunk = handler.rfile.read1(min(64 * 1024, remaining)) + except (OSError, TimeoutError): + invalid() + if not chunk: + invalid() + digest.update(chunk) + source.write(chunk) + remaining -= len(chunk) + if not DECODE_SLOTS.acquire(timeout=10): + raise APIError(503, 'unavailable', 'Проверка изображений перегружена') + try: + source.flush() + for decode in (False, True): + source.seek(0) + # Pillow.close releases pixel buffers and may close its input file. + # A duplicate descriptor preserves the original for the S3 PUT. + with os.fdopen(os.dup(source.fileno()), 'rb') as image_file: + image = Image.open(image_file, formats=['PNG', 'JPEG']) + try: + if image.width * image.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') + if Image.MIME[image.format] != mime: + raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению') + if decode: + image.load() + else: + image.verify() + finally: + image.close() # Free pixels BEFORE releasing the decoder semaphore. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None + finally: + DECODE_SLOTS.release() + source.seek(0) + yield source, length, mime, digest.hexdigest() + + +class Objects: + def __init__(self, setting): + self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required')) + + def head(self, key): + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'): + return None + unavailable() + except BotoCoreError: + unavailable() + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + except (ClientError, BotoCoreError): + unavailable() + + def put(self, key, source, length, mime, digest): + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest}) + except (ClientError, BotoCoreError): + unavailable() + + def get(self, key): + try: + return self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() + + def delete(self, key): + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() diff --git a/src/postgres.py b/src/postgres.py new file mode 100644 index 00000000..70e5fdef --- /dev/null +++ b/src/postgres.py @@ -0,0 +1,399 @@ +import base64 +import contextlib +import datetime as dt +import hashlib +import hmac +import json +import os +import re +from pathlib import Path +import secrets +import threading +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +import redis + +from objects import Objects, unavailable + +from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username + +KDF_SLOTS = threading.BoundedSemaphore(2) + + +def setting(name): + path = os.environ.get(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def public(row): + return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'} + + +def password_hash(password, salt=None): + salt = salt or secrets.token_bytes(16) + if not KDF_SLOTS.acquire(timeout=1.5): + raise APIError(503, 'unavailable', 'Аутентификация временно перегружена') + try: + digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024) + finally: + KDF_SLOTS.release() + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +class Store: + persistent = True + + def __init__(self): + self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600')) + if not 1 <= self.ttl <= 86400: + raise ValueError('SESSION_TTL_SECONDS must be 1..86400') + self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized') + if self.access_mode not in ('baseline', 'optimized'): + raise ValueError('DB_ACCESS_MODE must be baseline or optimized') + pool_size = int(os.environ.get('DB_POOL_MAX', '8')) + if not 1 <= pool_size <= 32: + raise ValueError('DB_POOL_MAX must be 1..32') + self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True) + self.pool.wait(timeout=10) + self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True) + self.objects = Objects(setting) + self.dummy_hash = password_hash('dummy-password-for-unknown-users') + with self.connection() as conn: + if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '004_attachments.sql'").fetchone(): + raise RuntimeError('Run the separate migration service first') + + @contextlib.contextmanager + def connection(self): + try: + with self.pool.connection() as conn: + yield conn + except psycopg.errors.UniqueViolation: + raise APIError(409, 'conflict', 'Имя пользователя занято') from None + except (psycopg.Error, PoolTimeout): + raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None + + def close(self): + self.pool.close() + self.cache.close() + self.objects.client.close() + + def ready(self): + self.objects.ready() + with self.connection() as conn: + conn.execute('SELECT 1') + + def register(self, body): + fields(body, ('username', 'display_name', 'password')) + name, display = username(body['username']), text(body['display_name'], 100) + # Passwords may contain spaces and colons; only length/UTF-8 are restricted. + password = body['password'] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid() + try: + password.encode('utf-8') + except UnicodeError: + invalid() + hashed = password_hash(password) + with self.connection() as conn: + row = conn.execute(''' + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + ''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone() + return public(row) + + def login(self, headers): + authorization = headers.get('Authorization', '') + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic': + unauthorized() + name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError): + unauthorized() + if not re_username(name) or not 12 <= len(password) <= 128: + unauthorized() + with self.connection() as conn: + row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone() + hashed = row['password_hash'] if row else self.dummy_hash + verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed) + if row is None or not verified: + unauthorized() + token = secrets.token_urlsafe(32) + with self.connection() as conn: + expiry = conn.execute(''' + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second') + RETURNING expires_at + ''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at'] + return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)} + + def token_hash(self, headers): + parts = headers.get('Authorization', '').split(' ') + if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii(): + unauthorized() + return hashlib.sha256(parts[1].encode()).hexdigest() + + def authenticate(self, headers): + token_hash = self.token_hash(headers) + with self.connection() as conn: + session = conn.execute(''' + SELECT user_id + FROM sessions + WHERE 1=1 + AND token_hash = %s + AND expires_at > clock_timestamp() + ''', (token_hash,)).fetchone() + if session is None: + unauthorized() + key = 'user:' + str(session['user_id']) + try: + cached = self.cache.get(key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError): + pass + row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone() + if row is None: + unauthorized() + user = public(row) + try: + self.cache.setex(key, 60, json.dumps(user)) + except redis.RedisError: + pass + return user + + def logout(self, headers): + with self.connection() as conn: + conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),)) + + def chat_in(self, conn, actor, chat_id): + row = conn.execute(''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND c.id = %s + AND cm.user_id = %s + ''', (chat_id, actor['id'])).fetchone() + if row is None: + missing() + members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall() + return dict(public(row), member_ids=[str(v['user_id']) for v in members]) + + def chat(self, actor, chat_id): + with self.connection() as conn: + return self.chat_in(conn, actor, chat_id) + + def create_chat(self, actor, body): + fields(body, ('title', 'member_ids')) + title = text(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + invalid() + members = [identifier(v) for v in members] + if len(set(members)) != len(members) or actor['id'] not in members: + invalid() + with self.connection() as conn: + chat_id = uuid.uuid4() + conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8'))) + # Checking after INSERT exercises rollback of the whole composite operation. + found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall() + if len(found) != len(members): + invalid() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members]) + return self.chat_in(conn, actor, chat_id) + + def list_chats(self, actor, limit, cursor, scope): + with self.connection() as conn: + where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [actor['id']] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT c.id + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND cm.user_id = %s + ''' + where + ''' + ORDER BY c.created_at, c.id + LIMIT %s + ''', params).fetchall() + return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope) + + def require_member(self, conn, actor, chat_id): + if self.access_mode == 'baseline': + return self.chat_in(conn, actor, chat_id) + if not conn.execute(''' + SELECT 1 + FROM chat_members + WHERE 1=1 + AND chat_id = %s + AND user_id = %s + ''', (chat_id, actor['id'])).fetchone(): + missing() + + def send(self, actor, chat_id, body, key): + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body: + invalid() + attachments = body.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [identifier(v) for v in attachments] + if len(set(attachments)) != len(attachments): + invalid() + content = body['text'] + if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()): + invalid() + try: + content.encode('utf-8') + except UnicodeError: + invalid() + canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + # The unique key arbitrates concurrent processes; everything commits together. + claimed = conn.execute(''' + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at) + VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours') + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash + , message_id = NULL + , expires_at = EXCLUDED.expires_at + WHERE 1=1 + AND idempotency_keys.expires_at <= clock_timestamp() + RETURNING request_hash + ''', (actor['id'], chat_id, key, digest)).fetchone() + if claimed is None: + previous = conn.execute(''' + SELECT ik.request_hash + , m.id + , m.chat_id + , m.sender_id + , m.text + , m.created_at + , m.attachment_ids + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE 1=1 + AND ik.user_id = %s + AND ik.chat_id = %s + AND ik.key = %s + ''', (actor['id'], chat_id, key)).fetchone() + if previous['request_hash'] != digest: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + previous.pop('request_hash') + return public(previous) + if attachments: + rows = conn.execute(''' + SELECT id, state + FROM attachments + WHERE 1=1 + AND id = ANY(%s::uuid[]) + AND chat_id = %s + ORDER BY id + FOR SHARE + ''', (attachments, chat_id)).fetchall() + if len(rows) != len(attachments): + missing() + if any(v['state'] != 'ready' for v in rows): + raise APIError(409, 'conflict', 'Вложение ещё не готово') + # Per-chat row lock preserves committed history order across replicas/clocks. + stamp = conn.execute(''' + UPDATE chats + SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond') + WHERE 1=1 + AND id = %s + RETURNING last_message_at + ''', (chat_id,)).fetchone()['last_message_at'] + row = conn.execute(''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (%s, %s, %s, %s, %s, %s::uuid[]) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids + ''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone() + if attachments: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments]) + conn.execute(''' + UPDATE idempotency_keys + SET message_id = %s + WHERE 1=1 + AND user_id = %s + AND chat_id = %s + AND key = %s + ''', (row['id'], actor['id'], chat_id, key)) + return public(row) + + def history(self, actor, chat_id, limit, cursor, scope): + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [chat_id] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE 1=1 + AND chat_id = %s + ''' + where + ''' + ORDER BY created_at, id + LIMIT %s + ''', params).fetchall() + return page([public(v) for v in rows], limit, scope) + + def upload(self, actor, chat_id, name, source, length, mime, digest): + attachment_id = uuid.uuid4() + key = 'originals/' + str(attachment_id) + # Durable intent exists BEFORE any S3 write; a crash always leaves a ledger entry. + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + conn.execute(''' + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading') + ''', (attachment_id, chat_id, actor['id'], name, key, mime, length, digest)) + with self.connection() as conn: + # Reconciler uses the same row lock; no in-flight upload can be collected. + if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None: + unavailable() + self.objects.put(key, source, length, mime, digest) + row = conn.execute(''' + UPDATE attachments + SET state = 'ready' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (attachment_id,)).fetchone() + return self.attachment_public(row) + + def attachment_public(self, row): + result = public({k: v for k, v in row.items() if k not in ('object_key', 'state')}) + return dict(result, status='ready', variants=[], error_code=None) + + def attachment(self, actor, attachment_id): + with self.connection() as conn: + row = conn.execute(''' + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE 1=1 + AND a.id = %s + AND cm.user_id = %s + ''', (attachment_id, actor['id'])).fetchone() + if row is None: + missing() + if row['state'] == 'storage_missing': + unavailable() + if row['state'] != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + return row + + +def re_username(value): + import re + return re.fullmatch(r'[a-z0-9_]{3,32}', value) diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..663e7516 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,129 @@ +"""Conservative two-store reconciler; dry-run by default, no API endpoint.""" +import argparse +import datetime as dt +import hashlib +import json +import re +import time + +import psycopg +from psycopg.rows import dict_row +from botocore.exceptions import BotoCoreError, ClientError + +from objects import Objects +from postgres import setting + + +def reconcile(apply=False, grace=3600): + started = time.monotonic() + # Fixed cutoff for the entire pass: objects born during scanning stay protected. + cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=grace) + objects = Objects(setting) + counts = dict(metadata=0, objects=0, head_calls=0, get_calls=0, deleted_objects=0, deleted_intents=0, marked_missing=0, restored=0, skipped_locked=0, protected_fresh=0, protected_ready=0, protected_foreign_prefix=0) + findings = [] + def note(kind, attachment_id): + if len(findings) < 200: + findings.append({'kind': kind, 'id': str(attachment_id)}) + with psycopg.connect(setting('DATABASE_URL'), autocommit=True, row_factory=dict_row, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=500') as conn: + if not conn.execute('SELECT pg_try_advisory_lock(2301004) AS locked').fetchone()['locked']: + raise RuntimeError('Another reconciler is running') + last = '00000000-0000-0000-0000-000000000000' + while True: + batch = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall() + if not batch: + break + for item in batch: + last = item['id'] + counts['metadata'] += 1 + with conn.transaction(): + row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE SKIP LOCKED', (last,)).fetchone() + if row is None: + counts['skipped_locked'] += 1 + continue + if row['updated_at'] > cutoff: + counts['protected_fresh'] += 1 + continue + head = objects.head(row['object_key']) + counts['head_calls'] += 1 + if row['state'] == 'uploading': + # FK and state locking also prevent deleting anything referenced. + used = conn.execute('SELECT 1 FROM message_attachments WHERE 1=1 AND attachment_id = %s LIMIT 1', (last,)).fetchone() + if used: + note('unexpected_used_intent_retained', last) + continue + note('stale_intent', last) + if apply: + objects.delete(row['object_key']) + counts['deleted_objects'] += int(head is not None) + conn.execute("DELETE FROM attachments WHERE 1=1 AND id = %s AND state = 'uploading'", (last,)) + counts['deleted_intents'] += 1 + elif head is None or head['ContentLength'] != row['size_bytes'] or head.get('Metadata', {}).get('sha256') != row['sha256']: + note('missing_or_mismatched', last) + if apply and row['state'] != 'storage_missing': + conn.execute("UPDATE attachments SET state = 'storage_missing', updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (last,)) + counts['marked_missing'] += 1 + elif row['state'] == 'storage_missing': + # Recovery needs actual bytes, not merely a claimed metadata hash. + response = objects.get(row['object_key']) + counts['get_calls'] += 1 + digest = hashlib.sha256() + size = 0 + with response['Body'] as stream: + for chunk in iter(lambda: stream.read(64 * 1024), b''): + digest.update(chunk) + size += len(chunk) + if size == row['size_bytes'] and digest.hexdigest() == row['sha256']: + note('recoverable', last) + if apply: + conn.execute("UPDATE attachments SET state = 'ready', updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (last,)) + counts['restored'] += 1 + else: + note('corrupt_recovery_retained', last) + else: + counts['protected_ready'] += 1 + # Paginated listing; never collect all keys in memory. + paginator = objects.client.get_paginator('list_objects_v2') + for result in paginator.paginate(Bucket=objects.bucket, Prefix='originals/', PaginationConfig={'PageSize': 100}): + for item in result.get('Contents', []): + counts['objects'] += 1 + key = item['Key'] + if not re.fullmatch(r'originals/[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}', key): + counts['protected_foreign_prefix'] += 1 + continue + if item['LastModified'] > cutoff: + counts['protected_fresh'] += 1 + continue + # A valid upload commits intent BEFORE PUT, and never reuses object keys. + with conn.transaction(): + if conn.execute('SELECT 1 FROM attachments WHERE 1=1 AND object_key = %s', (key,)).fetchone(): + continue + head = objects.head(key) + counts['head_calls'] += 1 + if head is None or head['LastModified'] > cutoff: + continue + note('orphan', key.split('/')[1]) + if apply: + objects.delete(key) + counts['deleted_objects'] += 1 + objects.client.close() + return dict(apply=apply, grace_seconds=grace, elapsed_seconds=round(time.monotonic() - started, 6), counts=counts, findings=findings) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + parser.add_argument('--allow-short-test-grace', action='store_true', help='Only on an isolated synthetic test stand') + args = parser.parse_args() + if args.grace_seconds < (1 if args.allow_short_test_grace else 3600): + parser.error('Production grace must be at least 3600 seconds') + try: + print(json.dumps(reconcile(args.apply, args.grace_seconds), indent=2)) + except Exception: + # Do not serialize library exception details, DSNs, signatures or keys. + print(json.dumps({'error': 'Reconciliation aborted; no fail-open cleanup. Retry after restoring dependencies.'})) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..6df84627 --- /dev/null +++ b/src/server.py @@ -0,0 +1,290 @@ +"""Bounded threaded API; persistent correctness state is shared across replicas.""" +import concurrent.futures +import http.server +import json +import logging +import os +import time +import signal +import socket +import threading +import uuid +from urllib.parse import parse_qs, urlsplit + +from objects import bounded_upload, receive, filename, unavailable + +from common import APIError, identifier, invalid, missing, page_args + +LOG = logging.getLogger('messenger') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + invalid() + result[key] = value + return result + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def setup(self): + super().setup() + self.connection.settimeout(3) + + def send_error(self, code, message=None, explain=None): + # BaseHTTPRequestHandler otherwise emits HTML without contract headers. + self.request_id = uuid.uuid4().hex + self.route = '' + self.close_connection = True + status = 413 if code == 413 else 400 + self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}}) + + def log_message(self, *args): + pass + + def reply(self, status, body): + if status == 200 and isinstance(body, tuple): + return self.download(*body) + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + if status >= 400: + self.close_connection = True + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + if self.close_connection: + self.send_header('Connection', 'close') + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + if status == 401 and self.server.store.persistent: + self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(raw) + + def download(self, row, response, chunk): + stream = response['Body'] + try: + self.send_response(200) + self.send_header('Content-Type', row['content_type']) + self.send_header('Content-Length', str(row['size_bytes'])) + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cache-Control', 'private, no-store') + self.end_headers() + deadline = time.monotonic() + 15 + while chunk: + self.wfile.write(chunk) + if time.monotonic() > deadline: + raise TimeoutError() + chunk = stream.read(64 * 1024) + finally: + stream.close() + + def body(self): + lengths = self.headers.get_all('Content-Length', []) + if len(lengths) != 1 or self.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > 32768: + raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт') + if length <= 0: + invalid() + if self.headers.get_content_type() != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Требуется application/json') + try: + raw = self.rfile.read(length) + if len(raw) != length: + invalid() + return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid()) + except (ValueError, UnicodeError, RecursionError): + invalid() + + def dispatch(self): + self.request_id = uuid.uuid4().hex + self.route = urlsplit(self.path).path + status = 500 + try: + status, result = self.route_request() + except APIError as exc: + status = exc.status + result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}} + except Exception: + # No bodies, credentials, tokens, SQL or exception details in logs/responses. + LOG.error('request_failed request_id=%s', self.request_id) + result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}} + try: + self.reply(status, result) + except Exception: + self.close_connection = True + LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id) + + def route_request(self): + method, path, store = self.command, self.route, self.server.store + if method == 'GET' and path in ('/health/live', '/health/ready'): + if path.endswith('ready'): + store.ready() + return 200, {'status': 'ok'} + if method == 'POST' and path == '/api/v1/users': + return 201, store.register(self.body()) + if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + invalid() + return 201, store.login(self.headers) + actor = store.authenticate(self.headers) + if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.logout(self.headers) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, actor + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + if path == '/api/v1/chats': + if method == 'POST': + return 201, store.create_chat(actor, self.body()) + if method == 'GET': + scope = 'chats:' + actor['id'] + limit, cursor = page_args(query, scope) + return 200, store.list_chats(actor, limit, cursor, scope) + parts = path.split('/') + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET': + row = store.attachment(actor, identifier(parts[4])) + if len(parts) == 5: + if query: + invalid() + return 200, store.attachment_public(row) + if parts[5] == 'content': + if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()): + invalid() + if query.get('variant', ['original'])[0] != 'original': + missing() + response = store.objects.get(row['object_key']) + try: + chunk = response['Body'].read(64 * 1024) + if response['ContentLength'] != row['size_bytes'] or not chunk: + unavailable() + except Exception: + response['Body'].close() + unavailable() + return 200, (row, response, chunk) + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']: + chat_id = identifier(parts[4]) + if store.access_mode == 'baseline': + store.chat(actor, chat_id) + if len(parts) == 5 and method == 'GET': + return 200, store.chat(actor, chat_id) + if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST': + store.chat(actor, chat_id) + if set(query) != {'filename'} or len(query['filename']) != 1: + invalid() + name = filename(query['filename'][0]) + with bounded_upload(): + with receive(self) as (source, length, mime, digest): + return 201, store.upload(actor, chat_id, name, source, length, mime, digest) + if len(parts) == 6 and parts[5] == 'messages': + if method == 'POST': + keys = self.headers.get_all('Idempotency-Key', []) + return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None) + if method == 'GET': + scope = 'messages:' + chat_id + limit, cursor = page_args(query, scope) + return 200, store.history(actor, chat_id, limit, cursor, scope) + missing() + + do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch + + +class Server(http.server.HTTPServer): + request_queue_size = 64 + allow_reuse_address = True + + def __init__(self, address, store): + self.store = store + self.instance_id = uuid.uuid4().hex + workers = int(os.environ.get('HTTP_WORKERS', '16')) + if not 1 <= workers <= 128: + raise ValueError('HTTP_WORKERS must be 1..128') + self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) + self.slots = threading.BoundedSemaphore(workers * 2) + self.connections = set() + self.connections_lock = threading.Lock() + super().__init__(address, Handler) + + def process_request(self, request, address): + if not self.slots.acquire(blocking=False): + rid = uuid.uuid4().hex + raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode() + try: + request.settimeout(0.2) + request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw) + except OSError: + pass + self.shutdown_request(request) + return + with self.connections_lock: + self.connections.add(request) + self.executor.submit(self.worker, request, address) + + def worker(self, request, address): + try: + self.finish_request(request, address) + except OSError: + pass + finally: + self.shutdown_request(request) + with self.connections_lock: + self.connections.discard(request) + self.slots.release() + + def close_connections(self): + with self.connections_lock: + connections = list(self.connections) + for connection in connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def server_close(self): + super().server_close() + # Bound draining even when a slow client keeps an incomplete request alive. + deadline = threading.Timer(5, self.close_connections) + deadline.start() + try: + self.executor.shutdown(wait=True) + finally: + deadline.cancel() + + +def main(): + logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s') + from postgres import Store + store = Store() + server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store) + stopping = threading.Event() + + def stop(signum, frame): + if not stopping.is_set(): + stopping.set() + LOG.info('shutdown_started signal=%s', signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats()))) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + LOG.info('started port=%s', server.server_address[1]) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + store.close() + LOG.info('shutdown_complete') + + +if __name__ == '__main__': + main() diff --git a/student-tests/faults.py b/student-tests/faults.py new file mode 100644 index 00000000..28b9fdb7 --- /dev/null +++ b/student-tests/faults.py @@ -0,0 +1,125 @@ +"""Actual replica stop after upload, interrupted request, and dependency outages.""" +import hashlib +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + + +def docker(*args, code=None): + result = subprocess.run(['docker', *args], input=code, text=True, capture_output=True, check=True) + return result.stdout.strip() + + +def instance(container): + return docker('exec', container, 'python', '-c', "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])") + + +def wait(c): + deadline = time.monotonic() + 60 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + if time.monotonic() > deadline: + raise + time.sleep(.2) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + containers = docker('compose', 'ps', '-q', 'api').splitlines() + assert len(containers) == 2 + report = {'replicas': []} + for container in containers: + alias = instance(container) + for attempt in range(20): + response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') + if response['headers']['X-Instance-Id'] == alias: + break + else: + raise AssertionError('Target uploader never observed') + att = response['json'] + message = c.message(alice, chat, 'file before stop', attachments=[att['id']])['json'] + started = time.monotonic() + try: + docker('stop', '--time', '8', container) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container)) + assert not state['Running'] + while True: + try: + data = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + assert data['headers']['X-Instance-Id'] != alias and data['bytes'] == content + break + except Failure: + assert time.monotonic() - started < 10 + time.sleep(.1) + c.message(alice, chat, 'write survives stop') + report['replicas'].append({'container': container, 'uploader': alias, 'downloader': data['headers']['X-Instance-Id'], 'confirmed_stopped': True, 'recovery_seconds': round(time.monotonic() - started, 3), 'sha256': hashlib.sha256(data['bytes']).hexdigest(), 'exit_code': state['ExitCode'], 'oom_killed': state['OOMKilled']}) + finally: + docker('compose', 'up', '-d', '--wait', '--no-deps', 'api') + wait(c) + # Send only part of a declared upload and terminate the TCP connection. + name = 'interrupted_' + uuid.uuid4().hex + '.png' + target = urlsplit(c.base) + connection = socket.create_connection((target.hostname, target.port), timeout=3) + headers = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=' + name + ' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' + connection.sendall(headers.encode() + content[:30]) + connection.shutdown(socket.SHUT_WR) + connection.close() + time.sleep(.5) + rows = docker('compose', 'exec', '-T', 'api', 'python', '-c', "import sys; sys.path.insert(0,'src'); from postgres import setting; import psycopg; c=psycopg.connect(setting('DATABASE_URL')); print(c.execute('SELECT count(*) FROM attachments WHERE 1=1 AND filename = %s', ('" + name + "',)).fetchone()[0])") + assert rows == '0' + report['interrupted_before_validation_leaves_no_intent'] = True + # Observe API directly inside its container after the LB removes unready backends. + probe_code = """import sys, json, urllib.request, urllib.error +sys.path.insert(0,'src') +from postgres import Store +from common import APIError +s=Store() +try: + s.ready(); ready=200 +except APIError as e: ready=e.status +try: + s.objects.get(%r); content_status=200 +except APIError as e: content_status=e.status +live=urllib.request.urlopen('http://127.0.0.1:8080/health/live').status +print(json.dumps({'ready':ready,'content':content_status,'live':live})) +s.close() +""" % ('originals/' + att['id']) + try: + docker('compose', 'stop', 's3') + result = json.loads(docker('compose', 'exec', '-T', 'api', 'python', '-c', probe_code)) + assert result == {'ready': 503, 'content': 503, 'live': 200} + report['s3_outage'] = result + finally: + docker('compose', 'up', '-d', '--wait', '--no-deps', 's3') + wait(c) + assert c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == content + try: + docker('compose', 'stop', 'redis') + assert c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'] == alice['user'] + c.message(alice, chat, 'Redis fallback') + report['redis_outage_fallback'] = True + finally: + docker('compose', 'up', '-d', '--wait', '--no-deps', 'redis') + wait(c) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 00000000..906356fa --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,24 @@ +"""Small deterministic PNG helpers; generation happens outside measurement.""" +import random +import struct +import zlib + + +def chunk(name, data): + return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff) + + +def png(width, height, noise=False, rgb=True): + channels = 3 if rgb else 1 + rand = random.Random(2301) + encoded = zlib.compressobj(1) + parts = [] + for _ in range(height): + line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels) + parts.append(encoded.compress(b'\0' + line)) + parts.append(encoded.flush()) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(): + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'') diff --git a/student-tests/plot.py b/student-tests/plot.py new file mode 100644 index 00000000..85d0cd7c --- /dev/null +++ b/student-tests/plot.py @@ -0,0 +1,36 @@ +"""Render only measured local evidence; optional matplotlib dependency.""" +import json +from pathlib import Path +import statistics + +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +evidence = ROOT / 'evidence' +plt.rcParams.update({'font.family': 'DejaVu Sans', 'font.size': 10}) +research = json.loads((evidence / 'reconciliation.json').read_text()) +groups = {} +for sample in research['scan_cost']: + groups.setdefault(sample['counts']['metadata'], []).append(sample['elapsed_seconds']) +fig, axes = plt.subplots(1, 2, figsize=(11, 4.2), layout='constrained') +for size, samples in groups.items(): + axes[0].scatter([size] * len(samples), [x * 1000 for x in samples], color='#4e79a7', alpha=.65) +axes[0].plot(sorted(groups), [statistics.median(groups[n]) * 1000 for n in sorted(groups)], color='#4e79a7', label='Медиана 3 прогонов') +axes[0].set(xlabel='Число строк метаданных', ylabel='Полный обход, мс', title='Сверка PostgreSQL + S3') +axes[0].legend() +uploads = json.loads((evidence / 'uploads.json').read_text())['experiments'] +labels = ['0,51 MiB\n0,18 Мп', '7,21 MiB\n2,52 Мп', '0,51 MiB\n40 Мп'] +for i in range(2): + values = [int(list(e['after'].values())[i]['memory.peak']) / 1048576 for e in uploads] + axes[1].bar([n + (i - .5) * .34 for n in range(3)], values, width=.34, label='API ' + str(i + 1)) +axes[1].axhline(512, color='#e15759', linestyle='--', label='Лимит на реплику') +axes[1].set(xticks=range(3), xticklabels=labels, ylabel='cgroup memory.peak, MiB', title='10 одновременных upload + download') +axes[1].legend(fontsize=8) +for ax in axes: + ax.grid(axis='y', alpha=.2) + ax.set_axisbelow(True) +fig.suptitle('ЛР 4 · измерения локального стенда · 27 октября 2026') +fig.savefig(evidence / 'research.png', dpi=160) +fig.savefig(evidence / 'research.svg') diff --git a/student-tests/research.py b/student-tests/research.py new file mode 100644 index 00000000..018f12d3 --- /dev/null +++ b/student-tests/research.py @@ -0,0 +1,123 @@ +"""Track 2 fault injection and measured traversal; use only synthetic lab4 data.""" +import concurrent.futures +import datetime as dt +import hashlib +import io +import json +from pathlib import Path +import sys +import time +import uuid + +import psycopg +from psycopg.rows import dict_row +sys.path[:0] = [str(Path.cwd() / 'tests'), '/app/src'] +from client import Client +from objects import Objects +from postgres import setting, Store +from reconcile import reconcile + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + good, content = c.attachment(alice, chat) + message = c.message(alice, chat, '', attachments=[good['id']])['json'] + sha = hashlib.sha256(content).hexdigest() + objects = Objects(setting) + report = {'method': 'real S3 and PostgreSQL, per-run fixed cutoff, synthetic data, 2s test grace; production >=3600s', 'checks': {}, 'passes': []} + with psycopg.connect(setting('DATABASE_URL'), autocommit=True, row_factory=dict_row) as conn: + def row(attachment_id): + return conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s', (attachment_id,)).fetchone() + def aged(attachment_id): + conn.execute("UPDATE attachments SET updated_at = clock_timestamp() - INTERVAL '2 hours' WHERE 1=1 AND id = %s", (attachment_id,)) + def intent(state='uploading'): + aid = uuid.uuid4() + conn.execute(''' + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, state, updated_at) + VALUES (%s, %s, %s, 'research.png', %s, 'image/png', %s, %s, %s, clock_timestamp() - INTERVAL '2 hours') + ''', (aid, chat['id'], alice['id'], 'originals/' + str(aid), len(content), sha, state)) + return str(aid) + # Fail precisely after successful S3 PUT, before the ready UPDATE/COMMIT. + broken_name = 'fault_' + uuid.uuid4().hex + '.png' + trigger = 'research_' + uuid.uuid4().hex + try: + from psycopg import sql + conn.execute(sql.SQL('''CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.state = 'ready' AND NEW.filename = '{}' THEN + RAISE EXCEPTION 'synthetic research failure'; + END IF; + RETURN NEW; + END $$''').format(sql.Identifier(trigger), sql.SQL(broken_name))) + conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger), sql.Identifier(trigger))) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + broken_name, 503, actor=alice, raw=content, headers={'Content-Type': 'image/png'}) + failed = conn.execute('SELECT id, state, object_key FROM attachments WHERE 1=1 AND filename = %s', (broken_name,)).fetchone() + assert failed['state'] == 'uploading' and objects.head(failed['object_key']) is not None + aged(failed['id']) + finally: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(trigger))) + absent = intent() # DB intent exists, S3 PUT never happened (process died first). + missing, _ = c.attachment(alice, chat) + c.message(alice, chat, 'keep missing reference', attachments=[missing['id']]) + missing_key = 'originals/' + missing['id'] + objects.delete(missing_key) # S3-side loss; referenced metadata must never be erased. + aged(missing['id']) + orphan_key = 'originals/' + str(uuid.uuid4()) + objects.put(orphan_key, io.BytesIO(content), len(content), 'image/png', sha) # no DB row + aged(good['id']) + # Wait real object age; don't forge LastModified or replace the S3 clock. + time.sleep(3) + fresh_key = 'originals/' + str(uuid.uuid4()) + objects.put(fresh_key, io.BytesIO(content), len(content), 'image/png', sha) + fresh_intent = intent() + conn.execute('UPDATE attachments SET updated_at = clock_timestamp() WHERE 1=1 AND id = %s', (fresh_intent,)) + locked = intent() + # Hold the same lock as a live uploader while reconciliation runs. + with psycopg.connect(setting('DATABASE_URL')) as uploading: + uploading.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (locked,)) + dry = reconcile(False, 2) + assert row(failed['id']) and row(absent) and row(missing['id'])['state'] == 'ready' + assert objects.head(orphan_key) is not None + report['checks']['dry_run_mutates_nothing'] = True + fixed = reconcile(True, 2) + assert row(failed['id']) is None and row(absent) is None + assert objects.head(failed['object_key']) is None and objects.head(orphan_key) is None + assert row(missing['id'])['state'] == 'storage_missing' + assert row(locked) is not None and fixed['counts']['skipped_locked'] >= 1 + assert row(fresh_intent) is not None and objects.head(fresh_key) is not None + assert row(good['id'])['state'] == 'ready' and objects.head('originals/' + good['id']) is not None + report['checks'].update(stale_intents_and_orphans_cleaned=True, fresh_and_locked_retained=True, used_ready_retained=True, missing_reference_retained=True) + repeat = reconcile(True, 2) + assert repeat['counts']['deleted_objects'] == 0 and repeat['counts']['marked_missing'] == 0 + report['checks']['apply_repeat_idempotent'] = True + c.call('GET', '/api/v1/attachments/' + missing['id'] + '/content', 503, actor=bob) + # Restore exact original from the test's external copy; reconciler can't invent lost bytes. + objects.put(missing_key, io.BytesIO(content), len(content), 'image/png', sha) + aged(missing['id']) + recovered = reconcile(True, 2) + assert row(missing['id'])['state'] == 'ready' + assert c.call('GET', '/api/v1/attachments/' + missing['id'] + '/content', actor=bob)['bytes'] == content + report['checks']['restore_after_sha256_verification'] = True + # Verify metadata FK and existing message survive every repair. + assert conn.execute('SELECT 1 FROM message_attachments WHERE 1=1 AND message_id = %s AND attachment_id = %s', (message['id'], good['id'])).fetchone() + report['passes'] = [dict(name=n, **v) for n, v in [('dry_run', dry), ('apply', fixed), ('repeat', repeat), ('recover', recovered)]] + # Clean up only new unattached diagnostic keys/intents; confirmed files remain. + objects.delete(fresh_key) + for aid in [fresh_intent, locked]: + conn.execute("DELETE FROM attachments WHERE 1=1 AND id = %s AND state = 'uploading'", (aid,)) + # Measure bounded scans at three populations; 3 repeats, no writes during scan. + report['scan_cost'] = [] + for extra in [0, 100, 400]: + for _ in range(extra): + aid = intent('ready') + objects.put('originals/' + aid, io.BytesIO(content), len(content), 'image/png', sha) + for repetition in range(3): + scan = reconcile(False, 2) + report['scan_cost'].append({'repetition': repetition + 1, **scan}) + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/slow_upload.py b/student-tests/slow_upload.py new file mode 100644 index 00000000..c17d2b01 --- /dev/null +++ b/student-tests/slow_upload.py @@ -0,0 +1,32 @@ +"""A continuously trickling client still reaches the absolute upload deadline.""" +import json +from pathlib import Path +import select +import socket +import sys +import time +import uuid +from urllib.parse import urlsplit +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +c = Client() +alice, bob, eve, chat = c.bootstrap() +target = urlsplit(c.base) +connection = socket.create_connection((target.hostname, target.port), timeout=3) +header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' +started = time.monotonic() +connection.sendall(header.encode()) +response = b'' +while time.monotonic() - started < 20: + if select.select([connection], [], [], .15)[0]: + response += connection.recv(4096) + if b'\r\n\r\n' in response: + break + connection.sendall(b'x') +connection.close() +elapsed = time.monotonic() - started +assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100] +assert 14 <= elapsed < 19 +assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower() +print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True})) diff --git a/student-tests/storage_checks.py b/student-tests/storage_checks.py new file mode 100644 index 00000000..7e100d87 --- /dev/null +++ b/student-tests/storage_checks.py @@ -0,0 +1,69 @@ +"""Execute with the isolated tools profile; inject faults only into new test IDs.""" +import hashlib +import io +import json +from pathlib import Path +import socket +import sys +import urllib.error +import urllib.request +import uuid + +sys.path[:0] = [str(Path.cwd() / 'tests'), '/app/src'] +from botocore.exceptions import ClientError +from PIL import Image +from client import Client +from objects import Objects +from postgres import setting + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + att, content = c.attachment(alice, chat) + api = Objects(setting) + admin = Objects(lambda key: Path('/run/secrets/s3_admin_' + ('key' if key.endswith('_ID') else 'secret')).read_text().strip()) + key = 'originals/' + att['id'] + assert admin.head(key)['ContentLength'] == len(content) + denied = [] + for path in ('/' + api.bucket + '/' + key, '/' + api.bucket + '?list-type=2'): + try: + urllib.request.urlopen('http://s3:8333' + path, timeout=3) + raise AssertionError('Anonymous S3 access allowed') + except urllib.error.HTTPError as error: + assert error.code == 403 + denied.append(error.code) + # API credentials cannot create a bucket, read another bucket or change policy. + other = 'test-' + uuid.uuid4().hex + admin.client.create_bucket(Bucket=other) + rejected = [] + try: + actions = [lambda: api.client.create_bucket(Bucket='forbidden-' + uuid.uuid4().hex), lambda: api.client.list_objects_v2(Bucket=other), lambda: api.client.put_bucket_policy(Bucket=api.bucket, Policy=json.dumps({'Version': '2012-10-17', 'Statement': []}))] + for action in actions: + try: + action() + raise AssertionError('API credentials have excessive permissions') + except ClientError as error: + assert error.response['ResponseMetadata']['HTTPStatusCode'] == 403 + rejected.append(403) + finally: + admin.client.delete_bucket(Bucket=other) + for host, port in [('s3-store', 8888), ('s3', 12535)]: + try: + connection = socket.create_connection((host, port), timeout=1) + connection.close() + raise AssertionError('Administrative/internal endpoint is exposed') + except OSError: + pass + image = io.BytesIO() + Image.new('RGB', (320, 200), (20, 150, 210)).save(image, format='JPEG') + jpg = image.getvalue() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=photo.jpg', 201, actor=alice, raw=jpg, headers={'Content-Type': 'image/jpeg'}, schema='Attachment')['json'] + downloaded = c.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob) + assert downloaded['bytes'] == jpg and uploaded['sha256'] == hashlib.sha256(jpg).hexdigest() + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bad.jpg', 415, actor=alice, raw=jpg[:200], headers={'Content-Type': 'image/jpeg'}) + print(json.dumps({'anonymous_statuses': denied, 'api_credentials_denied': rejected, 'internal_filer_and_admin_inaccessible': True, 'jpeg_byte_roundtrip': True, 'damaged_jpeg_415': True}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..45f0d800 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,177 @@ +"""Extra black-box tests, without changing the course's public tests.""" +import concurrent.futures +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client as PublicClient, LAB + + +class Client(PublicClient): + def call(self, method, path, *args, headers=None, **kwargs): + headers = dict(headers or {}) + if method == 'POST' and path.endswith('/messages'): + headers.setdefault('Idempotency-Key', uuid.uuid4().hex) + return super().call(method, path, *args, headers=headers, **kwargs) + + +class APIContract(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_malformed_json_and_extra_fields(self): + for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body) + deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}' + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'}) + + def test_unicode_limits(self): + result = self.client.message(self.alice, self.chat, '👩' * 4000)['json'] + self.assertEqual(result['text'], '👩' * 4000) + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001}) + + def test_unicode_nul_round_trip(self): + import base64 + value = 'A\x00B\\u0000👩' + registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value) + user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json'] + self.assertEqual(user['display_name'], value) + credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(session['user'], user) + actor = {'id': user['id'], 'token': session['token']} + # Exercise the database path and then the Redis profile cache. + for _ in range(2): + self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user) + chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json'] + self.assertEqual(chat['title'], value) + self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat) + contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00'] + for content in contents: + self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content) + history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json'] + self.assertEqual([message['text'] for message in history['items']], contents) + + def test_foreign_chat(self): + for suffix in ('', '/messages'): + self.client.call('GET', self.path + suffix, 404, actor=self.eve) + self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'}) + self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice) + + def test_registration_race(self): + body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20]) + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + statuses = list(executor.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + def test_invalid_chat_creation(self): + members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True]) + for member_ids in members: + self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids}) + + def test_cursor_scope_and_validation(self): + for i in range(3): + self.client.message(self.alice, self.chat, str(i)) + cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice) + for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'): + self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice) + + def test_concurrent_messages_and_pagination(self): + def send(i): + return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + sent = list(executor.map(send, range(100))) + found = [] + cursor = None + while True: + url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '') + page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json'] + found.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(len(found), 100) + self.assertEqual(len({v['id'] for v in found}), 100) + self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent}) + self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id']))) + + def test_idempotency_concurrent_replicas(self): + key = uuid.uuid4().hex + def send(_): + return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2) + self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies)) + page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(page['items'], [replies[0]['json']]) + self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice) + self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key}) + + def test_protocol_error_headers(self): + self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice) + + def test_idempotency_scope(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'same', key=key)['json'] + two = self.client.message(self.bob, self.chat, 'same', key=key)['json'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + three = self.client.message(self.alice, other, 'same', key=key)['json'] + self.assertEqual(len({one['id'], two['id'], three['id']}), 3) + + def test_idempotency_semantic_body(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json'] + two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + self.assertEqual(one, two) + + def test_idempotency_invalid_keys(self): + for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'): + headers = {} if key is None else {'Idempotency-Key': key} + PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers) + for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + + def test_idempotency_conflicting_race(self): + key = uuid.uuid4().hex + def send(i): + return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertEqual(sum(v['status'] == 201 for v in replies), 16) + self.assertEqual(sum(v['status'] == 409 for v in replies), 16) + page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(len(page['items']), 1) + + @unittest.skipUnless(LAB >= 2, 'sessions start in lab2') + def test_unicode_password_basic_and_logout(self): + import base64 + registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'} + self.client.call('POST', '/api/v1/users', 201, body=registration) + encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + headers = {'Authorization': 'Basic ' + encoded} + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json'] + actor = {'id': session['user']['id'], 'token': session['token']} + self.client.call('GET', '/api/v1/users/me', actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, headers=headers) + for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'): + self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..6cd6d017 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,83 @@ +import concurrent.futures +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from images import bomb, png + + +class Attachments(unittest.TestCase): + def setUp(self): + self.c = Client() + self.alice, self.bob, self.eve, self.chat = self.c.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + self.image = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + + def upload(self, content=None, name='photo.png', mime='image/png', status=201, actor=None): + return self.c.call('POST', self.path + '/attachments?filename=' + quote(name, safe=''), status, actor=actor or self.alice, raw=self.image if content is None else content, headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None) + + def test_binary_roundtrip_and_metadata(self): + att = self.upload(name='Софья 👩‍💻.png')['json'] + self.assertEqual(att['filename'], 'Софья 👩‍💻.png') + self.assertEqual(att['variants'], []) + self.assertIsNone(att['error_code']) + for _ in range(4): + metadata = self.c.call('GET', '/api/v1/attachments/' + att['id'], actor=self.bob, schema='Attachment')['json'] + self.assertEqual(metadata, att) + response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=original', actor=self.bob) + self.assertEqual(response['bytes'], self.image) + self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=self.bob) + + def test_bad_images_and_mime(self): + for raw in (b'not image', b'', self.image[:40], self.image[:-30]): + self.upload(raw, status=415) + self.upload(mime='image/jpeg', status=415) + self.upload(mime='application/octet-stream', status=415) + self.upload(bomb(), status=413) + self.upload(b'x' * (10 * 1024 * 1024 + 1), status=413) + + def test_dangerous_names_and_query(self): + for name in ('', ' ', '../a.png', 'a/b.png', 'a\\b.png', '\0.png', 'a\n.png', 'a\x7f.png', 'a' * 256): + self.upload(name=name, status=400) + for query in ('', '?filename=a&filename=b', '?filename=a&extra=x'): + self.c.call('POST', self.path + '/attachments' + query, 400, actor=self.alice, raw=self.image, headers={'Content-Type': 'image/png'}) + + def test_access_and_cross_chat(self): + att = self.upload()['json'] + for suffix in ('', '/content'): + self.c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=self.eve) + self.c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401) + self.upload(actor=self.eve, status=404) + other = self.c.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + self.c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=self.alice, body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex}) + self.c.call('POST', self.path + '/messages', 404, actor=self.alice, body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}, headers={'Idempotency-Key': uuid.uuid4().hex}) + + def test_attachment_message_replay(self): + one, two = self.upload()['json'], self.upload()['json'] + key = uuid.uuid4().hex + def send(_): + return Client().message(self.bob, self.chat, '', key=key, attachments=[one['id'], two['id']])['json'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + replies = list(executor.map(send, range(20))) + self.assertTrue(all(r == replies[0] for r in replies)) + self.assertEqual(replies[0]['attachment_ids'], [one['id'], two['id']]) + self.c.call('POST', self.path + '/messages', 409, actor=self.bob, body={'text': '', 'attachment_ids': [two['id'], one['id']]}, headers={'Idempotency-Key': key}) + history = self.c.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(history['items'], [replies[0]]) + for ids in ([one['id']] * 2, [True], 'bad', [str(uuid.uuid4()) for _ in range(11)]): + self.c.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) + + def test_valid_pixel_boundary(self): + # Exactly 40M decoded grayscale pixels, small compressed upload. + content = png(10000, 4000, rgb=False) + att = self.upload(content)['json'] + response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=self.bob) + self.assertEqual(response['bytes'], content) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/uploads.py b/student-tests/uploads.py new file mode 100644 index 00000000..c664743a --- /dev/null +++ b/student-tests/uploads.py @@ -0,0 +1,75 @@ +""">=10 simultaneous uploads with sampled cgroup memory and true cgroup peak.""" +import concurrent.futures +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client +from images import png + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def memory(container): + code = "import json; from pathlib import Path; p=Path('/sys/fs/cgroup'); print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','cpu.stat']}))" + return json.loads(docker('exec', container, 'python', '-c', code)) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + records = [] + for width, height, noise, rgb in [(420, 420, True, True), (1800, 1400, True, True), (10000, 4000, False, True)]: + content = png(width, height, noise=noise, rgb=rgb) + assert len(content) <= 10 * 1024 * 1024 + digest = hashlib.sha256(content).hexdigest() + # Fresh API processes make memory.peak attributable to this one experiment. + docker('compose', 'restart', 'api') + docker('compose', 'up', '-d', '--wait', '--no-deps', 'api') + time.sleep(2) + containers = docker('compose', 'ps', '-q', 'api').splitlines() + before = {v: memory(v) for v in containers} + samples = [] + done = threading.Event() + started = time.monotonic() + def sample(): + while not done.is_set(): + samples.append({'seconds': round(time.monotonic() - started, 4), 'memory': {v: memory(v) for v in containers}}) + done.wait(.05) + thread = threading.Thread(target=sample) + thread.start() + barrier = threading.Barrier(10) + def upload(i): + barrier.wait() + begin = time.monotonic() + response = Client(timeout=30).call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent.png', [201, 503], actor=alice, raw=content, headers={'Content-Type': 'image/png'}) + if response['status'] == 201: + # Parse because schema omitted to allow a documented admission rejection. + attachment = json.loads(response['bytes']) + assert attachment['sha256'] == digest + downloaded = Client(timeout=30).call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob) + assert hashlib.sha256(downloaded['bytes']).hexdigest() == digest + return {'index': i, 'status': response['status'], 'instance': response['headers']['X-Instance-Id'], 'elapsed_seconds': round(time.monotonic() - begin, 6)} + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + results = list(executor.map(upload, range(10))) + finally: + done.set() + thread.join() + after = {v: memory(v) for v in containers} + for m in after.values(): + assert 'oom_kill 0' in m['memory.events'] + records.append({'width': width, 'height': height, 'bytes': len(content), 'pixels': width * height, 'clients': 10, 'before': before, 'after': after, 'samples': samples, 'results': results, 'seconds': round(time.monotonic() - started, 6)}) + print(json.dumps({'memory_source': 'Linux cgroup v2: memory.current, memory.peak (lifetime peak; API restarted before each case), memory.events; samples include tmpfs and kernel charges, not Python heap alone', 'experiments': records}, indent=2)) + + +if __name__ == '__main__': + main() -- 2.54.0 From 2c7a00948a731dac752d9c0de2ca54584e4ae4d6 Mon Sep 17 00:00:00 2001 From: mayorovasonya <112+mayorovasonya@noreply.gitea.privcloud.tech> Date: Tue, 27 Oct 2026 13:19:00 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=9B=D0=A0=204:=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D1=82=D0=B2=D0=B5=D1=80=D0=B4=D0=B8=D1=82=D1=8C=20=D0=BE=D0=BA?= =?UTF-8?q?=D0=BE=D0=BD=D1=87=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B?= =?UTF-8?q?=D0=B5=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE=D0=B2=20=D0=B8=20=D1=86?= =?UTF-8?q?=D0=B5=D0=BB=D0=BE=D1=81=D1=82=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20?= =?UTF-8?q?=D0=B4=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C?= =?UTF-8?q?=D1=81=D1=82=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ts-stderr.txt => faults-second-stderr.txt} | 0 evidence/faults.json | 33 ++++++++++++++++ evidence/offline-final.txt | 2 + evidence/runtime.json | 38 +++++++++---------- student-tests/faults.py | 13 ++++++- 5 files changed, 65 insertions(+), 21 deletions(-) rename evidence/{faults-stderr.txt => faults-second-stderr.txt} (100%) create mode 100644 evidence/offline-final.txt diff --git a/evidence/faults-stderr.txt b/evidence/faults-second-stderr.txt similarity index 100% rename from evidence/faults-stderr.txt rename to evidence/faults-second-stderr.txt diff --git a/evidence/faults.json b/evidence/faults.json index e69de29b..e2406322 100644 --- a/evidence/faults.json +++ b/evidence/faults.json @@ -0,0 +1,33 @@ +{ + "replicas": [ + { + "container": "bf045a5f2da07f277b60fa4218fad624b82fa3f05c3d4870d379eef6ceb5608a", + "uploader": "410dddf690ef439bbaf15193aa235099", + "downloader": "9e726cd692f7474f856feaf1ebea5577", + "confirmed_stopped": true, + "transient_write_failures": 0, + "recovery_seconds": 1.818, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "exit_code": 0, + "oom_killed": false + }, + { + "container": "15c86e71f1c1c806a7b585fc3fc5c499f886408f75bc90defe3355c8377e49e8", + "uploader": "9e726cd692f7474f856feaf1ebea5577", + "downloader": "882125d9d2834b7aab25eb78c9f99a5e", + "confirmed_stopped": true, + "transient_write_failures": 0, + "recovery_seconds": 1.846, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "exit_code": 0, + "oom_killed": false + } + ], + "interrupted_before_validation_leaves_no_intent": true, + "s3_outage": { + "ready": 503, + "content": 503, + "live": 200 + }, + "redis_outage_fallback": true +} diff --git a/evidence/offline-final.txt b/evidence/offline-final.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/offline-final.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/runtime.json b/evidence/runtime.json index 2dd76c90..826f4ef2 100644 --- a/evidence/runtime.json +++ b/evidence/runtime.json @@ -6,7 +6,7 @@ "image": "mtusi-messenger:lab4", "user": "10001:10001", "running": true, - "started_at": "2026-10-27T10:15:42.778343664Z", + "started_at": "2026-10-27T10:18:07.132235089Z", "exit_code": 0, "oom_killed": false, "memory_limit": 536870912, @@ -15,11 +15,6 @@ "8080/tcp": null }, "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/s3_api_key", - "name": null - }, { "type": "bind", "destination": "/run/secrets/s3_api_secret", @@ -34,6 +29,11 @@ "type": "bind", "destination": "/run/secrets/cache_url", "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null } ] }, @@ -44,7 +44,7 @@ "image": "mtusi-messenger:lab4", "user": "10001:10001", "running": true, - "started_at": "2026-10-27T10:15:42.719072973Z", + "started_at": "2026-10-27T10:18:14.915499757Z", "exit_code": 0, "oom_killed": false, "memory_limit": 536870912, @@ -143,11 +143,6 @@ "nano_cpus": 1000000000, "ports": {}, "mounts": [ - { - "type": "bind", - "destination": "/run/secrets/database_url", - "name": null - }, { "type": "bind", "destination": "/run/secrets/cache_url", @@ -162,6 +157,11 @@ "type": "bind", "destination": "/run/secrets/s3_api_secret", "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null } ] }, @@ -200,7 +200,7 @@ "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", "user": "", "running": true, - "started_at": "2026-10-27T10:05:27.855523719Z", + "started_at": "2026-10-27T10:18:25.256591097Z", "exit_code": 0, "oom_killed": false, "memory_limit": 67108864, @@ -223,7 +223,7 @@ "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", "user": "", "running": true, - "started_at": "2026-10-27T10:05:27.820013219Z", + "started_at": "2026-10-27T10:18:22.341659035Z", "exit_code": 0, "oom_killed": false, "memory_limit": 536870912, @@ -239,15 +239,15 @@ "9333/tcp": null }, "mounts": [ - { - "type": "volume", - "destination": "/data", - "name": "14783007132b79e82933ac71a5d25b4559f3af285106fb42539e7967bef98ffa" - }, { "type": "bind", "destination": "/run/secrets/s3_config", "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "14783007132b79e82933ac71a5d25b4559f3af285106fb42539e7967bef98ffa" } ] }, diff --git a/student-tests/faults.py b/student-tests/faults.py index 28b9fdb7..09afb8bc 100644 --- a/student-tests/faults.py +++ b/student-tests/faults.py @@ -68,8 +68,17 @@ def main(): except Failure: assert time.monotonic() - started < 10 time.sleep(.1) - c.message(alice, chat, 'write survives stop') - report['replicas'].append({'container': container, 'uploader': alias, 'downloader': data['headers']['X-Instance-Id'], 'confirmed_stopped': True, 'recovery_seconds': round(time.monotonic() - started, 3), 'sha256': hashlib.sha256(data['bytes']).hexdigest(), 'exit_code': state['ExitCode'], 'oom_killed': state['OOMKilled']}) + key = uuid.uuid4().hex + write_failures = 0 + while True: + try: + c.message(alice, chat, 'write survives stop', key=key) + break + except Failure: + write_failures += 1 + assert time.monotonic() - started < 10 + time.sleep(.1) + report['replicas'].append({'container': container, 'uploader': alias, 'downloader': data['headers']['X-Instance-Id'], 'confirmed_stopped': True, 'transient_write_failures': write_failures, 'recovery_seconds': round(time.monotonic() - started, 3), 'sha256': hashlib.sha256(data['bytes']).hexdigest(), 'exit_code': state['ExitCode'], 'oom_killed': state['OOMKilled']}) finally: docker('compose', 'up', '-d', '--wait', '--no-deps', 'api') wait(c) -- 2.54.0