From 55a41e17c95a01eee9dc073498402e448c05cb83 Mon Sep 17 00:00:00 2001 From: mayorovasonya <112+mayorovasonya@noreply.gitea.privcloud.tech> Date: Tue, 27 Oct 2026 18:20:39 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=205:=20outbox,=20retry=20=D0=B8=20?= =?UTF-8?q?versioned=20pipeline=20=E2=80=94=20=D0=9C=D0=B0=D0=B9=D0=BE?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=20=D0=91=D0=92=D0=A22301?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 + .env.example | 33 +- .github/workflows/application.yml | 26 + Dockerfile | 13 + Makefile | 5 + REPORT.md | 219 ++- compose.yaml | 234 +++ evidence/build.txt | 55 + evidence/clean-clone.txt | 1054 +++++++++++++ evidence/durability-stderr.txt | 27 + evidence/durability.json | 766 ++++++++++ evidence/failover.txt | 4 + evidence/final-tests.txt | 28 + evidence/jobs-summary.json | 73 + evidence/openapi.txt | 1 + evidence/persistence.txt | 3 + evidence/pipeline-stderr.txt | 2 + evidence/pipeline.json | 119 ++ evidence/pipeline.png | Bin 0 -> 68147 bytes evidence/pipeline.svg | 2374 +++++++++++++++++++++++++++++ evidence/public-smoke.txt | 2 + evidence/resources-stderr.txt | 6 + evidence/resources.json | 1410 +++++++++++++++++ evidence/restart-after.json | 373 +++++ evidence/restart-before.json | 373 +++++ evidence/runtime.json | 373 +++++ evidence/security-stderr.txt | 2 + evidence/security.json | 15 + evidence/slow-upload.json | 1 + evidence/startup.txt | 80 + evidence/stopped-instances.jsonl | 1 + evidence/student-tests.txt | 26 + infra/haproxy.cfg | 46 + migrations/001_initial.sql | 42 + migrations/002_unicode_nul.sql | 17 + migrations/003_idempotency.sql | 12 + migrations/004_attachments.sql | 30 + migrations/005_pipeline.sql | 40 + requirements-research.txt | 2 + requirements.txt | 12 + scripts/bootstrap.py | 38 + scripts/restart.sh | 9 + scripts/snapshot.py | 11 + scripts/stop-one.sh | 4 + scripts/stop_one.py | 32 + scripts/wait_ready.py | 16 + src/admin.py | 127 ++ src/bucket_bootstrap.py | 13 + src/common.py | 98 ++ src/migrate.py | 13 + src/objects.py | 156 ++ src/pipeline.py | 332 ++++ src/postgres.py | 409 +++++ src/render.py | 64 + src/server.py | 296 ++++ student-tests/durability.py | 52 + student-tests/images.py | 24 + student-tests/pipeline_checks.py | 188 +++ student-tests/plot.py | 28 + student-tests/resources.py | 52 + student-tests/security.py | 27 + student-tests/slow_upload.py | 32 + student-tests/test_api.py | 177 +++ student-tests/test_images.py | 52 + 64 files changed, 10106 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/build.txt create mode 100644 evidence/clean-clone.txt create mode 100644 evidence/durability-stderr.txt create mode 100644 evidence/durability.json create mode 100644 evidence/failover.txt create mode 100644 evidence/final-tests.txt create mode 100644 evidence/jobs-summary.json create mode 100644 evidence/openapi.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/pipeline-stderr.txt create mode 100644 evidence/pipeline.json create mode 100644 evidence/pipeline.png create mode 100644 evidence/pipeline.svg create mode 100644 evidence/public-smoke.txt create mode 100644 evidence/resources-stderr.txt create mode 100644 evidence/resources.json create mode 100644 evidence/restart-after.json create mode 100644 evidence/restart-before.json create mode 100644 evidence/runtime.json create mode 100644 evidence/security-stderr.txt create mode 100644 evidence/security.json create mode 100644 evidence/slow-upload.json create mode 100644 evidence/startup.txt create mode 100644 evidence/stopped-instances.jsonl create mode 100644 evidence/student-tests.txt create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_unicode_nul.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_pipeline.sql create mode 100644 requirements-research.txt create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100755 scripts/restart.sh create mode 100644 scripts/snapshot.py create mode 100755 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/wait_ready.py create mode 100644 src/admin.py create mode 100644 src/bucket_bootstrap.py create mode 100644 src/common.py create mode 100644 src/migrate.py create mode 100644 src/objects.py create mode 100644 src/pipeline.py create mode 100644 src/postgres.py create mode 100644 src/render.py create mode 100644 src/server.py create mode 100644 student-tests/durability.py create mode 100644 student-tests/images.py create mode 100644 student-tests/pipeline_checks.py create mode 100644 student-tests/plot.py create mode 100644 student-tests/resources.py create mode 100644 student-tests/security.py create mode 100644 student-tests/slow_upload.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_images.py diff --git a/.dockerignore b/.dockerignore index bc16e556..8eb0c085 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store + +build diff --git a/.env.example b/.env.example index 119384b5..bc195d6d 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,14 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +APP_PORT=18091 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +API_CPUS=1 +API_MEMORY=256m +HTTP_WORKERS=16 +DB_POOL_MAX=8 +DB_ACCESS_MODE=optimized +POSTGRES_CPUS=1 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_TIMEOUT_SECONDS=20 +JOB_LEASE_SECONDS=60 +MAX_PENDING_JOBS=100 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..1220e717 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,26 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BASE_URL: http://127.0.0.1:18091 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: | + python scripts/bootstrap.py + docker compose build api + docker compose up -d --wait + python scripts/wait_ready.py + make check test test-student BASE_URL=$BASE_URL + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL + docker compose up -d --wait --no-deps api + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5e287bd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2 +WORKDIR /app +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ ./src/ +COPY migrations/ ./migrations/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] +STOPSIGNAL SIGTERM +CMD ["python", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..9bda7e0c 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,8 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-student + +test-student: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v diff --git a/REPORT.md b/REPORT.md index e98f75e9..f217f83c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,210 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5 — очереди и асинхронные изображения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Майорова Софья, БВТ2301. Целевая оценка: 5. Трек 2 — доказанная идемпотентность versioned pipeline.** Выполнено 27 октября 2026 года. Оценку подтверждает преподаватель. + +Продолжено [моё решение lab4, PR 18](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/18). Сохранены две API за LB, сессии PostgreSQL, Redis-кэш, keyset-пагинация, идемпотентность сообщений и приватные S3-вложения. Upload изменён по контракту: 202 queued после устойчивого staging и сохранения задания; resize выполняется отдельным worker. Исходные README/COURSE/contracts/tests не менялись. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine/Compose v2 и Python 3.9+ для публичных тестов на хосте. Сервер/worker — Python 3.12.8 внутри Docker. На этом Mac: -## Архитектура и решения +```sh +colima start --profile mtusi-labs +export DOCKER_CONTEXT=colima-mtusi-labs +cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab5" +python3 scripts/bootstrap.py +docker compose build api +docker compose up -d --wait +export BASE_URL=http://127.0.0.1:18091 +python3 scripts/wait_ready.py +make check test test-student BASE_URL=$BASE_URL +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +На другой машине клонировать публичный форк lab5 и выбрать `solution/BVT2301-Mayorova`, затем выполнить те же команды из корня clone. `build api` идёт первым: migrate, worker, dispatcher и tools используют тот же построенный образ. Одноразовый migrate под advisory lock применяет 001–005; API DDL не выполняют. Bucket-bootstrap идемпотентно создаёт приватный bucket. После них поднимаются API, dispatcher и worker. -## Доказательства критериев +Bootstrap создаёт случайные локальные пароли и ключи, сохраняет существующие значения и `.env`, пересобирает только генерируемую конфигурацию identities из этих ключей. Каталог secrets — 0700; полный s3_config — **0600**, монтируется только root S3 gateway. Отдельные ключи процесса читаются UID 10001 внутри контейнера; их файлы находятся в закрытом 0700-каталоге хоста. API не получает worker/admin credentials. `.env`, secrets, build и данные исключены из Git и build context. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Остановка с сохранением данных: `docker compose down`. Только для намеренного удаления ненужных **учебных данных данного проекта** — `docker compose down -v`. В restart/failover volumes не удаляются. Изолированный clean-clone стенд использует другое имя проекта, порт и временные volumes, которые удаляются после его проверки. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура и границы подтверждения -## Эксперименты и отказы +```text +Клиент → LB → API × 2 → PostgreSQL (сессии, чаты, сообщения, attachments + jobs/outbox) + ├→ Redis cache профилей (восстанавливаемый) + └→ S3 staging (полные исходные bytes) + ↑ +PostgreSQL jobs/outbox → dispatcher → Redis Streams broker (AOF always) + ↓ XREADGROUP / XAUTOCLAIM + worker → изолированный render subprocess + ├→ конечный S3 results//vN/... + └→ атомарный ready + варианты в PostgreSQL → XACK + XDEL +``` -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +S3 — SeaweedFS 4.51: отдельный gateway и mini master/volume/filer на сети storage. И байты, и filer metadata устойчиво находятся в `s3_data`; API не имеет доступа к filer, административный UI/WebDAV отключены. PostgreSQL — другой volume. Брокер — **отдельный Redis 7.4.1**, не кэш: AOF, `appendfsync always`, `noeviction`, persistent `broker_data`, без публикации порта на хост. Кэш остаётся disposable Redis без AOF. Единственная публичная точка — LB на **127.0.0.1:18091**. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Образы Python 3.12.8, PostgreSQL 17.2-bookworm, Redis 7.4.1-bookworm, HAProxy 3.0.5-alpine и SeaweedFS 4.51 закреплены digest в Dockerfile/Compose. Например, SeaweedFS: `sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8`. Pillow 13.0.0, boto3 1.40.61, psycopg/redis и транзитивные зависимости закреплены в requirements.txt. Это проверенный набор версий, не обещание отсутствия будущих уязвимостей. -## Ограничения и следующие шаги +| Процесс | CPU | Память | Состояние | +| --- | ---: | ---: | --- | +| API, каждая из двух | 1 | 256 MiB | Без устойчивых локальных originals | +| Worker, concurrency=1 | 1 | 768 MiB | Временный tmpfs; источник/результат в S3 | +| Dispatcher | 1 | 256 MiB | Устойчивый outbox в PostgreSQL | +| PostgreSQL | 1 | 512 MiB | postgres_data | +| Broker | 0,5 | 192 MiB, maxmemory=96mb/noeviction | broker_data/AOF | +| S3 store / gateway | 1 / 0,5 | 512 / 512 MiB | s3_data / без собственных originals | +| LB / cache | 0,5 / 0,25 | 64 / 64 MiB | Нет обязательных устойчивых данных | -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сумма CPU-лимитов превышает 4 vCPU локальной VM: это overcommit, не выделенные физические ядра. Измерения проводились на macOS arm64, Colima/Linux с 4 vCPU и примерно 4 GiB RAM. Ранее запущенные учебные стенды не останавливались. + +### Что сохранено при 202 + +API проверяет Bearer/доступ к чату, Content-Length ≤10 MiB, допустимый filename/MIME, реальный PNG/JPEG header, размеры ≤40 млн pixels и `verify()`. Исходник читается кусками 64 KiB в ограниченный временный файл, SHA-256 считается потоково. Полного декодирования/resize в API нет. Поэтому JPEG с корректным заголовком, но повреждённым scan data может быть принят и затем завершиться **failed invalid_image** в worker; заведомо чужой формат/несовпадение MIME отклоняется 415, лимиты — 413, опасное имя — 400. + +Перед PUT фиксируется internal intent uploading со случайным UUID, server-generated staging key и ожидаемой checksum. Под row lock выполняется полный PUT в приватный `staging/`. В одной следующей транзакционной границе с этим lock создаётся queued job/outbox и меняется Attachment на queued с current_job_id. **COMMIT выполнен до 202.** API возвращает сохранённый queued snapshot даже если worker успел закончить раньше HTTP-ответа. Bytes в PostgreSQL и broker не помещаются. Временный файл API не является долговременным staging. + +Уже принятая работа не зависит от RAM API или доступности dispatcher/broker в этот момент. Readiness API проверяет PostgreSQL и S3; остановленный worker/broker не запускает resize в HTTP. При `MAX_PENDING_JOBS=100` новые uploading/queued/processing ограничены общим счётчиком PostgreSQL под advisory lock; заполнение даёт 503. Это предел количества, не расчёт disk quota. Неподтверждённый upload не следует прозрачно повторять: контракт идемпотентности относится к POST messages. + +## Transactional outbox и at-least-once + +Таблица `attachment_jobs` совмещает job registry и transactional outbox. Создание job и queued Attachment атомарно. Dispatcher выбирает due rows `FOR UPDATE SKIP LOCKED`, выполняет XADD с envelope по jobs.schema.json, затем отмечает published_at. При сбое до XADD строка остаётся; после XADD до фиксации marker возможен дубль. XADD response с AOF always подтверждает приём/сохранение брокером согласно его политике. Это ещё не подтверждение обработки изображения. + +Worker читает consumer group `image-workers`, **count/prefetch=1 на slot**. XAUTOCLAIM возвращает зависшую pending delivery после lease. Dispatcher также восстанавливает processing с истёкшей lease и повторяет долго не захваченную queued publication. DB-инварианты, а не предположение о единственности сообщения брокера, определяют допустимость обработки. + +Broker message содержит только schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at; допускается контрактный trace carrier. Пример фактически проверенного envelope — 243 bytes. Ключи хранилища и credentials берутся worker из доверенных настроек/БД, произвольные URL/filename-пути из сообщения не исполняются. Неверный envelope наблюдаем в ограниченном `rejected-jobs` stream: только причина, исходный message ID и hash, без сохранения произвольного payload. Подтверждение ошибочного envelope происходит после записи этой DLQ-записи. + +XACK + XDEL выполняются одной Redis Lua-операцией **после** устойчивого ready/failed либо безопасного no-op. При потере ack доставка повторяется. Ошибка БД, не позволившая сохранить результат/ошибку, не подтверждает delivery. Доставка остаётся at-least-once; сокращение числа логических эффектов не превращает её в exactly-once. + +## Worker, варианты, retry и lease + +Worker — отдельный контейнер/процесс. Default concurrency=1, конфигурация допускает 1–2; при увеличении нужно отдельно увеличить/измерить WORKER_MEMORY, в отчёте проверен только 1. Производственная lease — 60 s; дочерний renderer ограничен wall timeout 20 s, CPU 15 s (hard 16), virtual address space 512 MiB. Worker контейнер ограничен 768 MiB, tmpfs 128 MiB. Исходник ≤10 MiB скачивается кусками с SHA-256/size и deadline 15 s; S3 connect/read timeouts — 1/3 s, SDK total attempts=1. Самостоятельные retry контролирует job registry. + +Pillow полностью декодирует исходник в **изолированном subprocess**, проверяет pixels до load, применяет EXIF orientation, alpha композитит на белом. Размеры рассчитываются с округлением вниз, min 1 px. Для thumbnail/optimized маленький исходник не увеличивается. Варианты сохраняются без EXIF/ICC: + +- thumbnail: JPEG quality=85, вписан в 256×256; fixture 320×200 → 256×160; +- crop: центрированный квадрат 128×128, Lanczos; маленький исходник допускает увеличение; +- optimized: WebP, вписан в 1280×1280, method=4, quality=82 для pipeline v1, quality=70 для v2. Уменьшение каждого маленького файла не обещается. + +Оригинал публикуется побайтно в конечный S3-prefix отдельно от staging. Ready фиксируется только после original + **всех трёх** вариантов с checksums/размерами. GET content для queued/processing/failed даёт 409, для чужого attachment — 404. Сообщение может ссылаться на queued/processing своего чата, failed прикреплять нельзя. Download остаётся авторизованным proxy через API без redirect. + +Attempt увеличивается только при успешном claim; job_id при redelivery/retry не меняется. По умолчанию максимум **5 попыток** на job. Временные storage/timeouts/processing failures возвращают queued с backoff `min(8, 0.5*2^(attempt-1)) + U(0,0.5)` секунд; постоянные invalid_image/source_missing/checksum_mismatch/result_conflict сразу дают failed. Исчерпание бюджета — failed retry_exhausted. В публичной ошибке нет exception details. Реестр failed и job_events доступны оператору, не клиенту. + +Повторная delivery уже ready/failed — no-op. Busy/early duplicate не отбирает чужой lease. Истёкший lease снимает ownership и создаёт следующую управляемую попытку. Final COMMIT проверяет current_job_id, pipeline_version, state, lease_token **и срок lease**; потерявший аренду worker не может опубликовать старый результат. + +## Версионирование и инварианты трека 2 + +Ключ эффекта: `(attachment_id, pipeline_version, variant)`. Ключи объектов: `results//vN/original|thumbnail|crop|optimized`. PUT результата условный `If-None-Match: *`; готовый объект с совпадающими size/hash повторно не пишется, конфликтующий не затирается. Результаты детерминированы закреплённым renderer и параметрами версии. + +Операторский переход на следующую реализованную версию атомарно помечает старую job superseded, создаёт новую queued job с отдельным job_id и обновляет current_job_id/version. Источник новой job — устойчивый исходник прежней версии либо staging, а не локальный файл API. Сейчас реализованы только v1 и v2; публичный upload создаёт v1. + +Проверяемые инварианты: + +1. Подтверждённая queued задача сохраняется после смерти API и рестарта broker. +2. Одновременный дубль допускает единственный claim; serial duplicate/reorder не увеличивает число успешно опубликованных вариантов. +3. Падение до/после S3 side effect и до/после результата БД не теряет исходник и не создаёт второй набор объектов той же версии. +4. Устаревшая delivery и уже выполняющийся старый worker не возвращают ready в processing и не затирают результат новой версии. +5. Failed наблюдаем, retries конечны; replay допускается только для текущей failed job с доступным source и оставляет audit. +6. Cleanup не удаляет staging текущей queued/processing работы или свежей failed. + +`student-tests/pipeline_checks.py` выполняет настоящие операции PostgreSQL/S3/Redis и завершает отдельные worker/publisher-процессы через **os._exit(77)** в явных операторских fault points. Это не публичный HTTP hook. Проверены: COMMIT без publisher; publisher после XADD до outbox marker; worker before_side_effect, after_original, after_objects, after_commit; повтор/перестановка envelope; два concurrent consumer; pause старого subprocess после записи всех объектов → завершение v2 → продолжение v1, получающее fenced no-op; поздняя v1 delivery после ready v2. + +Для первого crash действительно ожидается истечение тестовой lease=5 s; в двух следующих случаях deadline той же тестовой строки ускоряется SQL-изменением lease_until, чтобы не тратить по 60 s. Это воздействие теста на **новые собственные job IDs**, не подмена результата. Production lease=60 s не изменяется. После каждого crash выполняется реальный Redis pending reclaim/ack; в соответствующем prefix остаётся ровно четыре объекта. После after_commit новая delivery не увеличивает attempt. + +Transient failure воспроизводится настоящим S3 отказом подписи с заведомо неверным тестовым ключом: budget=2 только для этой job, затем queued → failed retry_exhausted, после явного replay с правильными credentials — ready на attempt 3. Повреждённый JPEG доходит до постоянного failed invalid_image. Проверены EXIF-поворот, прозрачность на белом, отсутствие EXIF в thumbnail и запрет увеличения маленького thumbnail. + +Команда исследования, **только на синтетическом стенде** (останавливает фоновую обработку, не API): + +```sh +docker compose stop worker dispatcher +docker compose run --rm -T --no-deps -v "$PWD:/work:ro" -w /work audit python student-tests/pipeline_checks.py +docker compose up -d --no-deps worker dispatcher +``` + +`audit` — tools profile, не запускается обычным up. Только он получает дополнительные локальные секреты для операторских/негативных проверок; API их не получает. Исследование автоматически создаёт новые synthetic users/chats/jobs, не изменяет чужие. + +## Операторские команды и очистка + +```sh +# Наблюдаемый реестр failed: +docker compose exec worker python src/admin.py failed +# Явный replay текущей failed job; UUID взять из предыдущей команды: +docker compose exec worker python src/admin.py replay JOB_UUID +# Новый pipeline для конкретного вложения: +docker compose exec worker python src/admin.py version ATTACHMENT_UUID --version 2 +# Dry-run и явное применение cleanup: +docker compose exec worker python src/admin.py cleanup +docker compose exec worker python src/admin.py cleanup --apply +``` + +Replay не сбрасывает счётчик, не повторяет ready и не создаёт новую копию успешной job: явно добавляет новый бюджет попыток к тому же job_id, сохраняя историю. Необходим доступный source; после истечения retention может понадобиться внешнее восстановление исходника. + +Staging удаляется только **после COMMIT ready**. Crash после COMMIT до очистки безопасен: операторский cleanup идемпотентно удаляет оставшийся staging готовой работы. Failed/незавершённый uploading хранится минимум **24 h**; queued/processing никогда не собираются. Cleanup использует row locks/SKIP LOCKED и фильтр managed staging prefix; dry-run без --apply ничего не меняет. Счётчики cleanup отражают кандидаты по метаданным: часть staging objects уже удалена worker, повторная очистка безопасна. Реестр и сообщения не удаляются вместе с исходником. Неактуальные immutable outputs старых версий пока сохраняются — автоматическая уборка old-version results не заявляется. + +## Результаты и воспроизводимость + +| Критерий | Проверка и наблюдение | Evidence | +| --- | --- | --- | +| Публичный минимум lab5 | Smoke: 202, ready, исходный SHA-256 и настоящий JPEG thumbnail | final-tests.txt | +| Прежний API и собственные изображения | 18 собственных тестов PASS: Unicode/NUL, auth, гонки/idempotency, variants, ACL, MIME/bytes/pixels/filename | student-tests.txt, final-tests.txt | +| Устойчивый staging/очередь | Worker/dispatcher остановлены, upload 202 и message; обе API убиты SIGKILL до publish; после рестарта со старым Bearer всё сохранено | durability.json | +| Persistence broker | Настоящий Redis Streams XADD; до/после restart с volume — 1/1 запись; затем та же job ready | durability.json | +| Полный restart | API/LB/worker/dispatcher остановлены, PostgreSQL/cache/оба S3/broker перезапущены с volumes; старые сессия и bytes сохранены | persistence.txt, restart-before/after.json | +| Отказ API | 2 реплики, 114 подтверждённых writes, 2 переходные ошибки, восстановление 2,534 s | failover.txt | +| Все уровни retry/final/replay | Реальные S3 signature failures, конечный budget=2, audit/replay и готовый результат на attempt 3 | pipeline.json | +| Outbox и worker crash windows | Отдельные процессы exit 77, реальные redelivery и четыре объекта на версию | pipeline.json | +| Reorder, concurrent duplicate, stale version | 13 групп assertions/invariants; old running process fenced, new ready не изменился | pipeline.json | +| Изображения и лимиты worker | EXIF/alpha/small-image, malformed JPEG failed, 40 Мп RGB ready, subprocess/container limits | pipeline.json, resources.json | +| Cleanup | Dry-run/apply/repeat, защита активного source и свежих failed | pipeline.json | +| Приватность | Anonymous known-key GET/list →403; в envelope только 7 служебных полей, проверены неверные типы/форматы | security.json | +| Полная валидация схемы | OpenAPI 3.1 / JSON Schema PASS | openapi.txt | +| Чистый clone | Отдельные новые секреты, проект, порт и пустые volumes | PASS: новые секреты и пустые PG/S3/broker volumes, 18 тестов, SIGKILL API с queued work и полный restart; clean-clone.txt | + +Все сценарии сохраняют tokens только в памяти теста. Snapshot содержит разрешённые поля state/started_at/mounts/resources, не environment или secret values. Fault injection и операторские команды воспроизводимы из репозитория. Дополнительные команды: + +```sh +export BASE_URL=http://127.0.0.1:18091 +python3 student-tests/durability.py +python3 student-tests/resources.py +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL +docker compose up -d --wait --no-deps api +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL +``` + +### Измерение ресурсов и задержек + +Это дополнительная проверка bounded worker, **не заявка на трек 1**. Concurrency/prefetch=1, один прогон на профиль, по 10 одновременно начинающих клиентов. Генерация PNG до измерения; отправляется реальное тело, без скрытых retry. Worker перезапускается перед профилем, cgroup peak относится к этому запуску. Считается время HTTP-accept и время до наблюдаемого ready; polling добавляет до примерно 0,3 s. Для N=10 nearest-rank p95/p99 равны максимуму. Сэмплер docker exec также вносит overhead, closed-loop не даёт верхней границы production throughput. + +| Pixels / bytes | 202: p95 | До ready: p50 / p95=p99 | Worker cgroup peak / resident anon | Успех | +| --- | ---: | ---: | ---: | --- | +| 64 000 / 1 294 | 0,080 s | 1,834 / 2,923 s | 78,81 / 76,82 MiB | 10/10 | +| 2 520 000 / 7 563 763 | 1,102 s | 4,155 / 7,025 s | 123,46 / 112,50 MiB | 10/10 | +| 40 000 000 / 533 859 | 0,092 s | 6,826 / 11,659 s | 253,67 / 251,36 MiB | 10/10 | + +После исследования и восстановления очередь полностью опустела: 85 ready, 3 намеренно failed из-за повреждённых JPEG, 0 queued/processing/uploading, 0 stream entries/pending deliveries; у всех ready по три варианта (jobs-summary.json). + +Все 30 приняли 202 и достигли ready; OOM kill=0. `memory.peak` — kernel cgroup максимум контейнера, включая renderer, tmpfs/cache/kernel; resident anon взят как максимум `memory.stat anon` среди samples, это не точный максимальный RSS каждого отдельного PID. CPU counters также сохранены в raw samples. Маленький сжатый файл 40 Мп занимает больше RAM/CPU при декодировании, чем более крупный файл с меньшим количеством pixels. Быстрый 202 не означает мгновенный thumbnail: работа проходит ограниченный worker и ждёт очередь. + +![Задержка и память отдельного worker](evidence/pipeline.png) + +Основной исследовательский результат трека 2: все внедрённые crash/duplicate/reorder/version сценарии соблюли заявленные инварианты; умирающий после COMMIT worker не заставил повторно публиковать эффект, а живой, но устаревший worker получил fencing вместо перезаписи новой версии. Это измеренная функциональная проверка, не статистическое доказательство отсутствия всех гонок. + +Для графиков: Python 3.11+, `pip install -r requirements-research.txt`, `python student-tests/plot.py`. PNG/SVG уже приложены. Workflow Application добавляет сборку/стенд/smoke/own tests/failover/persistence; hosted CI отдельно не объявляется прошедшим без результата платформы. + +## Ограничения + +Один host/PostgreSQL/S3-store/broker/LB остаются точками отказа. Доказаны процессные сбои и restart с volumes, не потеря диска/всего хоста или гарантия power-loss durability. HTTP и неподключённые к хосту backend-порты допускаются только для локального синтетического стенда; удалённой публикации без TLS нет. Docker operator имеет доступ к секретам. + +Существующий immutable результат проверяется по HEAD size/hash metadata, не повторным чтением всего объекта; защита от произвольной подмены привилегированным S3-оператором не заявляется. + +Нет распределённой ACID-транзакции. Outbox и периодическая republish закрывают разрывы повтором, поэтому сообщения/CPU могут дублироваться; fences и immutable versioned keys защищают опубликованный результат. Потерявшийся worker может ещё некоторое время тратить CPU до timeout/завершения, но не публиковать старый ready. Нормальные bounded render заканчиваются до lease; выбирая меньшую lease, надо отдельно измерять нагрузку. + +Состояния jobs/events и неактуальные versions накапливаются. Cleanup запускает оператор, не скрытый scheduler. Retention failed sources ограничивает возможность replay без внешней копии. Принятая задача может закончиться failed из-за постоянной ошибки или исчерпания retry; это наблюдаемое завершение, а не исчезновение работы. Image parser ограничивает декодированные PNG-текстовые метаданные 1 MiB. Антивирус и UI не входят в работу. + +## Источники и ИИ + +Собственное решение lab4 (Майорова Софья, PR 18); чужие студенческие решения не копировались. Задание и contracts текущего skeleton — основание обязательного поведения. + +- [Redis persistence/AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [XAUTOCLAIM](https://redis.io/docs/latest/commands/xautoclaim/), [Streams](https://redis.io/docs/latest/develop/data-types/streams/). +- [Pillow ImageOps/EXIF/fit](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html). +- [SeaweedFS S3 credentials](https://github.com/seaweedfs/seaweedfs/wiki/S3-Credentials), [PostgreSQL explicit locks](https://www.postgresql.org/docs/17/explicit-locking.html). + +Codex использован для реализации, тестов, fault-injection, запуска измерений и оформления. Все PASS/числа получены реальными командами, границы их применимости приведены выше. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..25366f0e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,234 @@ +name: mtusi-lab5 +x-application: &application + image: mtusi-messenger:lab5 + networks: [backend] + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + HTTP_WORKERS: ${HTTP_WORKERS:-16} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized} + LOG_LEVEL: WARNING + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret + secrets: [database_url, cache_url, s3_api_key, s3_api_secret] + tmpfs: ["/tmp:size=128m,mode=1777"] + mem_limit: ${API_MEMORY:-512m} + cpus: ${API_CPUS:-1} + stop_grace_period: 10s +services: + migrate: + <<: *application + command: [python, src/migrate.py] + healthcheck: + disable: true + deploy: + replicas: 1 + depends_on: + postgres: + condition: service_healthy + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + bucket-bootstrap: + condition: service_completed_successfully + bucket-bootstrap: + <<: *application + command: [python, src/bucket_bootstrap.py] + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: [s3_admin_key, s3_admin_secret] + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + audit: + <<: *application + profiles: [tools] + environment: + <<: *environment + BASE_URL: http://lb:8080 + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: 5 + secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url] + healthcheck: + disable: true + deploy: + replicas: 1 + dispatcher: + <<: *application + command: [python, src/pipeline.py, dispatch] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + secrets: [database_url, broker_url] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + worker: + <<: *application + command: [python, src/pipeline.py, work] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret] + mem_limit: ${WORKER_MEMORY:-768m} + tmpfs: ["/tmp:size=128m,mode=1777"] + cpus: 1 + deploy: + replicas: 1 + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + bucket-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + broker: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""] + volumes: [broker_data:/data] + networks: [backend] + mem_limit: 192m + cpus: 0.5 + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + s3-store: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64] + networks: [storage] + volumes: [s3_data:/data] + mem_limit: 512m + cpus: 1 + stop_grace_period: 20s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/] + interval: 2s + timeout: 2s + retries: 45 + s3: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16] + secrets: [s3_config] + networks: [backend, storage] + mem_limit: 512m + environment: + GOMEMLIMIT: 384MiB + cpus: 0.5 + depends_on: + s3-store: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'nc -z 127.0.0.1 8333'] + interval: 2s + timeout: 1s + retries: 30 + lb: + image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949 + ports: ["127.0.0.1:${APP_PORT:-18091}:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + mem_limit: 64m + cpus: 0.5 + stop_grace_period: 12s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 2s + retries: 10 + postgres: + image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [backend] + healthcheck: + test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1} + redis: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 64m + cpus: 0.25 +volumes: + postgres_data: + broker_data: + s3_data: +networks: + edge: + storage: + internal: true + backend: + internal: true +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_api_key: + file: ./secrets/s3_api_key + s3_api_secret: + file: ./secrets/s3_api_secret + s3_config: + file: ./secrets/s3_config + broker_url: + file: ./secrets/broker_url + s3_worker_key: + file: ./secrets/s3_worker_key + s3_worker_secret: + file: ./secrets/s3_worker_secret diff --git a/evidence/build.txt b/evidence/build.txt new file mode 100644 index 00000000..336722dd --- /dev/null +++ b/evidence/build.txt @@ -0,0 +1,55 @@ + Image mtusi-messenger:lab5 Building +#1 [internal] load local bake definitions +#1 reading from stdin 650B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 15.97kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [2/7] WORKDIR /app +#8 CACHED + +#9 [4/7] COPY requirements.txt ./ +#9 CACHED + +#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#10 CACHED + +#11 [6/7] COPY src/ ./src/ +#11 DONE 0.0s + +#12 [7/7] COPY migrations/ ./migrations/ +#12 DONE 0.0s + +#13 exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:7e99f0c6a7ac94b85161817ff88967106e59e98d4e7dd9de3938b1135e0b12cd done +#13 exporting config sha256:0e7c28f3d856890d8caee1ac1cdc0b496010b55a4e1f60fd6b733acc575f7f2e done +#13 exporting attestation manifest sha256:f9c54687679fe921dd39dde4f93898025e671dd4d91f692ffb6ec14f50df16c6 done +#13 exporting manifest list sha256:14afbe72a221379a2e0438f88629275c862f8347151aee0a33ac60708793d1c2 done +#13 naming to docker.io/library/mtusi-messenger:lab5 done +#13 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#13 DONE 0.1s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:lab5 Built diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..fbb76a17 --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,1054 @@ +Cloning into '/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab5/build/lab5-clean-7668fd68/repo'... +done. +Local configuration ready; no secrets printed. + Image mtusi-messenger:lab5 Building +#1 [internal] load local bake definitions +#1 reading from stdin 721B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 1.1s + +#4 [internal] load .dockerignore +#4 transferring context: 203B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 73.48kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [4/7] COPY requirements.txt ./ +#8 CACHED + +#9 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#9 CACHED + +#10 [6/7] COPY src/ ./src/ +#10 CACHED + +#11 [2/7] WORKDIR /app +#11 CACHED + +#12 [7/7] COPY migrations/ ./migrations/ +#12 CACHED + +#13 resolving provenance for metadata file +#13 DONE 0.0s + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:e06ce20ee092cb4997cdd7c8020b9659af43ac04b1b2dcbed69ea0ce95100049 done +#14 exporting config sha256:6b11dd9f29f79206ad928a7577a62cc834ad7502c3231f032e29534c803d81b3 done +#14 exporting attestation manifest sha256:861ed44d72400f7f11a9e27680aa5ff785a038924413c6784a4976384aabc2b5 done +#14 exporting manifest list sha256:8dd5aba56cbd0954c950d890bda5b57986bd0fd89535e5d140387bacbb97e65d done +#14 naming to docker.io/library/mtusi-messenger:lab5 done +#14 unpacking to docker.io/library/mtusi-messenger:lab5 done +#14 DONE 0.0s + Image mtusi-messenger:lab5 Built + Network lab5-clean-7668fd68_storage Creating + Network lab5-clean-7668fd68_storage Created + Network lab5-clean-7668fd68_edge Creating + Network lab5-clean-7668fd68_edge Created + Network lab5-clean-7668fd68_backend Creating + Network lab5-clean-7668fd68_backend Created + Volume lab5-clean-7668fd68_broker_data Creating + Volume lab5-clean-7668fd68_broker_data Created + Volume lab5-clean-7668fd68_postgres_data Creating + Volume lab5-clean-7668fd68_postgres_data Created + Volume lab5-clean-7668fd68_s3_data Creating + Volume lab5-clean-7668fd68_s3_data Created + Container lab5-clean-7668fd68-postgres-1 Creating + Container lab5-clean-7668fd68-redis-1 Creating + Container lab5-clean-7668fd68-broker-1 Creating + Container lab5-clean-7668fd68-s3-store-1 Creating + Container lab5-clean-7668fd68-broker-1 Created + Container lab5-clean-7668fd68-postgres-1 Created + Container lab5-clean-7668fd68-migrate-1 Creating + Container lab5-clean-7668fd68-redis-1 Created + Container lab5-clean-7668fd68-s3-store-1 Created + Container lab5-clean-7668fd68-s3-1 Creating + Container lab5-clean-7668fd68-migrate-1 Created + Container lab5-clean-7668fd68-dispatcher-1 Creating + Container lab5-clean-7668fd68-s3-1 Created + Container lab5-clean-7668fd68-bucket-bootstrap-1 Creating + Container lab5-clean-7668fd68-dispatcher-1 Created + Container lab5-clean-7668fd68-bucket-bootstrap-1 Created + Container lab5-clean-7668fd68-worker-1 Creating + Container lab5-clean-7668fd68-api-2 Creating + Container lab5-clean-7668fd68-api-1 Creating + Container lab5-clean-7668fd68-api-1 Created + Container lab5-clean-7668fd68-api-2 Created + Container lab5-clean-7668fd68-worker-1 Created + Container lab5-clean-7668fd68-lb-1 Creating + Container lab5-clean-7668fd68-lb-1 Created + Container lab5-clean-7668fd68-postgres-1 Starting + Container lab5-clean-7668fd68-s3-store-1 Starting + Container lab5-clean-7668fd68-broker-1 Starting + Container lab5-clean-7668fd68-redis-1 Starting + Container lab5-clean-7668fd68-redis-1 Started + Container lab5-clean-7668fd68-broker-1 Started + Container lab5-clean-7668fd68-postgres-1 Started + Container lab5-clean-7668fd68-postgres-1 Waiting + Container lab5-clean-7668fd68-s3-store-1 Started + Container lab5-clean-7668fd68-s3-store-1 Waiting + Container lab5-clean-7668fd68-postgres-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Starting + Container lab5-clean-7668fd68-migrate-1 Started + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-s3-store-1 Healthy + Container lab5-clean-7668fd68-s3-1 Starting + Container lab5-clean-7668fd68-s3-1 Started + Container lab5-clean-7668fd68-s3-1 Waiting + Container lab5-clean-7668fd68-s3-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Starting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Started + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-redis-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-redis-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-api-2 Starting + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-worker-1 Starting + Container lab5-clean-7668fd68-worker-1 Started + Container lab5-clean-7668fd68-api-2 Started + Container lab5-clean-7668fd68-api-1 Starting + Container lab5-clean-7668fd68-api-1 Started + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-lb-1 Starting + Container lab5-clean-7668fd68-lb-1 Started + Container lab5-clean-7668fd68-lb-1 Waiting + Container lab5-clean-7668fd68-dispatcher-1 Waiting + Container lab5-clean-7668fd68-redis-1 Waiting + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-postgres-1 Waiting + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-worker-1 Waiting + Container lab5-clean-7668fd68-s3-store-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-s3-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-s3-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-dispatcher-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-redis-1 Healthy + Container lab5-clean-7668fd68-postgres-1 Healthy + Container lab5-clean-7668fd68-s3-store-1 Healthy + Container lab5-clean-7668fd68-worker-1 Healthy + Container lab5-clean-7668fd68-lb-1 Healthy +Ready through load balancer. +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18092" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18092" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.131s + +OK + Container lab5-clean-7668fd68-worker-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-worker-1 Stopped + Container lab5-clean-7668fd68-api-2 Starting + Container lab5-clean-7668fd68-api-2 Started + Container lab5-clean-7668fd68-api-1 Starting + Container lab5-clean-7668fd68-api-1 Started + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-broker-1 Restarting + Container lab5-clean-7668fd68-broker-1 Started + Container lab5-clean-7668fd68-broker-1 Running + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-worker-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-worker-1 Started + Container lab5-clean-7668fd68-worker-1 Running + Container lab5-clean-7668fd68-dispatcher-1 Running +{ + "killed_api_containers": [ + "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56" + ], + "stream_entries_before_after_broker_restart": [ + 1, + 1 + ], + "attachment_id": "c087ac91-80ee-4954-a2c1-4c7f01787cd4", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "status": "ready", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "old_bearer_survived": true, + "message_attachment_retained": true, + "before": [ + { + "container": "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "name": "/lab5-clean-7668fd68-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.797245354Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56", + "name": "/lab5-clean-7668fd68-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.73913582Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "5567835c3bab8e2f4b997cb67968c2155a0922fe19240cd60c73e5e090b2760a", + "name": "/lab5-clean-7668fd68-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.927126872Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_broker_data" + } + ] + }, + { + "container": "65eed1838fce2bcb378be8d14628313d7c14fead8280bb63d211015767a70cd5", + "name": "/lab5-clean-7668fd68-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:34.154321143Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "c506b79db3e60b71e314e78e4abda0256838675bb9e9daba579efbb5b8f2e744", + "name": "/lab5-clean-7668fd68-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:30.132057402Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "791d5c6b17a6fbf020c38648a40d7c0fc0ae2db9cedd11aecb85b604da9fc6c6", + "name": "/lab5-clean-7668fd68-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:14:40.374957025Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18092" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "7378d8f662313d4f5475dc6b6be1f82e9e60736564a5d94c4e939a75ea87d9f4", + "name": "/lab5-clean-7668fd68-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:29.547624841Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "12dd22014a45a0658b2fa3c5117723c471c2f0c438a245e381fd7904eee04080", + "name": "/lab5-clean-7668fd68-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926579904Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab5-clean-7668fd68_postgres_data" + } + ] + }, + { + "container": "e5bcddc6ef40108e9af7bce37e209ea1524097dab06b232bf9126a0e0d41e303", + "name": "/lab5-clean-7668fd68-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926291214Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "310d1b7f9d4700a40971a9d27e5c49bfc7f3d818af445ccf985fc3ca5bab4a5b" + } + ] + }, + { + "container": "35e9daaeef21c5bfc0cfa46bfe9054b23236c1ad76e62bab70b8602d44e82c00", + "name": "/lab5-clean-7668fd68-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:31.546652936Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "a9677d401ea04710c1220e23073bd3d05c5f6955399379842d5357a3f6d8a5c4" + } + ] + }, + { + "container": "0847dd77bc06ba2fe81753689f8f46ff09b0dd1a6851d8109762c9552bb10f98", + "name": "/lab5-clean-7668fd68-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.925042463Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_s3_data" + } + ] + }, + { + "container": "e3343c9d17d09d716d03066cbac7bc24030fa1061164c7efc75eda9413954c0b", + "name": "/lab5-clean-7668fd68-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.738831925Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ], + "after": [ + { + "container": "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "name": "/lab5-clean-7668fd68-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:50.908562499Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56", + "name": "/lab5-clean-7668fd68-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:50.859526761Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "5567835c3bab8e2f4b997cb67968c2155a0922fe19240cd60c73e5e090b2760a", + "name": "/lab5-clean-7668fd68-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:59.65492626Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_broker_data" + } + ] + }, + { + "container": "65eed1838fce2bcb378be8d14628313d7c14fead8280bb63d211015767a70cd5", + "name": "/lab5-clean-7668fd68-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:34.154321143Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "c506b79db3e60b71e314e78e4abda0256838675bb9e9daba579efbb5b8f2e744", + "name": "/lab5-clean-7668fd68-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:15:02.066887519Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "791d5c6b17a6fbf020c38648a40d7c0fc0ae2db9cedd11aecb85b604da9fc6c6", + "name": "/lab5-clean-7668fd68-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:14:40.374957025Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18092" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "7378d8f662313d4f5475dc6b6be1f82e9e60736564a5d94c4e939a75ea87d9f4", + "name": "/lab5-clean-7668fd68-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:29.547624841Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "12dd22014a45a0658b2fa3c5117723c471c2f0c438a245e381fd7904eee04080", + "name": "/lab5-clean-7668fd68-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926579904Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab5-clean-7668fd68_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "e5bcddc6ef40108e9af7bce37e209ea1524097dab06b232bf9126a0e0d41e303", + "name": "/lab5-clean-7668fd68-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926291214Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "310d1b7f9d4700a40971a9d27e5c49bfc7f3d818af445ccf985fc3ca5bab4a5b" + } + ] + }, + { + "container": "35e9daaeef21c5bfc0cfa46bfe9054b23236c1ad76e62bab70b8602d44e82c00", + "name": "/lab5-clean-7668fd68-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:31.546652936Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "a9677d401ea04710c1220e23073bd3d05c5f6955399379842d5357a3f6d8a5c4" + } + ] + }, + { + "container": "0847dd77bc06ba2fe81753689f8f46ff09b0dd1a6851d8109762c9552bb10f98", + "name": "/lab5-clean-7668fd68-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.925042463Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_s3_data" + } + ] + }, + { + "container": "e3343c9d17d09d716d03066cbac7bc24030fa1061164c7efc75eda9413954c0b", + "name": "/lab5-clean-7668fd68-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:15:02.065936755Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ] +} +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18092" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab5-clean-7668fd68-worker-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-lb-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-dispatcher-1 Removing + Container lab5-clean-7668fd68-dispatcher-1 Removed + Container lab5-clean-7668fd68-lb-1 Stopped + Container lab5-clean-7668fd68-lb-1 Removing + Container lab5-clean-7668fd68-lb-1 Removed + Container lab5-clean-7668fd68-api-1 Stopping + Container lab5-clean-7668fd68-api-2 Stopping + Container lab5-clean-7668fd68-api-2 Stopped + Container lab5-clean-7668fd68-api-2 Removing + Container lab5-clean-7668fd68-api-2 Removed + Container lab5-clean-7668fd68-api-1 Stopped + Container lab5-clean-7668fd68-api-1 Removing + Container lab5-clean-7668fd68-worker-1 Stopped + Container lab5-clean-7668fd68-worker-1 Removing + Container lab5-clean-7668fd68-api-1 Removed + Container lab5-clean-7668fd68-redis-1 Stopping + Container lab5-clean-7668fd68-worker-1 Removed + Container lab5-clean-7668fd68-broker-1 Stopping + Container lab5-clean-7668fd68-bucket-bootstrap-1 Stopping + Container lab5-clean-7668fd68-migrate-1 Stopping + Container lab5-clean-7668fd68-bucket-bootstrap-1 Stopped + Container lab5-clean-7668fd68-bucket-bootstrap-1 Removing + Container lab5-clean-7668fd68-migrate-1 Stopped + Container lab5-clean-7668fd68-migrate-1 Removing + Container lab5-clean-7668fd68-bucket-bootstrap-1 Removed + Container lab5-clean-7668fd68-s3-1 Stopping + Container lab5-clean-7668fd68-migrate-1 Removed + Container lab5-clean-7668fd68-postgres-1 Stopping + Container lab5-clean-7668fd68-postgres-1 Stopped + Container lab5-clean-7668fd68-postgres-1 Removing + Container lab5-clean-7668fd68-postgres-1 Removed + Container lab5-clean-7668fd68-s3-1 Stopped + Container lab5-clean-7668fd68-s3-1 Removing + Container lab5-clean-7668fd68-s3-1 Removed + Container lab5-clean-7668fd68-s3-store-1 Stopping + Container lab5-clean-7668fd68-redis-1 Stopped + Container lab5-clean-7668fd68-redis-1 Removing + Container lab5-clean-7668fd68-redis-1 Removed + Container lab5-clean-7668fd68-broker-1 Stopped + Container lab5-clean-7668fd68-broker-1 Removing + Container lab5-clean-7668fd68-broker-1 Removed + Container lab5-clean-7668fd68-s3-store-1 Stopped + Container lab5-clean-7668fd68-s3-store-1 Removing + Container lab5-clean-7668fd68-s3-store-1 Removed + Volume lab5-clean-7668fd68_broker_data Removing + Volume lab5-clean-7668fd68_s3_data Removing + Volume lab5-clean-7668fd68_postgres_data Removing + Network lab5-clean-7668fd68_edge Removing + Network lab5-clean-7668fd68_storage Removing + Network lab5-clean-7668fd68_backend Removing + Volume lab5-clean-7668fd68_broker_data Removed + Volume lab5-clean-7668fd68_postgres_data Removed + Volume lab5-clean-7668fd68_s3_data Removed + Network lab5-clean-7668fd68_storage Removed + Network lab5-clean-7668fd68_edge Removed + Network lab5-clean-7668fd68_backend Removed +PASS: clean clone, new secrets, empty PostgreSQL/S3/broker volumes; 18 own tests, API SIGKILL and broker restart with queued work; full persistence. diff --git a/evidence/durability-stderr.txt b/evidence/durability-stderr.txt new file mode 100644 index 00000000..f13820d4 --- /dev/null +++ b/evidence/durability-stderr.txt @@ -0,0 +1,27 @@ + Container mtusi-lab5-worker-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopped + Container mtusi-lab5-worker-1 Stopped + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-api-1 Started + Container mtusi-lab5-api-2 Starting + Container mtusi-lab5-api-2 Started + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-dispatcher-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopped + Container mtusi-lab5-broker-1 Restarting + Container mtusi-lab5-broker-1 Started + Container mtusi-lab5-broker-1 Running + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-worker-1 Starting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-dispatcher-1 Running + Container mtusi-lab5-worker-1 Running diff --git a/evidence/durability.json b/evidence/durability.json new file mode 100644 index 00000000..6a1fd1df --- /dev/null +++ b/evidence/durability.json @@ -0,0 +1,766 @@ +{ + "killed_api_containers": [ + "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3" + ], + "stream_entries_before_after_broker_restart": [ + 1, + 1 + ], + "attachment_id": "06570cf0-e36c-4403-b10d-c216f92d9696", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "status": "ready", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "old_bearer_survived": true, + "message_attachment_retained": true, + "before": [ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475559008Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.552220082Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.074661473Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:01.993890605Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475268543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ], + "after": [ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.608120136Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.67064591Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:08:06.296755178Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679298033Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679577406Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ] +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..4a4f4707 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18091" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 2, "first_recovery_seconds": 2.534} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-tests.txt b/evidence/final-tests.txt new file mode 100644 index 00000000..a03043e3 --- /dev/null +++ b/evidence/final-tests.txt @@ -0,0 +1,28 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.344s + +OK diff --git a/evidence/jobs-summary.json b/evidence/jobs-summary.json new file mode 100644 index 00000000..f0c552ef --- /dev/null +++ b/evidence/jobs-summary.json @@ -0,0 +1,73 @@ +{ + "attachments_by_state": [ + { + "state": "failed", + "count": 3 + }, + { + "state": "ready", + "count": 85 + } + ], + "ready_without_three_variants": 0, + "stream_length": 0, + "pending_deliveries": 0, + "job_events": [ + { + "event": "claimed", + "count": 107 + }, + { + "event": "duplicate_or_early_noop", + "count": 3 + }, + { + "event": "failed:invalid_image", + "count": 3 + }, + { + "event": "failed:retry_exhausted", + "count": 2 + }, + { + "event": "failed:source_missing", + "count": 3 + }, + { + "event": "fenced_noop", + "count": 3 + }, + { + "event": "operator_replay", + "count": 5 + }, + { + "event": "operator_supersede", + "count": 3 + }, + { + "event": "published", + "count": 74 + }, + { + "event": "ready", + "count": 85 + }, + { + "event": "retry:lease_expired", + "count": 9 + }, + { + "event": "retry:storage_unavailable", + "count": 2 + }, + { + "event": "stale_noop", + "count": 3 + }, + { + "event": "terminal_noop", + "count": 30 + } + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..7de89ff9 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18091" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline-stderr.txt b/evidence/pipeline-stderr.txt new file mode 100644 index 00000000..3c0d05a0 --- /dev/null +++ b/evidence/pipeline-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab5-audit-run-b338f8a5a02a Creating + Container mtusi-lab5-audit-run-b338f8a5a02a Created diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..e56c8156 --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,119 @@ +{ + "invariants": { + "commit_before_publish_recovered": true, + "publish_before_marker_duplicate_safe": true, + "crashes_before_after_side_effect_and_commit": true, + "invalid_envelope_observable_dlq": true, + "duplicate_and_reorder_no_second_effect": true, + "concurrent_duplicate_single_claim": true, + "stale_running_version_fenced": true, + "old_delivery_after_new_ready_no_overwrite": true, + "failed_registry_explicit_replay_and_ready_refusal": true, + "transient_backoff_exhaustion_then_replay": true, + "invalid_decode_permanent_failed": true, + "cleanup_preserves_current_and_fresh_failed": true, + "ready_staging_cleanup_idempotent": true + }, + "faults": [ + { + "point": "before_side_effect", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_original", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_objects", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_commit", + "exit_code": 77, + "state_at_crash": "ready", + "recovered": "ready", + "attempts": 1, + "result_objects": 4 + } + ], + "retry": { + "transitions": [ + { + "state": "queued", + "attempt": 1, + "error_code": "storage_unavailable" + }, + { + "state": "failed", + "attempt": 2, + "error_code": "retry_exhausted" + } + ], + "seconds": 1.941, + "replayed_attempt": 3 + }, + "transforms": [ + { + "kind": "alpha", + "size": [ + 20, + 10 + ], + "exif_removed": true + }, + { + "kind": "exif", + "size": [ + 256, + 160 + ], + "exif_removed": true + }, + { + "kind": "small", + "size": [ + 5, + 3 + ], + "exif_removed": true + } + ], + "cleanup": [ + { + "apply": false, + "retention_seconds": 86400, + "counts": { + "ready_staging": 46, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 46, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 0, + "expired_staging": 0 + } + } + ] +} diff --git a/evidence/pipeline.png b/evidence/pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..ea12eb3b7f3e1b2954285e8c123ecc9001ca87db GIT binary patch literal 68147 zcmbTec|6o@*f*>qsidNmEv-T+JHuE)LiV++CHoe}ZU~i1$ev~FyE68DNr`M@HyBK1 z8H_P8%wP=9IoEwZ?|<(f@9Xoai!SCj^PA&5j_-DOucxEVdXnoT0|NuA#v@e&28QEa z3=Bu#oH!2tWyi*^4*UlKQ!|4Zdf3DKo_X0ZXg`C!boGF_Iz8w2we#|J@^BXy5fv2? zyTk7YgT3^I+`H%Y|9ppthnK^>SJCU>K^RzGK6>KKz`%K){^y88)`$Va5e5bgRb?ap zjI~+j&+Ow=@`ehrl_F%!m@lZN&%t;>Rqw-xYrd*Rsvk7ZoqBGncaDRbKRo>Q?GOBF z=g#IoA{wvot^^>d`v(cqJ-4Q6W(P#DgFbI8Z;2-)q-yxuXOiRv5PgeYm+6=P@1qm3 zB{=(kzW}dKouB#te#NhI?iYAI|MT(AkP+O4|NZ#O5yta>KYt~YBwE1zKVN-1Zy}PB zD&v{EyU~4Rpxm;#e`U0Yuw@;x`}4)GPgl$RG)D`y34>_s(AlvPLxF z-7%@+4v$Go1MCRm9v&$@C{KDki3d-=qcI2mJa$x;fkE~6uej^$QMgW{5`&u^H4)5g$1Sa(D$Hki9M{{Z?xa7F zSJL`Z#Up3xPH~F79;=JLsr9x+hk@bKo)OY&_KQ_3q+{#k1)=;$L4ODKcS*C#8AX|a zy%s@)+FRyzuH?hYUtOuv9+|S9e_v0vhqgDGwNU1Qe9eVkE#jX~pJ8C=Pw8rk!C|IeXyz`C)nn@p?U>F#bri)xNHW&MXn0)^o z%HHhxzov>#pLd`@{TQ;tr^q)|Y;3lh4QmjDr^r@B^g4p|lX+liLNK5nKYhuj?}sXBn*G+}`=A0ta{E-P*up)hOXr4r zACn3OBNR^Q)M3m$S{E;zNE%3&#z?IkUqT_|+~l?t+#44!V&r6oTr^{NWc%x}AueL} zTcK(4!8ml`iE59bw{RX+Dk_gNk5Aw$@%@jvL;~n12${g7+t*oxblLjCkhGt96`mQ-l}Rgw$GMh;O;wAHk@1t=ncEr;7b- zV|HuLRf}q>&%*fy*SpWI;bF@|9rFACejGv9Kl^dq`;bPV(&XYelXivcdtYQ0e{Kct&3P~Mr;Y}L3$n~r$9T}9hh=-g z&9tq0(eqrfXz!qHql-HtImqzh$LXTM_~db{xX;q?M&Sl$LR6bAsPgkKk1)KdpzV_e zxsIR}l0__6{W3uxn8aNP{@1}cNc;DLZGG`@4}Y({4{6mp4IQj__Tzy&CiIYovvThU zXDat6fmea48!s{J-Ug4-JW`}fbVrhgYQl~&AGhcp&QrG}VCr=%GDx=#3qQK-ZZ5JU zgR`G1?10KRlA@I&f++LAwkc!U!QZd?f)q^3EiV6qt5*+%3+iRRLU`=&Z0NU>%aCnc zu+&Es<>wi7QqE()h@`xS(VKyYscMHHD7MTLnN2P5#-Yl=pIx{%P83vUI}*Osf7x_6 zS8bNGBF@+6mJpv6ZN_UGw=TWARy}C(r0RvnR`rEtIdMT=S+CA8Fh{J%g8IiK$$Hzj_fe_T^V;dnK6v}A=J^=8qvaIPJ?cqD zuwp~;EFnQQco^~k*)hpnL0oPdz?vB0U=d(O?^>@&1 z67Y0mz{rqj>MKx*PHJxJFJi~9&AlCn1$TiGrZ14piNBs9PIDD!jDT0 z1*cdAmI&Wm{>(zAUC_#;{wYYY(tIf|gV{)R$kc%Cnpy??OcJ$yh_uV{|I0#Y*#l1` zYai}GUK#E7`SYELvhOxs!?Fj>o1Zak!28vCXRBw*2b^!gvasJ`3}%Iv8kNM`T*k@5 zlU>G&|M)dN|M_9cyEm~46t|!GE#1tF=L7}@{@Ux;nHgT4$bk6t(t?)yQxjfrm)1Kq z*i8qQ)FT7m@F@gOE)1lj?GeQSuxwB7_=G^|vXojwIZJ59_z&ZP1^P4;_y1>OzN*?< z4@E$OCTd)#MoV`d_QJE1A zgPBk%TQz9PIHp-I=Yj8G$kwpBJs381(hXf7{J|IOchUuP`KVl6#ZOwd;eywdx{JlF z4cyWbcU*52=#kUsvgTp~SA;RDRCTQ3UU1=-gO=NflzEGPr3NTxw@9&sF}y{0dex~! zwfxi9u*$9DKdhni+X`Emh=+{z^v)@Jxo!2!E{LP$x#O-;TNrm3{I!XFpm)xaP_h zZ}wna+i2o$>0nD+N>ae)->7@jv?AM^L!0out;)U<*OV9mM3C%FhxUU#mu9vLXg&Ln zVtQMxC2PWVd4k+{Fw{jGy%xP3(pvw0KeYrdUVm9}kBamTfA-^TqqQOO&$}12-evvc zX>AEUD47?(&W<*C+EO5JMPlzdObof*wgoR9Y^}O1X`hSI-AZLsP1WcAD(@>`%_ zL!~|1b_;1|nsAvx;Ua7R281QX+&IGUi7Oh6;o0VwU~+53$K}l-Q1aFG10QZ*7tp)+ zSH!ZZA%<6ONs}dWiY-aVSaXQm5}NAKx^3bYYmD`Uv}TSUGfCAy2+VP5O~V~fiIhMc zJjC;_>bJU*?uyYqxlgmre$ALeaDC?f35~wXvcqc;os4~J%Tv`w@bxBqxi@8vJg_zi zz!P>`=oGHBth}mkd1sT77Kj8-tgapTr219|q`YmeO z)aJ9f(8GxI-1+y+bgm^}Hj4>6_@QNYDV-c(S4SfaDuz_c(0G|E%|ZbTIX|E}?CV+e zruN=3MAk~sP)aqt%6?!~`V=)mSAoUf>RRtk%u)l1=DMr$5a7~EWxyX^o%nc3^gI07 z8uE^gL>SL3k3r+bqSue6s_bZUGAC4}Q7{#yoB1$X9LA&F2LP#oIS$gD=A87>wwr|y z{sdGzZ7mN!yGIPj({&@MOFs~^>wST{EN=f1T_}K`2|WS%Gs~Jl^U0=IXqxXddI?X_QXK4ZZMavd%3XYiI1GO>%7S&K;;cv%n}Mw|e=P z{!7WTOky6zP&iFLF zbYCXUHcNMq?wZ$~Zm?qV&5%;xJW+@Ed-YQuf3o7Vj?RkGpn*?Y-#x&Qh?|hXVG@M6 zQ9pnfRNY=3Cj_5tdd>dpy)P|?(aT5*CF`D16`rH%dRXSB!_6UoW-B#z>)dTK{ClPU z-ZCdiIsx+|T2T!Pp6Bu^kwM3-j+py!{HJlrXF!9d4qf;yld!}1*P6%XOA`}&1p^G; zb0wYApNc+r{o!Z8!luvPvuyTdEo&8G1<09)+U$_}>E%jmiS|M9LPD=~Nq$VPYRQdv zj@d2_6)>#9gUtWVouQqs#5cczG|ARX>n2#qUALORZRh5%G}hvBx8)~Ke|l~%XdvC- zmG8$DPM~dLq4u%Rq{R|&I41kb?nKAyu@b4oHODVduEqZPNq55^X_nI75wl-5^SQ&@h(3VS`oeq5hqK4R9QpCMtk5}B zkrIm2I{2BmnuK=PNdBCZ;KtOuanXw_A)`2AoRRgIQS%SnECNAaz*WfJ1%qYb9&|d*`s*XF+ywcz=}L{#Nsik#bS)uhtn6{jQOL>T zH^_UH(^5UOsdtN_7Y9N{De*CCb|u9((lynb6rDl^*_V^e-9_WxEy-zhhpRvd$C6ey z7aKil+K<{b%i2NUDeu8OOtuNrRQGzJ9>6yUWHc>3FH{!je*!(^Qx~^b$9&<3%T#+X zIi)`35wF1cwiMQZB-Gv~_>QZ1-|Y67ta5?jy#5WW(;rt4H*-78+f7S$_giiTNUZEq zo$J(jeJR}Wmht|Cn}01SnA7P`vxJ)dh`hfRB2TsPWC7-~}R=UTyrywQ3u zc86`Xb!{w_ayr?mpJ~={jMQ|s(CcxIfuJnzc)t4xl>}<%*6r$f55A(iaYNe5*z{g$ zY;g6;SV;ow8Nq+q+#3hsf(OaliIbnN4Y&B-AuBh)f^YL@qk{m_R%z;4yZxjpA1(tU z8QJ#9Zw;v|0a{vR7sJ61Fof!bU&H$+XM}zwO|mwbxuq{0xOQ${it=qQ`KW(*KqC;Z zgzV%vO?*kE&0Z$mYz%NWO$u2BNAgi$z&f{G;~wAIyR^@hCB!g8NnSZ-b-)MM+8@U8$x^leRIurlLXoPpie zgPd0d3YfNnW4xjJ8|s}1TO%%``MDLjYl1Nej_51Blq@S1+{Ip=F`QHevES^)XLawH z74M_q?sgwKzhhu{^=bKGl=wu%@m8PXXLPkqzL-6uGZref7HyTD>lQ#7kf!MzE~R_S z%%2S)>rD*k;JV7YAS}4!;W%w77`wZO?#q!^&(8kRA6&JGfT?RvG579$(i7t|Sgp;Q z0*8F(d@tANt9e!~iFX?0^xJr$$}dM}^~SX2SRWwMouPqSk6^UQ8L6J#sdp)~LaDOu zW-<0H!<+nuX@T={7=SHvj&!3o3$j1Dqaf4~?12ny`4j6!O^MV?mCn_43Gl$^--vbB z(w`rZZh~F;h5a9w2GTKDNfV3bN62Od**^o^m~JFBUMhbW%{BGy0|f5)rQD;PCN*5; zti$r{(Upy~=!8kXw7Tdj{cALy+cLLSU;k!ov=yA*mYaIfp%4kERWq#-kd^GIT#X#S z^7VQBG>ng5VqmzTN{5?gOY)Q5#bltMMQ!|SXnDme?_ZHBW~Ui`{#jF8aYuKOT7LSZ zQ0v0SXK@B&hEF2Wg60;CQqL4IH@ZKBNMm`MDhum`vvb9CQT}R~MsHX8?zPuozU$XI z8~Z(cP@z&8`|`hhzVTT3&381*?!+z`&5!QID-V;p8c+D#U#1uBzH6t!4eCG+ zMb2;CB3m)rjNYvN3o#{MWDvD7{31t(8A*sqS296|qKbfXKHs0>+K=ZP;)t7v4FBmE7E zdvWH?18n-!b`tHfV3Kw)Z5!Ph?gfw@EJfI``?GL)!?m^T)tw0k4Q$fM1LtW8@rym4PyT+5$)% z`fT$5UOjB{Q#{VfmAEW+LCC0rL?=}&Ky!(_GJ^*?fBW+$X8N;3rvt&o^!@pp7T^W? zi~~_QN!r7T;$q*IJg^PmQM18!@J6~~^ngrc3d&)P_~zsVFUx?%B;T~rhqZA0#McmX zP92>-L6H-53}Fi9h{c;zoW2EGDJUJJVNnd)>&IJG()95(YtoCJ_@6qX5U+(_&qA|+ zZXDDIxMf~cf;TdBu{S9M>j!S3rKvT%mYRd6%t$+r0*t&bf~STA zU%0LJ4J942zf){Y9XGXR-AjvGkLZ+f^Wg(NLlbsa?8PsetSo3yUC=b($1{B^pPTkK z`v@D>fXh)}wE;kp3b7U+t7|(vp!$ZuyKDg#`X4ES?mQ_Z_&h$mm@J=odBWzFK>*``Er=Gz7n`?naskvFl3)XkaVkm~8M)1=?Ywl``jDz*sv>3c~Ix zjBPB-wgQ?FlHxIx9fq)0{IR&kOpwOOeX;2XXK5Q9D+D?@vxUNBNaA(%m`Td-U8r@l zuNxJz*q2ftN<`qc8aIF;aZj#QE$X61RS29DGI-&h)eK+zQ*^ZS+~?nNI-%7W0X{Q$ zVoD+SnL(XV3KFS=#A{@AD`ipgv*9ZNt}8raP7Z)NwPwBHKye6nQ2VnK+wn1S>o4JS9XUA&$mwgOOp=TF%3FM;ZMQ`L z;5}!54Qt<+Z7?(k56)Gg4S3v9XpIM7;PO>8kRHO_tIcL!ur1cIpIT6@W((hNbpayE zdB)msCPp0uWRVL3D=q`wmSs9GJ$Y~c0I<#@*5W2%8IS&>xX%MVzG=n+*8$fyjQ(ma z-QHIK<5%CB#g-ZiBh#sY%N{D|a?4{ONKk?k#-I-(?Kn7`W*RD8KwXEVya#-EL<;Xc zmZ#3<_X-nm+h5Fk=`m#^izkfdNqa~ut+{|Gh@F2T-5}?)x{wC(aqd4R3l#G6REnba zQoj|lEPV|enR*v+zm+i|z~F4cncnBWS&ZmtJA_8G9msUMgTwSA#BGq4twjA=Key_0 zXH`xc;|HkOdy%Zq)eeJA4?kXd77^1%)XtKIceemxtDHuqMDqT6Z%=!=9;eRd$y|zv zL`eoCWIVsqm2!+tE74_HT+(^0n6adT@zfr? z_&WR6t^_>|E(fghKTjxfgEhm~kp?`fs+*s}^{JsiuB8UG25vtFI!sHB(Kjhhxyml0?7xQPq7f3;d>R{ zd9Kg$9&;@$nP}01sz*Xs3*cYoH4EayTkUTXjeUu8Zd!Mp&yqX4Ua>g*CxRt+n>A3i z7b}hSL?$UYI0xa9HgS~#Pu&_z1d*b5D77}zI2v-r#menEAxi1N)D#JK)R^EcL2J zHB4?+#`N0*>DnJGs-35zi-2t-Fj{UYCr37J*k_^}>Sr%MECjAH7<~^DS<%%Fy4;;k zlNHZOPyhY(>cMPjycgh7)a_y*3_PEyBze!P+%D2l2nX>KXkip72-GGQ^rTBp{W z048Le-5(=o4)EQAQFQVTKJ-UEs{xO^x*gY((A~1PTDAl<0L@yA6j zaHH%cb278YMewZ0uKbb(0FM`TK=;ppP9e=%%`4~o83=W$z!)!_-lRY8jqv|{eDJwa zhGy0XEBNGpK3;|XzeWYr{=cUS{{Q|6f0CT<+UH%+%n9>A#uJNMjE|&QDYN6vHRYdy;OMYhMh42ZZ}tk)V*<6% zOdg1aKS*B;u;r>rDOT~I0xHgX>f)XIz~g74f(nlR5y{pp zrxJXyi(5j;5x3;9;3wSHTPar5portFYuGEHA{GrcVE#cc9ZKj(kCu(&K}X+RWqMR* zTxM_O|A%SbtI?Ig7XUJQkP@Mzm_n=8Kvxe2r}JSv77}CTwp|}DlPRE=r5Y4yPBG}n zKtsTobpaSCh#I;L6Ed&+GSw1Ltvw50Eh*{;S&M!^(X^XjEAeY^28NfF+@jB4?=I)X zt`RSby|@qXkSO?Ztrhzt38tndl5iM29?;LUffMy~eSOA%K6+`&A!~bD2p|_pfGnSD9eI>c%=9Lz%h=S?&MI)_ zCS3{jUifuR+;y^|BH2_|NZ?(bogZP^F6f!yJGu`i2M2B|ow93;gfKAJ%q)%q)cYc2 zZ$%)edM)Z+o6RWDWhZK!Mj1P}teavIf%4gS?DT;QUQhj)WyCeFlgWaYCcZoom%83! z5lY*xd;JW^o{!5+zCH#fc|V)tzRlj&%8%rupX}q}BBI_p5XK%kR|uWB;Wy(MxdNm@ zYU9Vt;%P^Y9iOy%Vpyb8tIUf~RTa07IjY^eHHHkK725_~mIwDhI>pjA z6)h2cA)l-TdL$uelMb2e!Lgx-3u}pj^c=&H#k&rH*U=1Mf(_KjI1~L)rF-kPvd(d69nj!FzbM55JYK~|vfLNR` zo_M8h-)yVwVZKq6!pVY*RB){NdlCc)Or4)?hJv9Y%LfarCZLSu^} z#Ck~f>w(p(ywW(|(P8Ykq|3x4{yNU1?BI+X`V?1nt@C*4GwXXMNGquFQaH-bXKv+V zeG&eX5{?2c^~`|JXY zWL;KW+|sa5Y`c$OwQ%C&d`C)Z%+0$g4{kv9C<5F!P~W>p7QPzqemLRBe8 z96`>4IM-7&^^3dh5RKJo@)%lqKjVt->mAPp|u|=;u6OLIcV1 z^D^E2+<1z@<^j|nVvP`lJwf}=BW%Rh8u2wzf^ z>6HYG_6|=FgOWtQW?|nKDS~4F;5JRZ^Q0x2Ze^G0=RG_Ui|mYX!{y9hzu3L?@6Qhc zi3!Mi)Bw-5>DoTpWdz{7V;F&pSv1hZ?{$#qk*?XUz=H@jg_k!|=xqw!53+K2kZEGc zY+o%cn;{&m6lI;4SG4)YK+ULA|588Net6)5wl3DYN~r+^Sw4U8FE}!mH#HM02CYE= ztsIc7hOAK{Wa1>SrN+x$r>Yv8dA@>IuqE@h;6^+!IAKU@wy(mnfSL``rolLP zG=VUY950;|~q%;s%~ zVhgd2y5E?mhFdYmOL8lXj3Idc{BXe=c;SN=Vnl7b8e#{*p={=o zaQscrt;m^WN4BT0pgh0YYEd^o2$#Q15;jG_gI838fX~d5Oq!88xw8{;3(*ZQ(B%Z% zr<_A5++DLFJ22~g`C6HIvxeQX%!0}AW3tZe>}&q`uk{u^8rVf03|&t};&d|QSLg$! zdA8}`FdfoGoO$d3af z`(=9!akp=zK#L%z!(GD*1My1M`o$(>a^n9C|3uL<2cF?Qt6$h@f4oL_$T&6lEd_-7&$mI1ISW)~(d zd5IP`91H+}x|!Qlqjynrci2QFa9t_htLF|tEVKzjSGZo`_1TyQM#1$M2f=_B?l~$u zT!SUm!o}~8C4=nZI9x2j?0x~rBGfTQ+Jc!@4xq^tjst+=->=Y{&grAJ>e52v?ATGz zOlJkIp#}^>kzW(e(JUHXJ_~m%R2OIn=#)6AL5Y;?E>eeYAn794Rd&NbaJ)`^g% zG{v7w`JM5IdYtRpltSg+T9A7)9k;@doFDjUDzODEL8JXjJ#I(E}CVawd^4OeaWyH{A( zg5)7|p*|M*i0|akYdRO=e|qxwvPT*!DOmP$ALNkKqEEn7E&&a{delc`KEN%HbhJnX z1!7gW>DC|sy;_4P(`z2Mr7{m6W&x%73}%qt!H0O{GM`*1)<&`Zrss+9vIqx|5u6q= z1)>1sxB!RV%PLo+tNcR)rcfib4(nu}7G^J_bIQm+NcIojF-yd*ZGfyS!T9uZc!Zaq zt&T`y-ygMIo2(Q)wb1F6wep!Fuj3Q53R}__P{HDf`KK3_#s9qcD9iilJIliBs$ef3 z&dN!}Hli-2JcAeu-WIe+P@XK`{32XE(Bdd3`%&2j$2aW1Jl(%zVWZm0I75b?SXN}G z%#lmm5WRrwp)gw$((<)ydL~1_uD|&2mb<}muUx0l3Z)qD>%#$7(r1pT6{laJ8Wc2$3Q1HjBI96p0Qp+9d zb-yT3QEKg)t+G#qk1?Ntsi{Dp`{-wH3jFa3BnjewCb;X#EdJwoG^&>D@LsIedJunY zg!EU+1S4oBs~p;$-g{4m6Mi73CZLnC+~%|e#^#PCQsjGv&mHr+eh+?i&ZvVJV{Fv! zBfixQo|7P={K~N;KaE7`CJ}4HrPB(%6}p9yEr08S)5<}WTOE|MZ28SuN1jHUlI3mY zSZ3k)H7se#-xkK>^P3)hCrJ26Z1f+c42(_4e!xXrhvJQqAxXX@Kaw%}q{da*qq6k4 z7(BYp4fED#e!B6`SCVZgZhZOx1!tU{B#*VhsjrL$&V%>dIcw$8Hlg@d6WI&ZZQLZ= zcvRMJE}*FSdV1;Q*WABY>~Gb- zQrdwGDfzCs)bcAl=yvx3@p^RM@)&j27vp9{fCcX^rjUy=(Ms32uO`U=-=ACNuvRob$Ni0dxpb?O0`*tOrqT)!#-X(#?s} z*TAWE2|WP#bcwzNumWOjRf_47F7|0sZqCeG#?A07kCFWMXq(K+X`MeUDRhRl3-G5$ z>>GL{a({g|$aAh538s81AbgWpFz32oVWzQ_tHoe)^EODtt5+yn9AS8O6Co`FqWFm4 zeuEv|2rLvg8qfi#{P{0h5x$38qq<~}DQZsVk@dRaPXNt2t(z~Xzf}jfCqCkR#8P}F z0Ebut0{DFp{u2Y2ug6^tLooqX{8#dJtH4)*(1ZOYfCa@tBr>|xGK?N#%QUAdEDdH6 z71$`07Cry~qE~CLx4!j75o-wh2ve7DWUyI*!2)sEMaq3<*+=0XBo=0+XjD1v0EB@C zw>%^WGk*pN^C>*^RxMbnn5CQIFM!2DSDt#>g73senGvqwX3VoN;IdOLHaPze+Z&g3 ziE^1STKs&p!{*ZrM?v&FfmWiJ{T^{Y2zBM;SHsXpHz!%?diIo}BjmkhqgO#|55a{_ z!HRTjt`#IK0$pkk?ha6;#&n3~j*P%*^;o`IffoQAmJ0$8c^Bxv5q0Jr@iv2SaJ}q+QGQIdopEg7X!iIEd0kwow4WobD}WM2UqypNe@& zu{01trZ)kBzEZ$5jwcop?7)QjDf_HLLyJ9I<11ozC0N{u5e2`?L~w%O6`re zc(?fN^L+%^lwHc3u@C|l*|ygkLguh(0}-Qr6a@g=CXZW;)ODam)zhFue!9;If#58v zI7kxwSf(%A&!Mm#1$fN4gbN+d|;3N`QXMqRbF$Uz$j$jCJ1pf$pM|kEbM<@gx zy=B2KN|zy~YMcze;l?$3ywd{>BH0mI4~61g94*UMo9>Lkg&^(T?l18(Yrt-i0!hqz z-U7M-g4BuXOSwmBZZli2?ktytW#WlXf8_8RWyi=jt)ECDe}z<(46OR+4?dI0wNyy% zR=Y!qEf;nN*-~l%=mpg)0F=b@#%^r2<$(@Y6QgwI?9o{z-vg!NdED?!Qse^xD(Z^V zY%!bakuzUS-RN#NHozC?HlY`RtEAf7cs2Dr!2>vB;ZBfqdTjSXIj>}fx7Lv|tjswd zLjD*vBfeq45pcTleq$|_sKprw7Lz{GcW(ND8|UOPCqz}I<^n&dE(MfrwtQ?)^++22n8o73TgqW#^VPa5f z8N~)LS;>pLWvrPGe&j*SukJs>Jzf1+*j|qkYg#GA@nnXgBM_%JKp2cD_UE7#3JgNl zzrWy=SdlU>-EtEPgg-ZYSSuZ&4k0lTV5EInu;HcD_uD{FP6q;r%yN(AZH|w_Z?YA{A%Z4EMPBnJ1|p^&X-a<88zlHH4{fBR z0bL3X%3Qg12)6CCzq|>09K;v3I$lgDIK?gSi8QQ!Wh(3BuN^H@Ha}|TW;K$(AuzQS zm?u&gHRw}g1*NR4M}pTyxOz+V`MU*EmeSqwygoF2)4wd?Ca}#ot`--#5IA@L{i5&5 zB(b6~aee3nEJ5jz0^l9UL=|>unnFrVzH;dYjk$9$G@7YB8E`8{55_W! zxO;=75Ja=g<%SMGm%DB-CH*|fkvOs)hw?)ZxV2W_Ehc_GiQ|Hd!Aq4AQEW7VA&4zE z=##gA+u2$gy8Lv+x@6XDUA$Iq$aAShV34wc2Y$#&f9)LacD>RIj)$e<`Nb->6=}O$ zfi#Wb5dGW7x>Pn4c#>Yync`~7i%In?|g5lzAdK?LPfhN2yyJ^RU(T z!ix#AK}BC<$^4X|NHRvysJGW3R!$(hC`yW{E?7~T3#raIEo6tYU=(-*z}wWu_iH(B zxP7-{4E!Yl2M9FQkICz(U4+c2L>B4^!s|g>>Of}dEePj{MCHFvoNrHiA3-#)wrGZ8 z=_@xuBEQqWW;4W!g^%N{Nh|yK=Xo~o*_BS=G7K>~=)k8YRHF+gu1ys+p@x5;f$A~< zz#Ea`V_Q22gXU~V9)k*W^d>sZOei53E@YovoYCzrfvA>LXNYb6xIoy{@T3=lqx@w{ z&XuMIluNLzZ;iLMzrxOy&5QEQ`PI;i;AVf%u}`m@Ps>5Cw}Cy3cJp72Jzo`T!rXU` z_r3U}gufW=$G;k#Z=l--_IX@GOS|K{{2u94I)?I3C!^l~3THZX6h#5L8e=eON=io# zKH7n(Q`3^J_xa-Z4_W|qvH7l7rT%<2#y{8fEW#|xP+FSf5C9wj!8qM ztH~U}25cg5->u8&=%C1iI^NsnAT`B$gt;)*Y0s{GSWuP%tXTGfut~pt>4$?l zD)*J{98p&o$sn~8!uhiIB{lQZ@_xt4jeOUsP85?wyXK#VTE?tnEb0=!8cEG6I4d3V z(Z4>AZfAHqTYA4v2>9Ywpd3szf{suOFt`fp$xCih4;mOho<$M{(r5Mt( z{&cDvXt(DPhtn>|S`tuXM`}SH9*Of^pIH$yD*gm-19{?S^>!oq8tZi}+Y{z2nb!60 z2w`BUlXt<2sDS?gpaSJU_`%?UwpSM^k`k7K^^U{WI=rJe@Ae&0u-5TAGnB4R&&B8glzIfzw9VgU03o3v$v4P(_Du zJ6MYpSQ?>O0ST3FxGs88l;A{<2xn@EGsB`-0lJT73xcegM=Dtd&qL%W!-_ zYoex_jo?8#vlO5mBnWQc)71zBj_%1;H#jrV9TqPr3)rX+bJ=_j#9Ij^K0N{V;n1uz_;{WK-mR7{R-L;+O(BWeJ>(m)Qk9kHH z6Xxlay-|<#VJii(5ArUcl->LOA0sKeE1oPCk8gi{8`TA99o~4NOI%JVl`k`ueU+79 zo?-|cLAGpanzx%c6>4gmB3={}m70`j5{nu80ipixo*tMjT=2Bur%}6EM6fWgv90kP zuv^krr`cER(9-qkWrlpfhnH}En;N>%jT()Y~0$%e^^KXxK1dL;-xh4Lc}^e~8*ns)8MjCKZze?B-?c#$m0?9{b0W^i1A1qOST_<%u;w z2K8fu5ZKcFii3yveJu9UrQsnLJh2HS=%et}8)B*W0d>=-t=RjVU8l>JG|;awANjXm zp?%)fMRXk<*^NSN97`J2ylb@i1kEmVq`;e(5COu0)Yi9#yRc!|%1l%HBvgu8>N$w@ zxNLkKVxk0IMm^Ego?SJ&=<8LDtyOrV@M-ZVMIczu&G~8ar6`qt8i~8X-zd{M zsuHkc|5(Pp|MMRqvx;o29r%p^?m>*|tAJxmr3>h#1BVU&6s!%Q#ej{+vJ2f+$P8FY z{cLBaIvs*VpCRiJW5He(qXik98yI~rg;+)R5#6l$i7liUVbgL8WozqJYkBdv-v{bW zB*!K25FW*R%&=h_eEdr3vg~TBS+%{k91R#6|1fAt-JN>yn=*dy_kKk0 zzGw)yql<^y88X9&oIg}Y@H;uMB$Aokf()Bo+)jeoL!JCS0kr2i!U0Pi*2nT|0GJmG z^!?1SZd>_Quz#iX`+0m*vjM|1JrCB<2&29E^yqm#q@dmJV!yD2%Q<1QW421IpYv8W z$}*=`cWD4lOl7UAA+1iX9x1f?-8=LT-K+(Sz~{hWIzNKHeZ9LHSG7R*0^P#5kL%xa z8^Dc4$EZ<(P@E7Lx8gIShC$P@<#2l0 z4+ecYSpO4P!U&jT!3S-p6RK1bS@S66U--AJv~?isl4fOrrHg}ESWE0&LR*%U$DC`Z zX_pCq<)ltWOkU@b;lm-o^W50vw+p$!@4cX>3}#XtdOsU3#RM)0o>I46S|%1AXEz>v zVaFXQ6>Qd?IKl+LziO1nI0#x=i z$#Z)S+w6y=H6x|U;7w;tOzIq{u(|d9yJr?#B+ffWSgD{dRu+<+u_TI8V%Tqw7F~p` zW2lgvaw-WDIk9t-%T^VKg7`|j)yl`ocFg(Rp!O$}3xL>*Z8gn!V zkXfP3QDhzx5yC$2)$o0LAN$zH-uw9WUwi%WzQ6Z}wVw4n_kG>hbzbLrUN>*0{7Bs< z7x&faWbzQhCCB0PsV`^UU!SA)$mUe(T1WG8>IWsm`D0o+Bf9o-b&%Zs{9ff`bM?bw zv~p)1gLK3yfEwE9^YgIooBx?msG%`11m$hu1F=FI8_U3>$yMsoq3nw_zlVLKncCtx zaa$m>CozFA_}0%(I|GlcM$`U)o`Tr@52qD5&OmQyskyVg)8(+vApNCtyMxxNtRJ*n zy=pYFYt6Fg1;HmE{vE4-E-+xnr`|*Nn?I+y<>jaxf0Fm75I6R7x}rxP6o$U%Y^14M z7t+_1VRC~*^*b7%Ldi#Bb9_e(ojx4nYZt*Ea=wxHYJNaqJJ&`|!^(r24_2hUdgEqb zkT%M$^oaKnJ?Dq`tc`UMXq1Jhc#P!swk`6@36n+VwMylY{1{IFWNJ~ zHh=$vhK4F~`}3d7$;-O`{p2+tF>wJMZwVouq6cprA!ciXj;$&2o&i9828HqZ^Wg~J zVWcByf*^1PVgy^asS#_!WeyW0SHk&2snZv} zU^0$hdK%G|aW?BB&7oXu3T*!32 zK>y>RvhvruAbKAbBa2We^MOvfjb+ziLcv_hIJZuD7W4%ke9FwHR59O)ukfAyDbfedR_nQbxRyks)3Za_ose{2XrYo)Lw|qh zrx2QF=U1J05Yc$(I_uMYGSNt+PX<(PNj@0)ac#8_xmL~C=PSEYy5&IPCx%u8ov6gCqzpGLQ;>DZhAoa$* z@Iw+H)prj)+;N zvXnxXbH6(gr=Ap0LFF4OZS>Bt`$7qH7|w2NsXG?{TO--`Cg7Mw8%5BsI+iQ?!Qb%t z`Fo6g%v8Iuz+h@OLY%BrVbiK_6-Do}l_6VEgP7OqjThV^N9l zlH36$Pui;&G&RWP&w0F(O(+bzsiouCt&QY|cj{1Bw{tXjsH9xuUsZPM(%jc${CcR; z*EmMkc-;OI4X#znsi!?M$gP{-fzu2RPqPwe|6DHUdBwF-AC^Ydna8iO+Njf8yKg~N z!&>fsi|p)>a5|?(;-4${l98piOD$HmT~^zD23wlZf-RXoi&#B{gNGE3YED69t3{A?NaFL}6%V-yBTrtqLSlYW4H)n+R+_Dl#-DSfHP@HL*8^JYWh+Vg3s z{Y_EvlZIuEv_1gcDv*^>jCxv4s-rV9abu$n?n zigB_Fr6B+|tSm$)&J?~6Iq}rx1~pF=7V(hKK*M)zj@B2dS-)?`$vP`eGa@*H8Pk(u zLgq>WyKw97eEc&mTr%(WCVwBw#`6;hkdq5}UUesAXyR7{JaFj;3r!8o(P>qffc$s= ziaI;7(ja|`D*USTwQL#p$)#hHNjduy zV!V6*Vl6<j3t%61wGSieiB(Lo<@s8|A?_XbNa?0-Eb@19=T<*6 z?%uTTV%y31s(bc_-NZ}SFUgF{^p5=ipu?q04QJF35R8zkA>yM@_3yi$CR(vGUff*K z8B6^U`A6!Z(hFM}q_r7({{3HxTz2ebxf_CqrK(@&KRRHm+Rq&ZuphX;*V zSaKVC?8CPM}=R$puwK? z0b5q(GWKVfiuMbeP1?Ll)pBdK0H*;|bjf@IyD(=8rnnpIy}m|s!I*qqt=<3ir~FPH72R}F^=^$Ulsl<*8l z9q%2M+;4N>W16$OTlVMTg(3Il%6X352Ob-IvOb?%b%nWaw@+BqHY{Aba2L4zl~m*$zvA>HG^vkK1zO)?T;tkuA|*<@0} zx)`G9){cEMecS8voGfSaeS`_M0KYDTXT3Mp(DW2Rbm)8s5IL7vQD1PJkd}Gw_#;DYniO_0p zl{eQf-sK7OJ!?18CP9qN*yZ5Z@(BCJj}16~bG9W_K~p`y0i28SNU?PHa6kob=hJl3 zV>+D9<YnMoZpz41lg;z zH2ck~UDGYVg2f0+cB$Q_iX?&rTjvBFPLlvz7XioXtSIj)cIonGW2jd^{NICjSroEZ+W6)jE zokN{dxmomm!EbFC3J;4!2fD-C zRkA-{$m)P@F4c1*0m%;`c3o_}y;Db!$Y>)hALnupO~u7s6*6G;(louEfZhk?#pl_h zno`H;)x&o_Yj=)xMZu~Ho1SVUg;taA(fA!|lHIsMNmuYa%n5vnlk1T4aDDPE_Zigc zMyDDb^4%msMdgrEe49FX+m>To;}lQs<#AYuB%VIoojC0urBRcAyK45;SD~HrKeuMmJrcDFAo8YHPt_1Di9 zGNGov(++#D(~E_f=p?jf9-{nkPFqK3A!Jr#_4ZVMM{%($xZT4WBx1U{%LudR?InqR z;ieQH86x-Q=)Di8za9t|deHF|gIG$4@M+SM$DE`qpI`aXu;3{nz9{vz$4q}R#V=v% ze&^K#qbgl9yxYxSDK{yzYQvrv2M1t5_;k>=Ipe-SV^4qa!mQBI7iF<&3sYPvG}l@N ze(h`f9rcqJ)y=KEu!X5((Z^Rd`+KUmeQP;KPd_U$i<^GZ8o1ZEp|xa+Tdv6(BJLSV zU%|ez+)KRiHolW44T#U$mC2RCMce0X^}lnB6%W6_7L|>qZqQlJN|%Ez^8w51ETTt-l#W-#J#Ve;jN^ZJ9PYX7 z!Md5IH+2-Q3F)u~AMHG}=o*Y0Tm2&mi>xL==u`}GC6W4cgOOC2ob)aoSnu>_W%9dN*Y7MwL~O3dM$&cia}dakQHK~Op}4fg_C--F&@I&!DBnT zOEc|!wDbfup6~>d8$*5VU>qskrkm=o1NTD6$68{MH%~@XDV$D6iRZKT@_9_~4t z;yGs*n%9|Cx_cO2nCG4^y=;esuWAU)c;x19W%qyfpGsJ{^-@%S(Wn?Ji~(~+Lc}fK z$6uL7W^Mhh;;wK_U}^T)+wjM8U((a`QvM|gN*jZZAINBdSzR^L*xk9O4!!EL!VP!K z)N_SG1Xa%QdnfHZAI!+g&(oRk)c%)U(pCL0!w48jVCYeHD$5JbtqfMvBK`EGJ`oZFX{A7P z1-0rP79RDiVjO7#Y0j6hdMC}0@$NTFLf>+-7wKzorv%*8f)zkNt>lF zyd$m>BX`WE+8l0`5AD_XAY3bLTmE=L9yQ`Q4+;#^@j>0Wva05zT$7$iulRN`tpE-smc=#74_x7{qNYK4Ox|`*SZ6 z>DhVh8Z~1YQZN=3`RKSAj!Lld3nV}F(KdYyHNR=wb31P1(oSV3w|OvSvWp)MGxYqz zHZ=wG;>@w}tsbsMO`L{8Z(P`zF@Y} z_?h9E{AMn+Lvo@tEUgz^TyBv4oDDWu--S5@cbKBC5UG9|H;9m2zr%ms(LHBTE;<)7 z-TZ71516G^ijNI)tu&IcTefD;)~W1f%q#X}+$8>v>PxtAM0Wah(Nz@wJYiyxN)cJ% z)}{6$qbT-c`#jJ96NAjkap6=+8sLgZAXC%Uz8S%+zphU@-8mp0yBm2Sg*R(#fVrA7 z-*Mv7n8&A^-LK}!7t@?9h|G^uI-WO%v=!8sFr=bQLy_1g5jb;NR<5n(3@AXNP7-+M zm3PVML_jtioK2m+rDbD~G85_OH>W9xPvI$?W3 zZgqtKA2B>*(bOwK_WZ}w%vG|v6uN%=H0|ch`RTEUW|xrtoy}|#-{go!+5d1i zSGuCK{%(A@|6al6F?0?0daP|-#?Km{9(6iM%ar*ORI2qUybT=q53Er*5 z1KMqhQ-ho~NkH>@VCf-@r8+LP03?!#W@#T2sG|o-?~#}5xae}9RKxac?UY2a ztNc&<9mr3zYh=&C7Kn|EJt$2&+U(wPB*9GG6i6w7NeEKO?LIj~eEojp|AHF%bOj9H zjogsz1l{}b_pTniCCBvTZaWARt zK<<)++N!+m-vc{yRgUS-8$cyh#9)vZcM$|9$EN*U(R2@cre&ibdt5H4GO7S<3PO*5 zQy{f_I%fbOED{&$0!aKwsDQP}(@E5(EiW`Vq5oU;vkjMW4qR$mrp?mknKxLC&H=B` zZq;bDL5y(DG52lK?S9(C8F_6!=Lh z;_Zo6GQ)tp>iDsa*D>rTy+OjGqvs&BQU~bC)A!mEBnx7_-k4BXPq*262icA~%M5T0 zyh?)b{_h20)@UIms1C4l+r?D?XTF%cluaK)o zl4F@f&&gbf0`eA1AV3QfB?4pFPvA)*BsnLPWcL>Ioz<}J=IB!iB)?d6o>q>eO8sCq z5_rZvVzL9@I@ahD0jF~wszC#w_U#v{tZKrpC9qrLir?Rn<|o;##|cutXc^r$n}RWg z$l)=PxB`M`F;WNZy9yU>)B6a?|6Hm+I~|{NrVWwxWJ;gK^<6wQ2*_S#&m(*Ul|8e_ z_k6B_;H*~jo{9%{DecJCRrVCqZgRT?-ipvIXF$v;7!(tb|3vE( z947=jBS_jFE|t)%5)NIcC^>YLV8Vv+hCMYN`+)Sgm^pk@!$i1-$)rB0q4` zy`LyBe26$~SbK|2xRek#CPvy@O&9+Saq}1gkt=ocpv2c0Wb121EYGy(IZe9I?TkuW zQ%wK$+QK8*J|)%T32_QA5E&ZfPnd-MHFsfm#Sk&T8>*fsx?>A)Iep%SoZdSMuQIbj+}R5cMzEIv>%snNWL?nYD^D;?|0`#C9X zk+iGTI}ft{Yy!7Q>Rs1QgemJzUYy-IGBI3l9sNbj-@VAumyZU2^J+HZTGkn`frg#O z8emsV&CQ0k-dx??j|f0~Eiy{NVGYtU%)@^p$@-D?po;vK!T=u&x7`7VHcdY!20m04 zfZ%Fqt=*b+>ChrU9`ngh?xW#EML`ZW;GlVT-#csR2S3WhmRe#mv27j)N8RIw^GqQv zZ~8Xv;j7K>3a`;^DCgQcP{opV;MPZ$k!kfEm5D#*$jwcpFR97TU2*+vvYj=|=woyR z8TN>Cm5AKR*iY;a96)LsM?ed|B|B3#Z`F`;n~h=>7F|2_#RX@lDMA}%TtVI!5Nz7Jo==N2kbbldaSk7-wRdHZhBsl@+FgD)#RB#EJdTC;UXuaP*1x23AWFUW z8b4#2!%io?y60CU7Z`Tjp~9Srn@K8L0f8$Yx>-K`{K@w9@ZNOt^7YbN?{8i- zInDd{)RPtrgX$XWpz))kzAZHLNIb#cT%O#|PHW##HjDR0yA7vAURC$e_?q ze|h(?Fx}ZG&cFkUxI;c@(bqzIO@=@WIo^!k&LEA)mPjXQw(_{($Nq-+mG(y}cIa3b zBrlB-vJC!A7I&-q{+ZO3L*{qM0}jl?(Uqau|41(CJ4;bGxB9ymd~`jLV{3Rdz8pdI zJ6sEuN5uJ*g2Z)rZM! zbWmKhF`L+iQ8JaeBsZzM9}5r&l&j^dRAiH4_|)gLHcKm9Dx?}NHI+U&7~H~{{Txcc z;jk<7qyeV=K)c-e;`&m^JiKavljTpFX*~&u z73+?^ivurix!W2^V!BZfGnLlV-uit0TWl~QE&#_oci9o zHY0KljaC>wK2=;acQV?l=;xOe?o5xychi|=S^aI;z`CZKcs-5fy-%-OQ_&!P@z-;J zyxxuCd?=Y-T;>T8&j$f<*q%_VEMVk0D{(UA9NWoFv!=AEZDB0Gj`k?0h;UbdaA@hf z9;f8_vPy{(b^$7*MkblU1P12(Nb|h8dDX^{TUtg3O9Xzro)D|KRhnWa9!{C*m$11h z98d#k-yI{42e}tX2gWc7Eybw;>jz#2XIhnv2~MbRYqPa`V$Qh_{5~#dVFc#!ZYS>9 zdlAuvv6{!-)Q!alJXASiqP+YML_0;v`PY~5*8Sd1vS+(Fo8Ok{7P@5%93^JALFmzq z;{=REfNGYzX4`256A_BI9RhQUKg-Shcuv34igLIfkcI}wyASs=R!0;gv2sq^^g}_K z82g-3G%4stK`Msc!1C0>-9=DE6frM-`2K zn9uUY*|$42HfNk`38>hi>Ap5<><6lrW~Ya}y*RtAmaoFOJ2(_6rz@eS{`DJy>lgN& z@04>FMzrC4q(%7-8KYI`vy3PiJ}Et<1-&l1JQ`h9>-&GVOsR)E!={v9ugD>e=KoMk z%?r=d{+SILlKx< z{C!`QTXmi4WQc#F^QAu81wu4CG75LoYL7h3qghi>vsTC8H0g#e-p{}+Auj|g;W?O) zuaIQAoncSr1hsCSL&N;}1_Ce76EA}`(!FQzSq^QwbU2vz^d){~)@q>=GBIe>(dnQj z&(;uoQcnbmXp5G!xwNnMjtUm(^UK|E(xF;(vb5M|B?C1tHBCzq3EO@1M;%Dd<8QZI zYVtlU`ri#9CDcTHF6|MIU)&NrevHK+T=s#?+W|W4QY1YDHFRmhPF7C~T$XHIDqZlIqSgoUw32he8byhAW5$h)*aTgYy~+(7;9SbLe^E%J zO&(vO>)Hz%x+DUGj4$$vA z@3VKHi9Crky=;E-B=s{K*Kr{<2L&}K0qA)L?$TDULF}iDE1I-{N^R?V7C*I!rWJrY zhsBGGb1`g{6%QughL8NuYgdk$w4~6%I-=C7#8nox$mBMmZ&iaNIy4G$G=!g>d^@iO8&ux%nRbbIhB?|K+pb2xunPIzu;{uPEEcOvMB4WE zeQDR9TzsaCt>chriJhG$vtmgU@Rl{NaTKm+q|*hGU?wHfj=3U>dMn{R?fy&7iF*d2 zrTX@E#i8fa))g_e(XHk`!v5z7+VYz)#Ca{#4roA>xmhTw?$I-jgo|>$v(H#!RU}Lh zBcNwh_{-1nrxyE4wCH4008SAjuDz%wF2#?-L8HDZTB>p7X6YufPfW*x!QH1SUr8y5 zDk-4+ceq&p=X00ydXYIIek&RYHaug-WmPM+n7;PbPBf==KR0!5mx*<;AW%n_ocMcx~24IQlJvqM!UJWH_C!W5951MrIr@u{K8o3De z?a2V$O(8NS@^Vu!Qz&6U$TmuwI0I-!8Jb8eE`aL^g~kzQY6N%#VOXI%RAx#3M}2`9 zk&NmJfEI;3%&ohHZQ8_VGZ9CU4qjU`(vL2T7@>X5d7{-parpPyoT7f9s5~cTXzK(G z%r6xKJ7AGcw{s$}0}#}+=xN`j6(AHU>g&1EVhL!;C~yglO(ZFu$z$L_VP6L36fPZJ}UPpeN$5H=>dr$}0#MyB|dcdv$>)EUSzBQ+s|7~H%J`D%-|7K^25lb!6Z6vRSS05E`nI?0jhzg z)sOy>z%z8+y?vf~AwMTz8Q4pGuaI}sLWx9V}Y zx^%uhim_IN0to-$S1B#pg>Fu>S|!`?$BBXRu1yolMK^CH_ECBYq>7&G!~g3QEHfI= z>K{c2dvGzO_b37sw>&8XCLcs?@eAM^B`jfxgU*Any)qsCUIqD%8o93XRU@VevWBT7gbc$PT}yWvAf>;{Mjt}Cdg)}aB(0NumTD+xz5 zZ&^SWw1nR4Bis{+8hsqb8Qd3cDut4bL<&ptZ?7F?ulwI8T(!$J=z+-w1^YiIT$rH% zH6yJA;`ukdB%O4_9{8B;Mf$!M(>ZcX>d1^j!hf0lML8qk8T_Q?0yslby*1#2HkoVw z`&wuITD#B|u98HFPiZVP$hE(bSd?5*Y!4rigc)9N_X^d326#l&D^En1i5 z>X?m|uftq+S6Bx3dc@-PE*Ae5)?HdW@ZR3~h+L7}Pg;|%rrvEQsGFVB3ow1&r7fuJ zr2Mv%qL$bS42|xan+S4ugBAKvEvmxllf5`dhKWg+;vY6GKyLE&UTfgiwuej9h>OMm zO}D+Y8LY35C85N#IDX$DBP8ln{&;=@W@ zGRL7!2`6ji@L^$Y0*g*um{5XdQEL{P~x9T#UGk%e2$z|~$;;v42Ck(4dRmxgnj(8AzE%8Dl*>Lmj2SHJPaB3 z3>-k-!$oF;M&iEn6o^-by{DoZFUFBphm#0^v@a6rLIkR-Fto7Vo&}Br!rTfriPLdB zVB*Pb;@r-|ma~^l;^)G1CX%s{rrTb7@>|r`|9;}`|65Po-|zK*cdqaM|2{=Kf}tC3 zIXd#wjBq2qD_7DkvOgI`f_Z@}* zeTZO@Tn$3>(2_%WH*OB&Bf$W{sDCrTK?AYJtJ=t0^LAD6`B4Ma6?kna0`#ypL{!x(ZwG$JRB#7@!Nq3)6BhgJH zK|cw;o~9rGk)B}>S`0@VPomAo5Rn=rB{oM50nYen!Deytj)A2@_S$Zzzjc;OP7w}z z=<%H(OA_dt+)2XSB%m2_FZpFU^zRa0Cy1;rJKCRx=hD1#6PRAvt$&H(NQzkyf5evkmjjX3|5&`-iKl7qBi z1@e2Aob5)_NR*(XgssgcB5EFzE?Cas&J5S8>@I8=b;Im})0ZCWKNI|O zjk3R?2F~SVj+=pfITzU_6yaV}#}wc$d!r?tgg`G&=jbRU(;jC@YxZK}F_;Iyn6{>|}Jwg|9503gI za4jaM0n$bQ@y_qv2ZpxP&#oh-{tJV?E~xcW{v?Zk)1r^~x=ZSVC4|p4@MdBf1R2aO zWm5yIAG6rXaN*=RSIpol9(2e!jRpHPW?lI|zzs?J0X?8sG+y86DoUT$9(qFEQ*uj? zbDC`=GJl=Y0O}ADA=7#B`pRHFmJk0xV)6_F{o&sf!m0I}!$bftT+5nW)COWK^5|>l zejmuI{w&;k2;+^UZ#wFL%$`En-E-~bRktCm2qU}~q|*`DjTZzmUB^ww;UUo60e`A3 ze?PSJ_nz4>gEK=7Ii4E!x++jLHOMKdal@*R4`8IdmVf0}A(FPjN&SkB%n(56Pi@ex z^nghpw5F)eg2@)Hr_5hOu+8_0ev2#w$$>PNnu+)5NORj}49A-VnN_%rZ@r_m*4 zQ60rRQL@cQmywJ@_Y@}{V#1g{6ZU(fIsMJZ8{!YGz&G72>@=XA=`n3j`nW+DzD!hZ z^)M+Wq6e=^U=$5)Fz_?Q>`doee2G^7jV4oPq4hWYeMK_j4@t)4wE?yk`IwWH;CcBW z&e6%+4ZnZRQJ4HXk0;kn)A=V(kpJx^{y(Q10{*v&#?Aq)jldpHz%XP4#y0EqcIu-aV*&Ae%Wel%%9kgU=m4a`WbPEPM*&DtlpU_jJ|VUmma@qsf=3 zeV0B>j_d}-610T}`UNz1-r<*r0!7rRneIV~(u0YdeO#&CKXxwZ`lQ)?63ixlrT&;9<2>z_WMl|ZQ|ingNC zo;k$q8zzx|zCCgk8%{IG?zZ0Y41&Km1v4;zzwB(#1W!`By^YL6;cI7wO(^Pdb@9x< zC?j(Y;p=TGJ+SC1Zs^lv#(f|Y2(6Nt2y}%Vans{enSVqsIZd!rBBxQlsFGr;9?#Sa zt&TIBJ-*40V;1%kmhth)9F3^bnft&00NsRlL{)`HeqeLQeK4d6^?bwv#p@o} z%j6V`fh1vRD5C@0IFewb{}{q}7FlvKLsh$Oaz@5ieVz=k+WD%i9)KVjjX>fNOBR7b z4W}AIQWGi10Zv$QH9pH=IXP0T$rJx1VUZQ?2wuahvQs&&ghDZ91gSn8t zM1twm>O@9b4d$Zo#2GmmhT{q&31G9$suI7##+Yg0%`kt9^{7 zaD7M-=9yk;Ptv;!r~N}hOYDI=g>trX2Ljr=bLH>CEbsfPS{YBS4XNZ1KOeQvS+#!y zbR2)AIxLn19xlgU9jqko4@Bfgc@kH}ate8>5D}OV@q^S(#b*8rSwhJ;IH&`MuI;ea z;Um1U9=JiR;A8m;?9$Rp^^IZ zKrAYD88TUpvgd2K8mZwU*%H&(=I|e(AdMX#MLJE4eOSfjBDAXtin21%o-wq*kVp-<_-rsZD;VH zvLK;|ptQaz-$r_6O1j&xJ=(VhLx?lEs#vp*a#DF#;v`^4DsFatP1R9N{DR1E@{c{(>>R_l;LB%mV>&WYs z(C8GpJ|wb!VQuG8=EKW3#~alV`IfKv7ySsv&&wZ`#jP5Cjk9OQ^RnKvKW!NrnoA}h zf_KHuzWc2C?F)iVj>#KZlL_8sW0*c2P8?}Mf7OYL=qc2nc{;CT9>sPuHbE|c*C=6# z)8u2HllGUY@R22+2TRr=ZAfjL2^L873LJedqhsiLzdCVDv001^BsZStb$Bc6H;%M1 z0*|!W=sm8&YVywJinI1D@%=Gbf{8{{&MY+=DW3*gBHV8R^npf<5`? z`AZ1ow8ww!|M|~VKyKsFPVZT@`TYFLq6z`&*E26JxAc!X0pl$1x#s6~T~9*cvVb#h zFWRaK@#FmO-eJ}f>{<4Z_H?8POXUy6jb&^Yo5o?OVBG3B`l)^A3*5=^ONsn&$8o32Z>AA{U#LbW^WsbB2g9TE?n|&tcP-yKOfcgYmMBUIolri9w$bg z$B)6}DKdk~%mQ6fv{dcy_P)H`Gjx}8Riwt2niD)RLfsW>zklz~G`Z2wq?syw^JId1 z?dRYgpmQONPk5Q*pPt=62D~oybs4~T535TpllX|0bT^^XVAb`G7caKo^sAt|87a<02HBNM5T*bzJ!=- zb)FEMCsTJR#$U0ER=I|wffSBRg1dR<)qVF-BA#q#;y>1Nwuxxua&|=cD3g=6QHTUA z)V&!cTj(6AY|U#48FplFk`7yr1|Q4eYwV=vQtQ@mpP1RWUa5pD8^PKzXCk^(h!_E( z5KH6Ou4p{Rd1bN0#))!=7kr04X$KuiwPE2_ebR6E_bk1vvS@M0k347()KFN=g&xpi zzK6QWuwImCDooKzHDpsD9eAYWhkRVI+S#8lLLYTxk3iH9DRdfRjoQLANc=J_A)V$C z3%6=$Bf`nB?kww8N{J9dOJBC^!Y>C>SGHIaDsq$mc9Wl%zkl+K#mu8nB4jAv$^i%s>=A;kx zO0NoQsF-Hn`F%N!(jj`TP2W~MZ7Wi2g4oO>ss2}Q#Q4b1m-9$8E|RJsP5)Vq%rCmG zKpKl)2lk-?sWktn*Sz18CB69zClRXhlXtf76!@R>XD!vm64!U6wnf<9xmEg+JZY~B zSr_Nr2*jm@hL;vn-e4PQ?zrP5=UcWwDq8gA4Pg_&<>3tJN2st#!`wK>l-Ocaak#Xd zM$5A=Mn|-U2YeW?TW-f&lSr#x%9z5O1eha_@-m`aQ&!FU$0ql2Fc^BKRG zcfLaf5{`o{67Y2d9CqH)+qq2+SrFSxQsM}MC{$nM>7MD>uSv+6%{0{5oA^Mk2K7`G zwZLf1e9KPWbH_7iiNgr{)r#GuhbWiTb+~JQ#i)LMd8HaHKr3cNvU2M&!Dk%Gmk0m^ ze_Rn}XeEk&eZL`rBxiP^@uW+1^}BFPQzMf=8UgZilx6JB4L;A_Xf(fwc`b&P84r`e3RQwt@bpS^hyP2P(+l$Z%l3^fbKKROQJDu;rr3#pil4y{w$qkr{N zSe+Wpo!7vas=yCawO@e(4~41&6z)sviBS4P zYS(jdk=1BxoXG(J5YiGT*44ltr`m8}EVU&WIFt#;$tiSUV#R<@CJ`QEbT3BW2wi33 z5Q5Pdh0x_2^BQu$B1aNKVSld{@V)iN1OIPWs4QuUytrVKB6Qwux^)=e%(_YYnDd422-YKO%k6kG{ zCXfpcW#16|D4l%er9-ZHw5(?fiABU*z!VQ?QSU4~bEwjJLdB4JiwS#WOo@`N-_N?Q zeq^Vtcyr@gYX5XhRjex(M0*Eh*sWeFC|s(sQ>$R4X+S1;FmI9jWlaWLTnT$FXA~veNJP2J?p&&AmK*2<&`|f z=*GGX)CW)G@8ceZ;qYmhyKi(ebZmkIFF;daq-8uRF4}UnrI#V5&j7_#GgJz)lO7%8 zzovkw?!n`G)?0yeFVC}(1Xxn)Ir|a*{^7huK-PfPM#H_FK zb6k~Nr~yc%JSG>g?Jf%5?DY&SW2nMK$CQ@Vfc#~3`N-0{Ev{q$TC)P7#4w+7+?sqN z13+FlRJN{aE3Wj~zu$=3hOskcB3y98;`ek#v_b;{DK{?FZcf!3f!bDxDi!=olY6~0 zT9~gXo_&0yZ~Hx~rO~5hfU!wEW=(%(nq&JVu~cVrPjZ# z!gTKz7y2Mt)s|IcbU5Rh)WQk|ds@Dhx98G*Wag3F8_P9|*6ck$(7=~}D&*c{Rl!Ha zhzC%IjJBq3OQH;;42#<#(01BQx)Ckg*XrDGRFivtu}9)B%KY8*Q-qu`$~&!=4a_GgTHP?u6pCAXUgTs zWP-_oN!!%8^2&&rJ5Y^Km|}$$_6{pATE`{I+xq676^luzl>DZvHMZHF9;3aA<9eTM z4s>`X&Bg87K&K%s>=`VYD0ky<3f&zzn&}^|4XWvicD^QL5qvYz-o5BRiFq>teF+`zXRM`E;&agr&w&fp~$vZo%@F5r1+{3Z-<;y zy9{@>S@+rQOgz5vP*Spla(k{Q1T_5WtWD9P_2+oOcc{D23}TR-GdTp<1VLKPJQ>bX>dpS zyRp91)llT3KgoP7Wjnn5cy3l>7E@-;Yw>2qd+IKu04LTGlNOkzU31oZ=VUcGGQT3@-69{U%V@85)^1GhHic=OyG6}RtJ za#?p1*0k0;!6)^l;KX{$Cd}j%XCJUy;Gz9_Nghnje%i6uW)nXbPX+yV54Dg~2gBO% z5sw~iHdXmU9KH;byyw^PYTTbn|2hHFTknb|{nK)_ZxYY90u8egkl^{DrYps-`wj@7 z`}=sS@a_%lirRUcL`^#HM(*L4NVkuvW~X-CGCvo8quy7;WYd|F+_)9JrfO7MvBVwi zM~?TrCc^`+zJK>~N6m9zhHri&V#tO%m==HWw~d#3iKd}d(!msw%YKf-Egzzf#n+jU z+-Ddl>y>C{BI&qeblCJA4d3-sjr*1 z6PtociLP1RC*R>^5kZEwY~IH&z(XW<-l8TA;^9LTBbv%$bbeI7akquEqWbfWJf59k@NwNN^}gVEO6#6?k}}yg?JcI# zKtQWU6wPGmPJ!5N+Bm3MKd*gFeApN3uSLiOJ_c*MM7>gF9(;8RCSJma>ysNDM`6gClPO6B)HL_uerZN5SwT?~ zyBMXR-?_h}>$ABpe_^@H_L&39i)fVepAQFT@n-vWmzqfqUwZ5t!s>UKcJqVm;Rb_M zS1R@#JZvK8crPu$gkNU(+ol~K!XIC!%^T~S*nAQV!44&ftG@HY8Gfnt# zfkrk@yWO>(z@}75=u65JJw9YN1B*Adc^VvyynT(}BHFZTz z_F1TMS&z%j>dJ4E38sm+XFJ*dOCwBr_^1My(eTCSS7;GFwwDwG{u|D zaqN4_e4JA;oWk&5%DovHC!l(!oxJD2+@d_*7Jv9*9X$h=-U|lmFJ|i9{8Ky!oTIN-&31uv5Cm zL9E>|s4&p+T;ZVGq4i-!>stn#_+tvispL1#?03_8XBIKHCwx3WJVY?`KG3Mjxz^5_ z_PyGyoIK4DFesfC9t+BHSqcN5(n8Bv!k-$PTv zU&S2msfu$t%=|9f!asYL(tf``Zqngkda9`F2tRL6`6;<_UBP!vA?2IPj{bL(-o=ej zQ4eeK%CrP4B$XMTVdvdg6QEryW4-Ic=W%Is80Z}#rl#(km@={|bN69L5t!#YPSaQ@ z9=`P0IOJx+n#zg>9p+vn)EW0{wAUq|XOv^sf!c&4OUEctJ5pap#FOv&I6%={%e(HM zR%+B9Jx2d7+s{Wj#(0oS4bM?1xWLrMPA#P^ylX~-z3&aJrqdQ#=U+2QgicHEDGnrz zpXIXR&F45uZyy}s1i{r6r_=ZSqZ2~`L9(;8J~h6|q1_<%J-H!?B#Hs`=u&?_xR{DW z)X|^pd?@0(84bsbMbT*GG)#s(>SqoCC)07o`|HroFe?|m8l&Z| zeZ)ZKaHNREnMz;rW2=c|zEbfZhL6DxFL|~wZ#0c*r{q;FQQr224{?)x?R?8Wzi(0; z2u!rw@*}_WvHbqiCsQv3c(d3B+o7F)LH{`HJZ(oVT8cJ6hl>^xlNeQ zYWX(s3oAG^ZMo3t&Dx3%(s^HNEtu+B<%-m1ydy+*j)x-aNS8d(ng1?v>uh0A<1fF@ zuNKo4mKdy2>f83v4;nj1Y|Dlvu#GLsy~$!~Bvdq*Y5xK%_E zAOh_<(sna_?YEk;SfN=QHCFa|1?7_goUGuX=TMX4_330T`JGylXDrw@J-NsD30@Oh z<*$${rcuJ&+V`DNupi9O6$fUXVO(o&5QYAJYe^LK<2Dn57`+{-zhaDY}KUg%s!l`HZaH2|#D#3Wb9)ZhFVd56G6ZFOrH$eb2Pu-mtsiqHBj zU9EVE_@Mbp5c_-qojn9?g)pWmhl&Glb0q8u9ApYvR_L}hjgSj&4(oYUaE^JHOMJ_Vy zj?7-M@*vOc;PNGe<1bkoqGMIj^!5m}M!kh#SpZwESooia@-DxiC?Y{vtbhY+g?Cf~a~Yw30qnI>LXs6JR!3co zT@RFSrwTfaevJCS|eHUXMKN%(qOZ{DWb6M$>bdiVr%-OKXS)>u}8FJFy7 zagKnLRKy$+kDV7&)r;Dh0>H|yHdQVMIdNz>ehPn_2a)BSdyX5g~hJl)X~+cAw`} zy+7~IeH{1i_};(!{^vf9yFWTUy+4=Bb-iBC*YkOvkMl7OT+Fnaae(V+`X(0WpASlW z-8o@@dfAjpFAFNYWGDIlFGnFhWC|YERi3;Vf~5LA>7 z?06J_1gALyv@K{!gpj@P3Xlc_{IfusP&&RPo(HZ|b_Uj@S_$%RcZt&PuDd7%tHAR0 zUEa#4%0JKh0klf_RX*f|zGbaOob)!5;7F5tz_2L7qN7LALS`M#96HD%y8PUA?4ttG z`Q&c&38D8EAgTuFR|R7UeV`Uxi*?1cf)y74`DBqv(yfW>1pm0>Oa#L#s}Y<+s}742 zoHE^bfSYbtuet@>almc)(Ss!)d@aV;_#S(|ex#;TJqGZEqfv3n-(;>3YT1 zLKB!L$idSs3)rS9(@6ke3^R`?8i}N-WTeUNp?oWHuz5q>Dk#HZS@fhaHTgZpvH*~!+vZN$FzUm0hTQ@Lv`=dHBRqFE@t(z($Dhe_16H= z-MARa#gMRt>9E&+YDMVm_}>=6ce(|Wm;AeXW5!XX5|*>g2OnpE6GM=DhbAf0;~gzG z2IjTQIZWq4yJ2!{0H{ebmCJS@@;1HkRQCw6OCzPCLMbJv-tVcv3=}sV(}dc)7?wT& z`tO{!lNZMSlFiui(J*nW{Z*B3>sV*^~_# zK84vP;hrBom2$m6<@JSRhXi6mf!%SxY@aGA!%RW)$aQ{*qJ4Y&ptfZDPAM>qC|=EA z^XKTHPX^Bu67E_O*`)`J16f#Aa)?#~{dUJ^)(cFy>L_$emPT-rBo@^;ainRzC4F@v zc$-&$L@usRj9md>#3V-Q$J{{D;)B1|KR<3j=z8Q%tnAt-dF^A;=ZkPun1&n;6mEDe z_`Cw_GrLjjIL9^KZwo_^ucs-CKf-Qjy1+;Iq|1JkCoB#0M64ONEFC#-O0^v*S(e-U ziifuQr5Q@tT(}$#DO2y`^o{;?7nn-pkt|8~v^h+rLExy9$ZNnN%KGDE!*2`fS%c`O zgtM>c^&2kBq|^2~Y|H)& z!mAgqwuu6zHTPiMwBqG6drpF}iR>O9o?Zo*nT%f)t4#T|PzK zDK(!mKYt^c8T~NJ0}ZH7fobd*0XhgUnn$hk(m#Evim|{H#65fVYwgxaw=2RO8M-EJ4t)_>R_@Hnu zgs`E+M+qEC7T1BWb^})S01R((tw-L%q4}XkuRB$Pmnkzv@wxlnTv>@{3kp`-f4r1D zW7RP9YzYie{4`=Mf#3THnFj0f(3wmQZq3*i>(C=oqPKwpb?vS`2)g*{UW3OMY1{OC zz68gm=`l0l*$P~YBMUK~V13d;L0mkZS{Nk%boKE3T)`fGMvrRz$CozV7smuLX1;K=h zdVb+TD*JnK%t{+)7NS0CC`P^2h=r7|R#=UG;SN=J;9Xw1F?6#xArHLt8mwKu zaXnEb0&@q%AjknU{24Cub%(EpT5COOLgmV(&Q4*o;Z`-9Y<(HLZY6m>AwP#}6N<9T z2;%Z!^e@&8a&MVTVRmp-*ijoMW8+2QC2L~)h#L-z$90`w)W!N_4T7@+%o86=&xLSL z+P&M``eQAM`$CJK@DlYq5WeD>BzzcRBv<)no^*v6c7!dIm$*a#VlYoQWKB z1ea|Tj1k~E_zfQa=cRq;*(r!H*5b#p90GgiWvWW1YJmS&jnlUh*hy_!_W${?Wa5$Z zCr)CJL&qGP`1fD&83-RH;lJ<#wo}OD)w7)JJ1>94e|>n-NSMohHMYh|RN2n~mC_Sl zUI^b)%K9&v)%A_)CT%FDt$-|#>~j_17C{_v1(z<_Cfe^D8Qzm-zUwU!%<8T^Ir<{g zM!m$*PI!G-y^(W-A>Ch_0skf-t(!uw-(_DGQDew*sOBVpEDjx^X;*c?^Cr|Zb@QD# z@vc22rWDfpj-f&bD7RjV(SN_zB?eCAGSzhLHV~6O^O$z6O3r>bnKJoe(k$o9Y{hv6 z1(Fpbvey}aCShYRQlCiXYv0ci{r5I{@v3bElhJBl@3>uLcf9p=!!s#}a158AHyX+< zutCTbM%6^I1xJKCsny5den(?Fg+-o!T@VyRZF4}wZ~a!EH3dW#g9}JLC;f3{rtX6V z=6(3Bk9%$a>weZ;E}EeNaUyt-h}fHoS~+<@3142gxPRfldR3@AeF1ng$OTJNO6KiqV-!xY4NbdCk_UmIBs z(qCX0Lt&>?3t^NBl zi$8%pq<J(C&9G6G3RDWOHd-Gflp~0Qw6)x9eKez`_H~zffp< zshNh6P_aA}wMGDsfZ3zS1U7$qM7~ z=0B22NUzy0{;aL$0am*$~BQwe=|w4j2vK>%6^ zrWX$cet+fnuAlHzZ#8gp7g7e*s0y0)=1nLXqACh?qH8@^}0rFghCYM}-4~-tc_raf_>9czI{33k;%EGB3a79wu2Hufn-n_LL-un!IAj zkH5Ks9hR^FRIB_*hJ{?Mqg@z6HlmTsU zt>RBqfJ3+%s@3=HBsoq+a=EjnW>1OW?^VX;DI0OJ3FIK}m&|GI3P81rbeZezKi~nF z?G$|7TXo49I6Kek@smjZRWEs>zHb09pOl+&Wei+J8pZ$M7NDhdL8cL&_W-lSO$p^`FSWQ@! z@4eI74-anc1PEHFUq^v*sbdbRYrEjHPkvvgO)TF@&qZ0FJ1^R;yMRAuLtzmx$=Mw~ zaKoZ5X?bypFX8v%_@fu9I9u~opd`Iuglp*(+-@L0)o_pU(}yJqDsUU&cRlb|NiBtP zA~=L=PTcV+?|mteLW34#h0uR@S;M8xcEgmJP!eMbq}he?QzINYC+{f>u-it~5yo?MVtT^9ffDbrT zlN7(?06ZcHNS6RU0&>BTk!I4SMULnWZnMvZNyuZ3#4jLbpv`@ zBR$Y!smIX$bI`Eu<7BD`5pyXaMM^2M#S8EKyREv?2R(KR2i{!>&UXNK427B2fklhI zZ%h{R_%S5Ql%WvhGtP{`Y2xwF`@cU<=5?Wxl9Yi3j|LHK8N?FOB4DkQTRbIm{Tfum zS*!GQS3pNaLvmCG(0fel>y^{1Nt!sxgZdYwz%cC?B2h4~76fj;)xR&m4bF!ButA?1 zKo3qZh4Zr0gZb>HHWev}yme%*Yo1H{{Oz|#WDNU|?hNP73KAX#SSb=FWl?nseQNP2 z+IV#Pv5J`m1DWv)R24#QQh$X_Wd5EW>Px^J8ec*}KMI?N`2m0|lz*Q-+>KUH?tcHY z9Bgzm6-snrV1GcReFv(fjw6Y1BIOFo19!@DW&WSbKWX9@4!^nMN&|T@qCj~F4ULxq zm@*^Ddi3wh#Jh(7F@};3=TeaEn;wj|mgul>p2~bW6tKhFL%AXICgX3JB>oUvNSvcz zI8yUV{=SFw1qzviiyM_Y-Z*`%RxbJd0OS8Dp7kI3*Z*|A{?E>8`rpSE=@SJ1f9{u1 zA6O)Pa<(QEojSsOK|GKjFgZS8P4a;R<#-w}B1zmo>h-+da=^I3kc5918(-lT%ud)Q zkPc4fKL;nX6A&VhY(;{ zsU5Dez=(nLX$b^EtKm5y0q{_XR70;}LLvC__ea(q`n~RMH9nW1YIt>1;|VBiWvhTm zXoM=uHFvz5PWMFjkN6%|6Uo`W+5$&A-8V5E{2fM#u_{1fv3eVF;nAk_i; z2RMUG$v(bBSC=k0Z9#%o!aWQCN)Xl#cq64xZAru9l`Ovx(<-c*(QXgx#IF9(+H)w+ zJ{FPf2yCz8@tfkIOIjBU!F}zjkXl%?g1bgs%*s*QW;G%^506DRlRar7A%1i>wBe}F z8xtV{m)9i08prh$z+<9&)_lnsi~;@@8q>e^xXiNv*M$LNw!1M^qZSQaLhO5`0Tdr! zda&a#dBUBbrG$rvM*RB*L9IywJ!`1?yNf;~Ed=)&QenQXp|={g-HycaXsdh87r zD0gXZRrzy!{#LZ;00&c6)76Q!)mnsavAVi`{rdXO5}3@{aNLKjstKr)hpHCQzHAQ0 zWdBrqzRYa?Zt+6AM5CTlHN0>KF5!a%x0W9#Ss*}7bK(W7>k0xC3xc?Kt=n2UgRcSMQwOAf>ceg&-qVb zV)wVmLCnbMLkzrlHn4$oqh9N2#*pT3c$dk@-2U%(NdMp7b^qP#{cn5|hGU%9(c=(m zN&?KHPQWuD2w>nqR@G5H^g;rE4VM_Ye=aba`36fIypU3WVCr!Fk_I$GQ=rIv1|dGj z*vm;92Hpt&L<%BW9C7Lvy~e}<=(h{7PZ|fc&gb12U(%4JFN6AxLS}qD{$bk_C(C94 z{(EQ`k^sI3AwhFM3ZTu%1!V$LBeq!YeE;Jbx83N_S%P_&h|E?g2u3kRKERr8p^YiI zFKPBjdPhGh%QHl~$v7u5$SB$2#6kK!_}3K~@F)btTHP}+{hzIlwka3Zy$~zs*F&OpRQ}&9Bq~h%hQrTBo-^C8kmy_j z?&LX8IWGv1TKla*IjXeZN#YoIN(aDFmJh@R^3&}9)Q*w|4m$KP3t7Cuy&XhZoydrQ zX#^o=`uFa3STw>WTAaH=Lfn{Lfg<^E-9l*BZ=aNRqLv8iVI=QeAyD&am!1CSs*>eN zC$kW7s*5`a0yP+KR1F1evC5J{K-hxYIGt$#bf9;{k>#thd8%Y zAb}T2#3W4zRHW6gpO9~`gfP>~pw9{MzxErNd9$X{|KLHmH*>C;Aydt9vN*4&8~ehA zZoS)Z>pJ`t>b2J8jedDf@t=yvpP;aDY^*fjP`xW$9PG zgHR3ge3IJu7!{k!{`Zl18ENtpD71SsQzIe>_AcI>e~^+d#(01HFT<=cdje<<)UQ_osCYR6 zPp%)Q9LGP63??hw0I~L!aR^!~a)oX~bGESO(i}|65BIO`E0Sh4Zr*uoZraP#Q-Tt=#kAaIs-)q>he!gIOA?T|U zoGL|H^A$0mWSVt?b8cNNdX%7>2F{}O0c7eMU7mnICm4awoH8!9lJ|GsoR?#J^TriZSq!BLwXB77%(#R0BV z0-%OBK;3~xFbU9l{>6>{Q&3}1H~fNzIT=CfefwzhA0!IXyS)}C-&)RQ3R;+ zx{mB~({rdXLdBku%inYR@6ATfTPt9uOjLa<90T!BNIeh`K0;za2Nrm_hkeLT0K1JJ zYjqI6$-H5&{|gWNeR^aiBCWw}xlaI4)tW5s{rlfzK*MbOWksmVvR7Uh{>F+}{!|`a zlxgIYS6S-0LtVF!wk5WiV>zU<2^Y)%0@VD#{2M=RNu4Y!lVm+}K}#@4E`|;_%+tSb z02x~b(?za2DgVU{KV+uiXmCC7;rANEw#{9gjB{~L$y|6ilv#U>;Bf3?R|hOldxKSJC)070ZOq%RaLe*n0%{0KMOmHsgx zFBo2iH9Yz+qw6$AG2rYTe3-fGJV`%xYYwmp(u@l0&_|E9;Gtv%+_4Z4O^%dUNaQOV-*f03nYpJ4oTo?==yrYE%016TwR%P2?Lh;Gk`8b z5X**sAOurh;Q(>V#trm9{!>US2f6`z+2wvYakXrtmRH>u<8H?(AJdRo%(dt`i5JeF z3%e4GT0i=_hdkTj+Zo}Uc7U%Yam7VUa|8yb$W_?7QeWwGs9 zH!%r?Pbj#Sg}N;FvJQaSAa4|2X{j7|_>iT$)t7>O;kx1=8Uq3Eo@-3mC?>$>>!?OX zH~>ivXn2bWB{qBe+2H-z?BbzngR$)F3Lu)FvwlFsCWz6+_n*NY0)mc{RR{REJS|xm zB;nN-h?|B$>G%wQ_g0927i%cM_-E`0&|nuOi=@u)@kPy)97z>dI1j3|ZNJ^W)Eo>h zc}OQ)Ed~^MJu^GJ8(CSfO0E*y;jIQleKz@Q0>W0S_mEvcJ9Ok|4p^Gi*dEe|f2Lm#!cG{ghLi*`M z=SW&L+6xcFYhdy;?vV5@az_s&4oNpg#lB3O*)y{6M0zloH{eUHf;B>^sdA@PU=||D zdZF^dgLKzh=Spgb@VliVYrI7wtSTDp_Uny?6oRwRrc>U1yi^}zw;XtSSDCe?x#*Uu zHB)4;<<5nZg8_-z`}78xCvSIEG(4|7jb#&bjJcFg-+Qj<@djB?-`gVUpGV}`wy>8h z*-eBdZs#Ont22?~@d^K*%VBqJnNlj#Eig*+^clTo|3aS)U$|BMDo(Yi=r*uyzX|i^ zE>m1oY;8n|6r%oKm#1#AnGc7aQpus5=hzY$W|u z31nQEcY^`^4vQ7u(&fiS!tm2%pEwVa6YJA;gqK! zptK8HTg-X_Ho{Y*h;v-FAelT7NTR+RNQ7qx&7y5i)B-R-(u|`!Gg`MIdkQ@VZNi|< z5vlVem9)2%fB>dN`?$lbu=){;XgktfXUkx5`k?I#5+-G#{37Q9~R& zu=o>RLRQAn|64kZ{~$}=r$6wY(V#T{zl-7jhXcg_I^VebhbY8HI*RPFQNnTwY_*BE zzTZRd)qF^(uZH>bUP3i`BkbhWH9@`{ETHSgE$OD1kOuqcH~%QuZCpYiWZ@IKw5UR& z8s>~2aCSaI&e+-pb+9eqW?Z3XVmP)4?O z7dfxJ9THpu)TQm_X?W19kz5L6hb;f79deB?SBfn!DDS{g_`)-WHHY)72KyWw^7O3UqXZdCB(08h9dx zgWEp3RgD-S{1;M_1ZAZhJT|5UioSr8Pf$TCcO0sFJ{Xi9 zL$r^Qy3gd6n4q7aQunsxQ7C|OnZ1yeMwoqAv@f(HC3gzAH)2=IFES$J z8t_K~_~`&lh`~CA>bXD|@YOHeopTrs<)5kSPFmiUZt%$lFGi z@4zUoe`O4Z4V_0bB6g7lAR;LOj(l4#p8GNh0Ih!eDs+gE(85uEp0L+}sFp}L%Qh^( z0FH+(oN`6nMzE^KFM&6w?{Y@fc&}X99mx?)M@)oE8_8EAw7+2jdMS1G1}E4FBf(^Y zYPoup=m^BAOZ6D~0EP>B+#Mh>Zqz+Y{@{?@zS@UqiclM--v3ej@o>=_Q2=U((dquM zsHkXzdGwu$F7}4;w(N4iO{%y{Bj9|i1$T)M_&$}8bThcJ49Dk(aoZ;=MXzbxnERf= z6WabL!Se?|!nS~F_YA{sHCTBhWBSHB;3pAO7)6DW+@a9moyE;65^#`u-KVIy%u|q& znL>#4(kYqK>}h~@Mi=Lc!Mr!*#0t3bkhAoHa(o0_&%`Sxp3k%&_5Ui;Wmvj@D+%{> z5ne1F5$vb`C_%{$@b&1*-jTQ3r3;=c_7z&Mo6Vg;=N3|r=^thpGr>AGA#wop&Dr9{ z9;^dHoJ%;vj}iA+8>E^9gyQ2&(L8AU;|6hf$uNGB^({Fu-%i(GQ!~f#%9=|Q)L)Uf zfjh8nJiAW3n_8e#;%P2G)HZEmtH1o>F6;++$s{V|5|)45gnmipH#0r?+3U&hk>)Qn@gDU;&&05q<{x5 zWEwOkUUVsmX&L*Xvk-}iC{Y>A^jKw^sd)vly zGzVrnl!&^uW7MkGcV&VXQ=qAM^7wQ|gSqZP0cgRyvfLx9S7yfA8q!tT(VnTR3ZR%D zD~|kZ52P58T{HOe+k5u4^3)g8(B74g7tMzO+$uLAB15^kuW!Ka^vFdp3tCyzfhJG^ zo75m zid#!|pEg4{O?lGUO4!TySaia#Ev{2tdsqizf-I8^&ncvhAOT%2$dl`pqsH?qQ2vE9 zi-NqmtMcgD!>{mYnI80^zUW5H5!0`kG48loE8nT8&RL4>DHPGIJAJu6%fJxv%m!{% z!#noh!EKCKb=L*nZ~P=fx)ZUg$^GjB_a$&F#eh(oP5_g)S}~%NW}50ePzl|gC*Too zN?;YTxU60CuymK6V_oE79oBGNqT8PZ(zvhDj^hqj!x>7;X)UGx;M_XV_bsT9FFwoL zP30t*7yyTYJ)qCS#L*YoXLc&G*!68IoB&?tn;r6-mDOlDe(vD*@130@f3tU*ISWc4 zSN+Xu{#@oqetvV1TY?gJyikmv+KTZO3KBw^X8@}^S_+#oA0^*yb6`??=*+#_%zM2p zv3~f{LDeHRqUA$vXGCe1`v-oVTiWY(`L-WySTE)lv|S8#W$p$Ot4?&Uj)dq|eN91^ zRmTaw(g~+aGgOrZDYw-Z-nGEfV3w!xo>#9f?w0RqQI*D@`!61oI?ddqlyvnrux4%% zc~?bmz=7DG3~qQ>K5S&H_*6aR-WW5w3h{YGv5Mf6d7wkww^C02Jw-Q5c#5#fU8}^m zs#$RXP%*Vzx-veYN~KVBdNEmsL^CGI>n#oE_fscd@gR?b=gv%w*NzfGpv$0nC`#jW zJhK1&o*$BQVE}} ze(w_x#>jUf^QFQumCVEC$gNl8brN~h(-$UiiY?&dLG$RKj|82K{hJm!G#m4^YQ6?hPR&PEBJPS&YBeUQjk>_R>RG*z&j_}JEk=#BG%iEkqpf@PoGrfe&WMcB zh)t?Iy;9%j#v~jn>uE?2vGL}^{;}PKxk|BQE?j!TKC!A&AzO^Xm%jqMV@{#4wp;`w z9CHb(8RpNvz7liGfI;3TZD@TAMh_xmsq7lyL)_>_o!sHnEmD9Vjyu5m;Nyux)p1Ll z#wy{LJyb)*Pp&UaC_D*(?y?#=x(P67!kiwPh%TtVZU*06fogedW!d7xlx^&auegX2 zxt82T^!pY5m79Sb99Z6bTy?7EZ?O#?6-+)y5v*`~X zT%eE`&=CY1;i)$ibkDG{IDY}^0>>TZ=(&R|;~kD3k2?k_?ZT|+l)4&4_lI};hBXL1 z#awr@T$8q|h;9Sxaa?DvY==Utqb=;@t;;MNwe#}PonQ>-;N$@AId{#1xd$m-9flXm z>^$J{z9idEt+PZpehbZ>fZZ2y(5I4y%K#-pY6}fRzT|a_}V-^crb&5RIRJ9_e8BNC$2caoCne5S@tu5N8r}oXk8-t3&VeQ zNa#ma*+dDQTuFF;34=%pok+^GC?c6K7#zP_3oxM^4BtPaYZi1xoovE;Ha9y%vap!!3gsGz#Q5Y zDUvoGINd+*WW?Irvg10Dm+b2|sEUr`{J&V<6r7??@g_e$8yEre&TAK!@O*@cuQ!df z9@L_j(sw|KMzi~;957!EBCLaGJPbP!z|qX;c@?Rpx-R7co7z}Pz{67#3~#u042SS4 z!xnZCN*!L1cb(uZ`st1mlWOpjGhkAMX0^4u6KSeQQ9lKphHPoW*OIl-J?Hwj27aJF z)?8R=Mm@qI-jQ!*pf$`i!RE`QYE7f0+eoaWtvy>I_iedr6&9k`)Rma}`K~p{t}$V9 z;uK2VKo8z@kpS^qlCr`vdQA~>b4<|8f@!P$&$m<(W?|fe2YK22?`m)A<0u#%^*)lG zNw*C;*3CCrjnz`m_;cyQ^EWFy3OO_8-A&Y|wu)X}FdI}j&B&l?Rxo_1BQ_djI=NR1 z05-~1lOdcAw`Dbd0ZV=tgTLBECI9PS9zA=IYb!!!TXr8~Ync7EY6D7GV~3d}vh8w= zE?h}T^?km=KQig~!1^jHO}=R{2WS_Ex>iB{&;W%!Q&0V?1HunZ@>an8EqRi(6h_Te zIZ?2>T)VdHSL_L%fSns|sV4jDY*uElRJHw-PHAN3EcIha+*r}IHIF&Yd#Bp^B{5#g z*wGNiXUQTnKnH16fBjAOaQNs$cjmY0&+XGyjwQkqh+fZGJRJ;WN)iHUtLM17%9;Eo zHkLiq%uAhA^^s~cEPJj#M*WPVwfD%jITy;u7*eWrNXhA*O5Sr1vy&gCrS@#{oGeJU zUMzwt#-LCXZVeB$YLc750lUg-qvhmZNsMVDEUTzxT9oO(!+*QV?B3S8sS4Ymsk;-14e57dK`;ov{@5hifN-BDHGKaG_pTTZJ7OiS$jr+G7ZQlNB>nC3> zk#YdrwCac6CEgJol2q9i$^;*rAWmp(mymEso+;3LoU-pb>)O1hX-I;2jhcpzK6EGj z-ra@naw$*0SdfnxFM|z5dvhZ3IUJ+8``U&d>^hQ8Q6ZN<<52a9kfN>hkYHfAmu6d! zSWdE-&MrcnKNu6y$j9{OPLZ>{L`U|j(`f3NKUE`a8oNF4f$FzHAm{rV%`C^LuWjDR zo!P>nRyI)f>WswGq<2ysDc`DqIlrsPpuhRo$Ne;0In|CDKVzLqP@!{qb=3uE_;NrX zFf|vZ*^|-r#%#Vgb@lAqiy0&8{-^5NAK&7X z6eVVSsq48)(eMMKjM@^Gpq;BOU7g_-esLQhvVEs=4WVB)Qr-l8ju52;#wcmI+JT|h z*j5TCKk`kFryqU&oXbfMEXkJq<96m6pP!J_bsApxVNWj4R)hR(9X4Zr8@LPTg5Od5 zrI6)Zj>vYRbAJj_C`;92C1^V$H2yowJ+?3vB5c=dCLrTD^%|m+5?=<1( zo*ddrd2J04-Hsq;>Ljk#EW^5eJIomi#OV~ciu5&Zjd@JD7ah*~A^f1IG+Dz(Gf;R= zXXc|y{}~$|Y5vC4fm-TBN~4_eDks*8LISm0@jk&@x6GUc)NT{88j<|n28XTa;6Me+!t#1FrHAfYDfpLYhVW@ z_ZkqeN#Puj($FwqR2}?eXsOhG@D8YNa@AX`0?yGY66JhqzRS@pHg1+?c=zpl{>J=x zD7CxDZoI}-?%wIb&-lRpB}XYQB&Uji=kFnUoz1GdQnzhURbcYHF3g+!NSt>28J-3H zd!L{ccbAt=q+(~_LXwlPh__}YWz0jrBu{a+8q1m88ztApYk|!#?XLJ+;Sd&Np!LM? z89z_UqD5cduaAp+-E0-Voqa3U{{#?Wp@x$Ts}aLSSu4?XrNVD;(ltO6y$!@&q6cCxiS_lTd9vzz` z?NnO0tD|B!94c98-s3IaB5V+n&ictlm+ZBlX5}bjzla897RWL>3iN!gHKqt$d{dq; z|3as*SvWbLH!z#PssNjBQO9Bfv-?PO5_LCNsI_h=V`x>U5TvQ4_YD{6vUQYVV^J0C z5t&=6ZHfE#y@^js)G6Xh;tuOIjZ<}Bg$haxZTXu_4WAkDzfmdiH8>_U^<${Wu!Ne@ z(lIKYp!y`~YdjKhL=q+{thUAU%MNb+G*Mk`Qqn(#kuD+mv>~Ri@IO5b*@tsrOy_=tfj! zRZ7BHxv6-jW`%lE-}Vb_IGY7Zw3@l)Ji*Ww1ZY zykyf7F$L7!n}m6#btF8tyj<&s7Ig1zoRhSgvN%oGagU-BXu{1NiX2hS`uU)Bz$3gG zwu;hM1R;*D0Y)GS%>hk;-k#%EC0p(C;B-81tvg9hPOLbN?pOxk$-b_|2?~f=H@ysC zpF0GnW_?cxGs_AAcrIfG*wsQEwJ@ggNo#E^S!VqSnkw@-HOP{jxL=*OeGtyzoKfU9 z-W4pFHs@)vH%%O0fztr{TWNqezL+bF;(oaR#Wr7>O6F9Mmf6cuUeNc;j;5C^8W$G9 z3W@}v*DiSUwAN|XG(c;Bsd*j4vAtqF7odm}ty)BN*8uucIzStk@j5&hTmLB#SW`I; zuahvM0>b9nc%k1{5#EXpH|979J)7Ixv(NcdGA3 zxpfQ50%U%wL@{x9pMp|TKnKL_MOER0A=ia?$+pY*6aXc3`_FtwsF5Q90foMoT0^fO zIG9JPd0-q+iDNHn?ct;=c6$VclU?ux5MR~9jU%`U!1%IsZDzwQsLErxN5K+m976_a z2GA&BotSW*uV6du+kF1Lgy=8XuU(4WD(Q{I6K0Hs1}$T^4E?^MV)dQo86^t zaXO*DE^Zs8Y4|l0m`@0L^(r2d@{y0QloWa!K34OPF@L>7kR@XkS{$u90JmlnkyqLj zG)lL{ngxJbImZo~c%io4n`FA_HJ?JF&lLCble^L;KnN5xNJA+EQ0CrDW_IW}HizDc z7`?~XHQHT5Bl^`@chcK&Q6a{GlAYfuOQw29?DOUbB&_N>3qrFr*-Ag^NFJ?LA zm$CI2qp)!E#O`ss)5bXbxHCk%A~3_m(A_;HC)jy?Dzz7$>D)P5(ODfZ8kt>96cL7q z(Xc#z&!=ni17`BE6KU)${LKCy3bzY-mKun4jx>xXhH^X1HjgeFn?(RxUQWv!_o;9t ze*vM%LB^QL+OxZKu0yt}$<)h%@;VEt0=?9+XoGpQ-A$-F=Fep?6q=w9yEOG{b5RPz zch=pvzQZnf(^D`N1_Z@742PsecNvn~=rHl?maK_fX569Ia;~}qv~hE;QSm3jd}HC! z?z$4soNxCVn)5rgSV1dsAaqn^PTjzZ;5o3u4JimK4xp;F*?neZw&vt0w=jL-h#r;Y- zz^0F<)Tl-=FqoW1hJM&?6Ui(UHu#*{Sd{$)&E#9_`r8vso>i8{tj$%16|{`x?R~K- zXZ-~5S92iYRogJGZ`#Gd%aLKYsu!5{sZ zW?Z0qU*dhT(P{=@<91cgK3VEcK9ZDr>|fA$p%_L^>k0=0VW`Y!_$6AFr_#w&Yx4{j zxNis5DqmOc+_aL%qyFOAAkeO;?H@7Pa2DzkG{qpZ!gbX0HzINjXt@=$b4y?&tbYL& zHReZX)`B(7N+4};o!BU@2OhDN2C#v7i?+wFQ!4bwp4E<&gBsDgiDnAfrX9JN_HmLY$CA>m!CrQ7^K6`z^DJYc1s;oclG2;*bBrWox2851bN&0Q-1hpsMl3z)K?*Vx(nbOTyW16p1 z8Kl`lzs{qnBu^fs-WzIpa$1I|Y`;~Mzp^)=;3+RCuURdbUX9dOMS&#Md;-!^a=jEE z9(~oirRk&$Pe+rzd1si6b z&)SRZX*lp|vmD_JyaB#?x%yT>!7`Oe8$0BE+bT5NqUX)Jj&_A7du}TCUY=~XTeo$O z=>FsnNr#v;c-LS8(;j&*)T=d0b60A)g-s)$Lb2aUa&V;P!JAiaz53iVQwrcXFc)~F zR`U*ix3yXl-uKF9uW)lvs)QYIxw5CHF(M&ii}n-KBYLTUg_Eyz+X8tT?_S)$C&?hh zej2cZGEuphnulU?CFv|DM)DJNiJzQdW(l}WLUHLim9>OOPR2HI(Q8c&vergzD?y#% zhYk-dCa}DRiiiMn#f_wCrP-fIHJlN@i8n0yhwk}5)LH$Hbt`OCra;_A@ijtB;RbAe z&`sR0p0lU&U&dIOscCplk$>w?Kuf5M2oNSqPd_KCIEV_4>N!BX2Er*gFT*_$z-i3M zI#2gCy>)|hZ+I|d8$?w4NJv5198?Y?Npq+xM4yOkn{iF8Z7LAippv00igF9DSu`?7 zUm2467YCPL2Vi1#0%?mbHqc8tDd_UTkf}m~BRG}gq|3T@ zRq)_By8J6>|bGO-|?OfT5<-c^G4UX3)?MNG6*?Eh7ECP@8)o*IraPpSjgCfAcLc9sm@7$uJAI3Ryyd>+N`ZYO;ru!n6c7A;62A22YVlL z{de#m%|X~fVU~Ea9H=_ID|f(@$#nI0^RRAL^l1uo(u!WX!@)Yr7~^1f?%X-mavC;s zO(^h+S4S9kk1+1>g~OrUAO&{RHiHx(ZiGV8jPUmrEWXk9LL1b<15kR=z4l3uDo~HK z8weQ)y_x>93e7VqdQ#RG^tf-%8mkRX&%?o4tsE$5bj!*&_65Np_8C}Ya&?TWI>R96 zQrUvu%ue(wuv4qumcm+5gYhN#Wxc;zCm%+h-Zul|j!+now3e#*D%){A1F}dx`q8hZ zEvGw9*g^~R87jhRO!Y}V_5RM9Z%z(DB}J7Jipjlz$i{p5?Wx$5qA?FF6cQ^JO8W6) zzR;zKxbsik(|LihnGc)v-Slo4r>Av%IX)C=3vzKu$1MesXnr?4F*c&Mw$#1W?(k|z z7+{3>F6)9Ww8eSE$X}VhImczKfJU@1$!veUy~e#y6tBxb8YBW!@hQv~h>c3ng)8V| ztOvZ;IZTI(gdM|vuD=MzhCPV>>}&4^j(}-9OpMoSufcb<#|da2W|`bs0Kak(pfJrM z>vP@Qt}#$fan^zwR}>x+-;pH>Vj8?TyUY8Un}Fh98K1z}IVw${Q3>L7f>u=on5Pj2 z@M&Pw49x|a9@}DPt^ki_u~EUD36`cii-0s1FL7wU&NKj+=5(HEV^YAK4&VvsYr=ls z-R77vcX^3!8k5}z;om+Zt!}o6oF;-oB*YoR9Ogh#!~NZf)7)dR8fIAnGNEYB-|+I9 zlmqK@r+{Wjiw96WM2BpEN+X|8D~!y5n?94b0r-;m(Y=O=hP7as_oEF$;~LB9*k-`G z{IeBo2X+ul8ad8?g4uo*BTtGqEa(MeyHE?$|NZSzkycM_z=-WqGbH;|k`VD3qzEj- z+7GF;DX=l?DkAo*x)&cU`gx`no_ofoJ5-kKfX=<=q5z6F`->Iy+J=Wein~_V9#h=0 zndopFP_T;wxFHhNUl-uX4^JuyHg6li3ay|M{HabjMUL=0icVJS7wCp2S!8IHqF9wK z2aFYyeD``$i7w574H4>4WlO8U1;Sw`_rO$S zujItqoh1in;MnTjd>F(RcXR=u7Saqk1{wki3K1vy@mE06^@4jQ+KVo9^&LMVT^)R_l41^{_ma~v$V zSE8Ta9S8PW$6^V{Z*t>3HJ_WdWqru?KxL>LOq<8H-iJwYZKE<~9pt&Bwzz7BmR%F# zd$qCY7#mOtkHc*Ba?lK{Sw0NaFm21&X|)Pq?a3wTbKT>R-Bvj*I^BtQG3ln>yqr@^ z=Xx=z{F!|nN{wc>7Y@ZdG*LLwDa;CT{f>H-i2!5peBgwhKBymDJn;-k>)G0vV^ zm-iauWF|C372om0YO94fm#iEi7^8NJfG)|#MM313WYA6%j2z;r1^MwXXt-^)?fa4h z-vEd;3<(^k-;BfKXOIW9ER8oT#I|w0;*jOx_S-O~@0bDIy5GpW+eKp9%L1G6(By;h zlZipd!huMsW-o`nq#5JapnvN}12^bP#$IOF&Q!pKNOaQJOmuAb*kh-MV2hrPp)GS(L4klnRl{O4Nzr70MV(g&vt=9OeB|DN~=?9pbNJs zx*YbPCQ2vWp&X@$4Xex}#l6hEfPe0WOCZ<^1VpE!94ZNq1|jL-O8h18m>ASsTMRY; zUyhKVQKFmcze5T!zzfiu$*iLB8UmWI&;N7zdi7^x*~~Yj_pb z095*vWd?b;xpM){ql=V%0zPrQ(+N?KU3(|rzDc-LC(v)-c)*UNudK$Z?44mv9rD)u zMw%G?SctIGW{J8!HK51`H!HrtyUPxJ*YnmOW`eV-01kFrFf1@=xp|wROhZ%T?iq3J z#;g*Mo^hxcoj}ZLmbz%Eaql+KDO6oZF%*RdzFxs%^jmGQu#IOSxg- zD_Lx_VPtrB%1qWk^VUdDf0OA)&%5+fJU1GcF)8s1_-y~)y(Bv|ZWPIx7#vo4ummqD z_GGVXd14KA1->K~@qN>Al%RVxD+zR(yW;e(>cTo<x9>jP$)9@vUdrhnlere&M^6<_FGeTeq<417`mwDRfyOYRbp`eh4>o1M8 zL#LK6>)br*UxRz=3DaG2t{P9Sq>^ou+3#%#Zh3ELn-bVP!#CNVeP7`N#9W}B^EQ-4 zmJrVMp*6Gf%;K*#!AkO)6iZV;7;21!c5{l$B(w` z!`C}M0xqdPmE)n{v)-DC=t;EnuUCD(j5))l&|}`@J(QBEbYzFm5+moU%(}Dky<@4? zX#IKBzV*v|eo60-)DNN5yvEz2cP_%_=560CoJQ-EeAI9_tcB?9%~e3BVZtj?O&6Ch zd^Kdp=a4F?mP6IC-yU6$zNwPAAxk`_>hws+_htF_@gHgJiH-h6K$Ai@-dhuLySxW} zG&`xy&~V))#iALHBoB|vJ6Pd@P>`NgfF zZx;fy>vDXX-apk14VR^3FWF&#&Y^Bf>{OjIg9SC=IAf2xy|mId-L&?mMNyX3nGBb< zGj~~@SpsBBe-+J$wNjEi1E>wCyHeuiquC|j0fR++L06O<882Ypv7EZ^y7OSAu;Y+Va=`j{o9 zFJVB$KdiA=)xEQUpkCNue?lA63ilec=PAZH{DOvwWYic|s^-5@-OyXG(~4oM`sU*^ z&+7Qkx?QCVyehhZ=6IiK)(~^@U<9=hG(=I=-Kz3r6{hyi%0y`i=Hy?`?BAfxRZf~W z|Cm*r+jh$SKy?XyW3NDI&4W}i-#Q%!DeL|D7(>LZUc&ULenZUk=k>267#)c3w{X>d zh`oA=ru?ZvQ>x%&JE`8Y{SjClsh>Pzo4QS+k>hYP3m@00&_et0gT!-YBhRCCU4Ss3 zoHo6|vTzH1t)+h@!^jmN*LKJ*HEuNvO*I3QZ2++9$(rgXOsY}OyN=`C_^_$wqh{Xo z$g?A9qgr|2rZ?J*=*;-ev-{tDn$Wzzq;px-UrFN~6wWoeMfBS=9=1t;Oj1+5WVJDg zEN*jtSL|MO@m8Ti9qD+xsk&5#h-j6ZD-QibA;3EG;VPhHmc=t zd4+bUdMUm=%TSY|sK_R@BY>-daq}zrO;0Cgh%jWI%k}?MO>}eJe(*x;ldUGo#s-$P zm#x^>Z#TTiJaa>3|3;d_Ne-Fz`5$^iw;g3rr$1n$=~f;~w%=Ivhf9PqyWHtE^vbqd z2tWT3o`odx^Ss?RA9H!CDW#BALTij|VO8TZQ$Sf0kiug5G4Hfy93eS}%X{TW<_xjm zT1g)xpU|_!eblP;&&vSvr_OmLN6x@0) zGBqdfYnvAkCIIb_Fx@ZkH{8}+NAXR?DcZS(YX`?#GVSqZOaGBA(~~VgB-Wx; zc8dA&JbzC`oxc~_F*$8eyerUjEhX|L0D{NIH6#qFQaWq_6&drQOzBY6xo|*V`rb|_ z4RW?k$**46*Bd%YzY2%(f_&N1m!DGhB$G_jrc}>xX-y8#UVvJTL-k*bEc?c5YUKIn z1&tXlPj6)zmD7ByPII;|tY!OIh9diAqr7lm?-$Ad=#&C^Nkf`B&L~R)eIs7o`p=W)Nmrw>fu{IIYvz^Q zUq&%RD1<_FCHe>Y`o?Oe>Y>ltAL+T?OM@uAzjwZ2LTRCX&vt^YJ{!Nu!|S)BZ50*R z_(v2wJx=Mxy7e8XJWT|*yvLyh-6`eXE)R9jepcgTHaT(BM~)NrO`pYVexu++R1Fl> zNc!}=T>r)NSM<(>8o2|C!Lo1U2!{ImfVrAVYo23_`dKY%WXJY_?3 zQ2_y0zKZjdH8x1=tn}7VAKv|n%#vmMr@0E#k>r>+$z0nBl3u*k9RoRCv#g0ay?I9z zMmvLlQ~@oMcK%Yzw900M^JULIja*3-Nz;(uU@*h{JzNkjovm_}la2#F;UVwL(@o6{ zN-c3{cQtUrUj!ztHm%ZuPO2LgVU{jiG*pm$P3_oW!Ncwwu7>PaO{#=uSHrOQ0^DX_STH-dwu=WcHx>>^L}M`!}0qP9Y?;u_{H`r13kUf zPo~a28i-b^4ZRo+V`_nCu}8mzZd822ks8w;@>56{27S(@*dUdss%l92)P{svQIWl* zzx;04Csv&miE>X|g?aQOCFn=i>Ds9=q`G~SY>_DTV4miOU_*07=^Y~vCjHGzM&LY@ zUAq=Y)f++=nCIlDz*9S&nfR+m?N^$ez;e0o21`w7W=y|K!m7)g+QA`aG|#6dQUsE$ zuAr7xQA+d^KoG!6D zckMVx8~8^iQqIT}dGe^~r|`Xqi#(112$P-RQMOORU5;drgWY)xZk$98Nw<@yYcqf2 z9cG6>t7(%Mh%}3sq4}8!|LE;JxSBr0D2}XPMNqJ)SQ*tyKoKaQD6>Wb2oClz zkfK<^UI8W15K)FAsMQh=4Ph@MLu3dDX{9PhB&T&K zT z>yXP~B&rM=?@~C?a&W(4zV-%FS0^-RabKW{SJUUS!d*6>HI=5$ms1w>Hp!8&UB(&M zjkE0rr^5pylL*ci%y!dUJVwq9-FlEvx}HWpU$~G{b!VJc5P(S|jxJ8Sco`KZJ&{jv z^2tjw!^6(Len1H>e}$=@Zk|>p-R@DiY}2b113?y~$Y5wp&sf(kpAqut4#)KTmx0Y@ z$)q>@-OkgUv`HKK@@1qRH*|Uer&@hRXM9^*gm8{t=MwX*nR~z|b@6td_G32+XCaj> zGukTi>^OYfoWy;CVB~LmHyr=+U^X<_7LCD=_ky4!8&*+ZJXh-g(&Y=QL!a;y$PgK^ z29vP`XmB-rWKE6%I}5xqa2+e#0KKUi5ufV|q6h=g)f0>r1XgRJp6+f7WWQ?qUA?iO z7A!L{>>hT8RS=uOQhON6C;`hi<0?XVwB`eRF@kW1ng64)J`x#@M)p{Jkiml3q z0b)}W1LsD!lA59F7-b=^BatPSvcJP2l^drJv%Rub6NnsxNMe*-eWmrTc5HngiWgtv z0Z9}~mSmmW|N8bU7^LO78R%QC$)o&Tw)gNghn$W|a=lDKs2_Np4{L%3nHy^#xR)hB zfIJkIJihLFQfm6pJ5z2-r3Bqb`apO)j+6kX1*%`dx0LH#L=1zj}`l$Tj5) zX_GQx1==)@UI9>AYkN^eUIr6W$~QUO6Z?3qNr+-)b(sF1tyN^Nsb)>dRwy20 zPWs9w)p<|j@h6!>M?R-#Dq83y`^(;4B@TB88;UbtG}Ha^+(fg5Q8&ETg0@M+&XJeB z<@ypmmmw82H?ydrNMl5EdOGV4Q!60O&W~IN_%Q*`^W{#!V|{>}A~Cdw_v74p@Lp+4 z%}GWNe8axUVkxG;7KQuJColtQ`SNW~^@javl9w!8uEy#>VX8lyfz%fD`Is#bE*688 zCt7~R;?fy&#HRUpVU%dbAr8k?@-YAnyWv@Ti^6e?SToHoXNWnumAUEz4^Z<2o1B$w z>^A3JhD!||6{>h%GKoLIC?Qv2w>bISB(;{_E~tMm9XWr-a3pM%|u}@V1gO4QZ~Y0u5rG1 z-^^1Ol@b;&hjZHAvJNXz1%*TYpik~exsJ2F^B;YP_U35+zZhsK+_ir|X^0{H|Ijuf bizUiC-9{GORZt(1SG7NJ`gr-V9~1rtPL+FC literal 0 HcmV?d00001 diff --git a/evidence/pipeline.svg b/evidence/pipeline.svg new file mode 100644 index 00000000..56bd1015 --- /dev/null +++ b/evidence/pipeline.svg @@ -0,0 +1,2374 @@ + + + + + + + + 2026-10-27T18:09:20.663716 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/public-smoke.txt b/evidence/public-smoke.txt new file mode 100644 index 00000000..33326b12 --- /dev/null +++ b/evidence/public-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/resources-stderr.txt b/evidence/resources-stderr.txt new file mode 100644 index 00000000..55c9f9e5 --- /dev/null +++ b/evidence/resources-stderr.txt @@ -0,0 +1,6 @@ + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started diff --git a/evidence/resources.json b/evidence/resources.json new file mode 100644 index 00000000..8fa4ee6e --- /dev/null +++ b/evidence/resources.json @@ -0,0 +1,1410 @@ +{ + "method": "10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache", + "experiments": [ + { + "width": 320, + "height": 200, + "bytes": 1294, + "clients": 10, + "before": { + "memory.current": "25993216", + "memory.peak": "26140672", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 25546752\nfile 0\nkernel 540672\nkernel_stack 16384\npagetables 196608\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 25509888\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 78624\nslab_unreclaimable 200616\nslab 279240\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 9455\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 217755\nuser_usec 202908\nsystem_usec 14846\ncore_sched.force_idle_usec 0\nnr_periods 2\nnr_throttled 1\nthrottled_usec 27668\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "67489792", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 12288\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 309872\nslab 392928\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 115649\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3037727\nuser_usec 2854014\nsystem_usec 183713\ncore_sched.force_idle_usec 0\nnr_periods 33\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0001, + "worker": { + "memory.current": "30593024", + "memory.peak": "30740480", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 29769728\nfile 0\nkernel 548864\nkernel_stack 16384\npagetables 200704\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 29741056\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 210896\nslab 292912\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13515\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 361161\nuser_usec 337348\nsystem_usec 23812\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 3\nthrottled_usec 189684\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.222, + "worker": { + "memory.current": "46673920", + "memory.peak": "46821376", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 45789184\nfile 0\nkernel 667648\nkernel_stack 16384\npagetables 270336\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 45752320\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 254584\nslab 336600\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 21899\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 551236\nuser_usec 513426\nsystem_usec 37810\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 5\nthrottled_usec 314604\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.4257, + "worker": { + "memory.current": "76713984", + "memory.peak": "76713984", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75374592\nfile 16384\nkernel 937984\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75313152\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 343120\nslab 431152\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 33898\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 762649\nuser_usec 705972\nsystem_usec 56676\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 7\nthrottled_usec 412358\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.6332, + "worker": { + "memory.current": "75657216", + "memory.peak": "76713984", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74448896\nfile 20480\nkernel 933888\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74432512\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 340424\nslab 428456\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 40861\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 940967\nuser_usec 874394\nsystem_usec 66573\ncore_sched.force_idle_usec 0\nnr_periods 10\nnr_throttled 9\nthrottled_usec 498775\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8211, + "worker": { + "memory.current": "66772992", + "memory.peak": "80965632", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65732608\nfile 16384\nkernel 794624\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65708032\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 305592\nslab 389888\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 45711\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1139170\nuser_usec 1062496\nsystem_usec 76674\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 11\nthrottled_usec 553897\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.024, + "worker": { + "memory.current": "66785280", + "memory.peak": "81715200", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65732608\nfile 16384\nkernel 802816\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65732608\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 87376\nslab_unreclaimable 305824\nslab 393200\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 52903\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1343743\nuser_usec 1254425\nsystem_usec 89318\ncore_sched.force_idle_usec 0\nnr_periods 14\nnr_throttled 13\nthrottled_usec 643571\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.2249, + "worker": { + "memory.current": "66695168", + "memory.peak": "81715200", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65744896\nfile 32768\nkernel 806912\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 8192\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65761280\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 87576\nslab_unreclaimable 316592\nslab 404168\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 60107\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1548475\nuser_usec 1449214\nsystem_usec 99261\ncore_sched.force_idle_usec 0\nnr_periods 16\nnr_throttled 15\nthrottled_usec 693176\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4268, + "worker": { + "memory.current": "66973696", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65957888\nfile 28672\nkernel 942080\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65961984\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88632\nslab_unreclaimable 322992\nslab 411624\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 67356\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1750126\nuser_usec 1637151\nsystem_usec 112975\ncore_sched.force_idle_usec 0\nnr_periods 18\nnr_throttled 16\nthrottled_usec 791006\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6289, + "worker": { + "memory.current": "81682432", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80556032\nfile 24576\nkernel 1040384\nkernel_stack 49152\npagetables 450560\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 12288\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 80568320\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 90792\nslab_unreclaimable 405952\nslab 496744\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 74920\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1943185\nuser_usec 1818774\nsystem_usec 124411\ncore_sched.force_idle_usec 0\nnr_periods 20\nnr_throttled 17\nthrottled_usec 838763\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8231, + "worker": { + "memory.current": "80900096", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79634432\nfile 16384\nkernel 999424\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79622144\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 380472\nslab 468504\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 81785\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2131478\nuser_usec 1996963\nsystem_usec 134515\ncore_sched.force_idle_usec 0\nnr_periods 22\nnr_throttled 18\nthrottled_usec 936012\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0172, + "worker": { + "memory.current": "81068032", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79790080\nfile 16384\nkernel 1015808\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79736832\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 397872\nslab 485904\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 89023\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2327586\nuser_usec 2184030\nsystem_usec 143555\ncore_sched.force_idle_usec 0\nnr_periods 23\nnr_throttled 18\nthrottled_usec 936012\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2159, + "worker": { + "memory.current": "80629760", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79171584\nfile 20480\nkernel 966656\nkernel_stack 49152\npagetables 430080\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79151104\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 355496\nslab 443528\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95908\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2514361\nuser_usec 2360865\nsystem_usec 153495\ncore_sched.force_idle_usec 0\nnr_periods 25\nnr_throttled 19\nthrottled_usec 1031276\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.3631, + "worker": { + "memory.current": "71213056", + "memory.peak": "82165760", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 69812224\nfile 20480\nkernel 925696\nkernel_stack 49152\npagetables 397312\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 69742592\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 86192\nslab_unreclaimable 343336\nslab 429528\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 100672\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2666143\nuser_usec 2502667\nsystem_usec 163475\ncore_sched.force_idle_usec 0\nnr_periods 27\nnr_throttled 21\nthrottled_usec 1126801\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.5456, + "worker": { + "memory.current": "67538944", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66646016\nfile 28672\nkernel 806912\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66662400\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 85096\nslab_unreclaimable 317720\nslab 402816\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 107013\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2860171\nuser_usec 2686677\nsystem_usec 173493\ncore_sched.force_idle_usec 0\nnr_periods 29\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.7395, + "worker": { + "memory.current": "67612672", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66646016\nfile 12288\nkernel 794624\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66646016\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 309872\nslab 392928\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 109894\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2920115\nuser_usec 2742447\nsystem_usec 177668\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.9027, + "worker": { + "memory.current": "67510272", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 12288\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 306320\nslab 389376\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 112771\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2978023\nuser_usec 2797357\nsystem_usec 180666\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 0.05672016600000007, + "ready_seconds": 1.678756416, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.0797383330000001, + "ready_seconds": 1.9898555410000003, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.06636754199999984, + "ready_seconds": 2.3032809579999998, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 0.07966337499999998, + "ready_seconds": 2.923389875, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.07643879200000003, + "ready_seconds": 2.608790333, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.04658795799999993, + "ready_seconds": 1.019897792, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 0.06349704200000006, + "ready_seconds": 0.717395875, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.06630558300000011, + "ready_seconds": 1.661077125, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 0.06494524999999984, + "ready_seconds": 1.346463375, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.06040191699999986, + "ready_seconds": 2.305937583, + "status": "ready" + } + ], + "seconds": 3.093689791 + }, + { + "width": 1800, + "height": 1400, + "bytes": 7563763, + "clients": 10, + "before": { + "memory.current": "23658496", + "memory.peak": "23810048", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 22904832\nfile 0\nkernel 512000\nkernel_stack 16384\npagetables 188416\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 22904832\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 78624\nslab_unreclaimable 188336\nslab 266960\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 8545\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 195466\nuser_usec 180783\nsystem_usec 14682\ncore_sched.force_idle_usec 0\nnr_periods 3\nnr_throttled 1\nthrottled_usec 45259\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "68448256", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67538944\nfile 28672\nkernel 798720\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67538944\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311312\nslab 394368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 298507\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6584973\nuser_usec 6143467\nsystem_usec 441506\ncore_sched.force_idle_usec 0\nnr_periods 75\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0002, + "worker": { + "memory.current": "29175808", + "memory.peak": "29544448", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 28377088\nfile 0\nkernel 544768\nkernel_stack 16384\npagetables 200704\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 28368896\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 199184\nslab 281200\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13036\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 372577\nuser_usec 350883\nsystem_usec 21694\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 2\nthrottled_usec 100680\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.2164, + "worker": { + "memory.current": "45006848", + "memory.peak": "45006848", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 43810816\nfile 0\nkernel 663552\nkernel_stack 16384\npagetables 266240\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 43757568\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82856\nslab_unreclaimable 258800\nslab 341656\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 21272\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 573424\nuser_usec 538003\nsystem_usec 35420\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 4\nthrottled_usec 139589\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.3946, + "worker": { + "memory.current": "50413568", + "memory.peak": "50561024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 49602560\nfile 4096\nkernel 745472\nkernel_stack 32768\npagetables 294912\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 49602560\ninactive_file 4096\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 296960\nslab 380016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 25614\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 683616\nuser_usec 637335\nsystem_usec 46281\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 5\nthrottled_usec 183385\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.5611, + "worker": { + "memory.current": "50548736", + "memory.peak": "50696192", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 49602560\nfile 4096\nkernel 741376\nkernel_stack 32768\npagetables 290816\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 49602560\ninactive_file 4096\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 295968\nslab 379024\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 28491\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 743694\nuser_usec 694372\nsystem_usec 49321\ncore_sched.force_idle_usec 0\nnr_periods 9\nnr_throttled 5\nthrottled_usec 183385\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.7123, + "worker": { + "memory.current": "66285568", + "memory.peak": "66433024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65290240\nfile 16384\nkernel 786432\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65290240\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 306256\nslab 390552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 36810\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 890617\nuser_usec 813340\nsystem_usec 77277\ncore_sched.force_idle_usec 0\nnr_periods 11\nnr_throttled 6\nthrottled_usec 206722\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8867, + "worker": { + "memory.current": "66142208", + "memory.peak": "66433024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65302528\nfile 12288\nkernel 786432\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65290240\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 301712\nslab 386008\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 39693\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 952467\nuser_usec 872127\nsystem_usec 80340\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 6\nthrottled_usec 206722\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.0403, + "worker": { + "memory.current": "84230144", + "memory.peak": "84418560", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75423744\nfile 7593984\nkernel 962560\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 82972672\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 344560\nslab 450352\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 45282\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1087134\nuser_usec 992095\nsystem_usec 95039\ncore_sched.force_idle_usec 0\nnr_periods 15\nnr_throttled 7\nthrottled_usec 293202\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.253, + "worker": { + "memory.current": "113782784", + "memory.peak": "113782784", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 104636416\nfile 7610368\nkernel 1118208\nkernel_stack 49152\npagetables 507904\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 112173056\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 414480\nslab 521464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 57704\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1298250\nuser_usec 1183670\nsystem_usec 114580\ncore_sched.force_idle_usec 0\nnr_periods 17\nnr_throttled 9\nthrottled_usec 398918\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4618, + "worker": { + "memory.current": "75698176", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66265088\nfile 8372224\nkernel 806912\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74604544\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 308728\nslab 414544\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 64522\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1495060\nuser_usec 1363487\nsystem_usec 131573\ncore_sched.force_idle_usec 0\nnr_periods 19\nnr_throttled 11\nthrottled_usec 458319\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6743, + "worker": { + "memory.current": "90038272", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81072128\nfile 7593984\nkernel 1019904\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88625152\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379064\nslab 484856\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 71423\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1677684\nuser_usec 1529797\nsystem_usec 147886\ncore_sched.force_idle_usec 0\nnr_periods 21\nnr_throttled 12\nthrottled_usec 536499\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8627, + "worker": { + "memory.current": "121225216", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 112369664\nfile 7610368\nkernel 1138688\nkernel_stack 49152\npagetables 524288\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 119922688\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 415728\nslab 522712\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 84185\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1870527\nuser_usec 1709663\nsystem_usec 160863\ncore_sched.force_idle_usec 0\nnr_periods 23\nnr_throttled 14\nthrottled_usec 650482\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0596, + "worker": { + "memory.current": "75735040", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66523136\nfile 8372224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74850304\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 309976\nslab 415792\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 89179\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2032115\nuser_usec 1860152\nsystem_usec 171963\ncore_sched.force_idle_usec 0\nnr_periods 24\nnr_throttled 15\nthrottled_usec 658121\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2097, + "worker": { + "memory.current": "88981504", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79937536\nfile 7598080\nkernel 1028096\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87494656\ninactive_file 32768\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 392328\nslab 498120\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95737\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2209175\nuser_usec 2025323\nsystem_usec 183852\ncore_sched.force_idle_usec 0\nnr_periods 26\nnr_throttled 16\nthrottled_usec 696184\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.383, + "worker": { + "memory.current": "115290112", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 106123264\nfile 7610368\nkernel 1122304\nkernel_stack 49152\npagetables 507904\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 113680384\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 419272\nslab 525064\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 107250\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2388567\nuser_usec 2192880\nsystem_usec 195687\ncore_sched.force_idle_usec 0\nnr_periods 28\nnr_throttled 18\nthrottled_usec 798513\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.5736, + "worker": { + "memory.current": "78794752", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66629632\nfile 8372224\nkernel 827392\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 2797568\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74973184\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 109352\nslab_unreclaimable 312008\nslab 421360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 113800\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2580929\nuser_usec 2375202\nsystem_usec 205726\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 20\nthrottled_usec 903938\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.7743, + "worker": { + "memory.current": "88862720", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79921152\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87429120\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 374264\nslab 480056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 120326\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2743632\nuser_usec 2528947\nsystem_usec 214684\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 21\nthrottled_usec 991261\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.9742, + "worker": { + "memory.current": "117268480", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 108314624\nfile 7610368\nkernel 1134592\nkernel_stack 49152\npagetables 520192\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 115888128\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 418352\nslab 525336\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 132381\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2940734\nuser_usec 2702402\nsystem_usec 238331\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 23\nthrottled_usec 1088687\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.1634, + "worker": { + "memory.current": "76079104", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66633728\nfile 8372224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74977280\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 311360\nslab 417176\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 138400\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3117143\nuser_usec 2867732\nsystem_usec 249410\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 25\nthrottled_usec 1141971\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.3522, + "worker": { + "memory.current": "90132480", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81235968\nfile 7593984\nkernel 1028096\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88784896\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379552\nslab 485344\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 145311\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3299749\nuser_usec 3039554\nsystem_usec 260195\ncore_sched.force_idle_usec 0\nnr_periods 37\nnr_throttled 25\nthrottled_usec 1141971\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.5539, + "worker": { + "memory.current": "126955520", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 117960704\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 532480\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 125509632\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 414424\nslab 521408\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 159342\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3517509\nuser_usec 3247462\nsystem_usec 270046\ncore_sched.force_idle_usec 0\nnr_periods 40\nnr_throttled 28\nthrottled_usec 1316301\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.7603, + "worker": { + "memory.current": "72503296", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66854912\nfile 3964928\nkernel 806912\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 1036288\nvmalloc 16384\nshmem 3932160\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 70770688\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 94464\nslab_unreclaimable 310936\nslab 405400\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 163062\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3674848\nuser_usec 3388522\nsystem_usec 286326\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 28\nthrottled_usec 1316301\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.9159, + "worker": { + "memory.current": "101822464", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 92762112\nfile 7593984\nkernel 1052672\nkernel_stack 49152\npagetables 466944\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 100274176\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 386520\nslab 492312\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 172742\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3863955\nuser_usec 3568778\nsystem_usec 295176\ncore_sched.force_idle_usec 0\nnr_periods 43\nnr_throttled 30\nthrottled_usec 1379485\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.0905, + "worker": { + "memory.current": "122384384", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 113397760\nfile 7610368\nkernel 1150976\nkernel_stack 49152\npagetables 528384\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120975360\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 418992\nslab 525976\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 182834\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4035822\nuser_usec 3727955\nsystem_usec 307867\ncore_sched.force_idle_usec 0\nnr_periods 45\nnr_throttled 32\nthrottled_usec 1480906\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.2686, + "worker": { + "memory.current": "76140544", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66908160\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75251712\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 313240\nslab 419056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 187677\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4195036\nuser_usec 3874239\nsystem_usec 320796\ncore_sched.force_idle_usec 0\nnr_periods 46\nnr_throttled 33\nthrottled_usec 1495617\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.4545, + "worker": { + "memory.current": "90464256", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81584128\nfile 7593984\nkernel 1019904\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 89149440\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 383696\nslab 489488\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 194607\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4378271\nuser_usec 4044745\nsystem_usec 333526\ncore_sched.force_idle_usec 0\nnr_periods 48\nnr_throttled 34\nthrottled_usec 1549546\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.659, + "worker": { + "memory.current": "125095936", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 116039680\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 528384\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 123584512\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 419808\nslab 526792\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 208081\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4587405\nuser_usec 4242955\nsystem_usec 344450\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 37\nthrottled_usec 1749887\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.856, + "worker": { + "memory.current": "76509184", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67264512\nfile 8372224\nkernel 819200\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75608064\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 314056\nslab 419872\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 212364\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4735651\nuser_usec 4383984\nsystem_usec 351667\ncore_sched.force_idle_usec 0\nnr_periods 52\nnr_throttled 38\nthrottled_usec 1754778\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.0193, + "worker": { + "memory.current": "76652544", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67260416\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75603968\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 314056\nslab 419872\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 215245\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4796252\nuser_usec 4441669\nsystem_usec 354582\ncore_sched.force_idle_usec 0\nnr_periods 54\nnr_throttled 38\nthrottled_usec 1754778\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.1811, + "worker": { + "memory.current": "89493504", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80687104\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88195072\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 381256\nslab 487048\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 221804\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4957854\nuser_usec 4593277\nsystem_usec 364577\ncore_sched.force_idle_usec 0\nnr_periods 56\nnr_throttled 39\nthrottled_usec 1829478\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.3667, + "worker": { + "memory.current": "118333440", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 109322240\nfile 7610368\nkernel 1134592\nkernel_stack 49152\npagetables 516096\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 116891648\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 420624\nslab 527608\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 233922\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5153684\nuser_usec 4778296\nsystem_usec 375388\ncore_sched.force_idle_usec 0\nnr_periods 58\nnr_throttled 41\nthrottled_usec 1938995\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.5561, + "worker": { + "memory.current": "76836864", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67391488\nfile 8372224\nkernel 819200\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75735040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 110416\nslab_unreclaimable 314912\nslab 425328\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 240645\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5335204\nuser_usec 4950658\nsystem_usec 384545\ncore_sched.force_idle_usec 0\nnr_periods 59\nnr_throttled 42\nthrottled_usec 1950630\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.7535, + "worker": { + "memory.current": "89817088", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80695296\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88252416\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379344\nslab 485136\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 247172\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5502620\nuser_usec 5107103\nsystem_usec 395516\ncore_sched.force_idle_usec 0\nnr_periods 62\nnr_throttled 44\nthrottled_usec 2082147\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.9541, + "worker": { + "memory.current": "121753600", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 112652288\nfile 7610368\nkernel 1138688\nkernel_stack 49152\npagetables 520192\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120221696\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 421440\nslab 528424\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 260098\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5706772\nuser_usec 5302254\nsystem_usec 404518\ncore_sched.force_idle_usec 0\nnr_periods 64\nnr_throttled 46\nthrottled_usec 2189973\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.1564, + "worker": { + "memory.current": "76931072", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67411968\nfile 8372224\nkernel 827392\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75735040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 315688\nslab 421504\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 265246\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5876598\nuser_usec 5464049\nsystem_usec 412549\ncore_sched.force_idle_usec 0\nnr_periods 65\nnr_throttled 47\nthrottled_usec 2198351\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.3131, + "worker": { + "memory.current": "101044224", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 92147712\nfile 7593984\nkernel 1052672\nkernel_stack 49152\npagetables 462848\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 99667968\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105584\nslab_unreclaimable 389784\nslab 495368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 274641\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6068527\nuser_usec 5641347\nsystem_usec 427180\ncore_sched.force_idle_usec 0\nnr_periods 67\nnr_throttled 49\nthrottled_usec 2248335\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.4887, + "worker": { + "memory.current": "122351616", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 113147904\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 524288\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120672256\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 107184\nslab_unreclaimable 424736\nslab 531920\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 284818\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6244347\nuser_usec 5814145\nsystem_usec 430201\ncore_sched.force_idle_usec 0\nnr_periods 69\nnr_throttled 51\nthrottled_usec 2354429\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.6681, + "worker": { + "memory.current": "77053952", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67534848\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75878400\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 316504\nslab 422320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 289874\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6411394\nuser_usec 5976052\nsystem_usec 435341\ncore_sched.force_idle_usec 0\nnr_periods 71\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.8605, + "worker": { + "memory.current": "68456448", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67538944\nfile 28672\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67538944\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311088\nslab 394144\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 292753\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6471910\nuser_usec 6034578\nsystem_usec 437331\ncore_sched.force_idle_usec 0\nnr_periods 72\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.0149, + "worker": { + "memory.current": "68554752", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67543040\nfile 28672\nkernel 794624\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67543040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311312\nslab 394368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 295630\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6528449\nuser_usec 6089129\nsystem_usec 439319\ncore_sched.force_idle_usec 0\nnr_periods 74\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 1.0932762500000006, + "ready_seconds": 6.383178083, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.744891333, + "ready_seconds": 2.9478767920000006, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.7652272499999997, + "ready_seconds": 3.889252625, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 1.0959794999999994, + "ready_seconds": 5.7605763340000005, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.747986042, + "ready_seconds": 2.326131459, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.978114916, + "ready_seconds": 4.420152333000001, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 1.102462333, + "ready_seconds": 7.024895291999999, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.6511005000000001, + "ready_seconds": 1.919602084, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 1.0380668750000002, + "ready_seconds": 5.422940084, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.7656660830000002, + "ready_seconds": 3.5762043749999997, + "status": "ready" + } + ], + "seconds": 7.2014601659999995 + }, + { + "width": 10000, + "height": 4000, + "bytes": 533859, + "clients": 10, + "before": { + "memory.current": "26193920", + "memory.peak": "26193920", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 25034752\nfile 126976\nkernel 516096\nkernel_stack 16384\npagetables 184320\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 25026560\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 79544\nslab_unreclaimable 196456\nslab 276000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 9312\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 213801\nuser_usec 195818\nsystem_usec 17983\ncore_sched.force_idle_usec 0\nnr_periods 2\nnr_throttled 1\nthrottled_usec 47256\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "67629056", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 139264\nkernel 794624\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 83976\nslab_unreclaimable 310864\nslab 394840\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 698354\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11899130\nuser_usec 11309189\nsystem_usec 589941\ncore_sched.force_idle_usec 0\nnr_periods 120\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0001, + "worker": { + "memory.current": "30728192", + "memory.peak": "30875648", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 29769728\nfile 126976\nkernel 552960\nkernel_stack 16384\npagetables 204800\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 29741056\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 82936\nslab_unreclaimable 203176\nslab 286112\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13510\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 368020\nuser_usec 346137\nsystem_usec 21882\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 3\nthrottled_usec 209770\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.2346, + "worker": { + "memory.current": "48369664", + "memory.peak": "48517120", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 47390720\nfile 126976\nkernel 700416\nkernel_stack 16384\npagetables 278528\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 47357952\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 82936\nslab_unreclaimable 280032\nslab 362968\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 22319\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 568126\nuser_usec 531312\nsystem_usec 36813\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 5\nthrottled_usec 326949\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.4368, + "worker": { + "memory.current": "78430208", + "memory.peak": "78430208", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 76587008\nfile 675840\nkernel 954368\nkernel_stack 49152\npagetables 421888\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 77107200\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 343600\nslab 434920\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 34179\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 765407\nuser_usec 704850\nsystem_usec 60557\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 7\nthrottled_usec 430066\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.6374, + "worker": { + "memory.current": "242069504", + "memory.peak": "242216960", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 239763456\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 240287744\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380000\nslab 471320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 77128\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 967740\nuser_usec 874248\nsystem_usec 93491\ncore_sched.force_idle_usec 0\nnr_periods 10\nnr_throttled 9\nthrottled_usec 535278\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8379, + "worker": { + "memory.current": "244908032", + "memory.peak": "245055488", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242790400\nfile 675840\nkernel 1327104\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243273728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380728\nslab 472048\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 80744\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1171239\nuser_usec 1072807\nsystem_usec 98431\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 11\nthrottled_usec 642609\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.0389, + "worker": { + "memory.current": "244006912", + "memory.peak": "245055488", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241872896\nfile 684032\nkernel 1331200\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242372608\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 380536\nslab 472056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 84399\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1382200\nuser_usec 1280845\nsystem_usec 101354\ncore_sched.force_idle_usec 0\nnr_periods 14\nnr_throttled 13\nthrottled_usec 734464\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.2484, + "worker": { + "memory.current": "258646016", + "memory.peak": "258793472", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 256446464\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 798720\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 256978944\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 382760\nslab 474480\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 90841\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1594891\nuser_usec 1486645\nsystem_usec 108246\ncore_sched.force_idle_usec 0\nnr_periods 16\nnr_throttled 15\nthrottled_usec 831199\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4458, + "worker": { + "memory.current": "89042944", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 86626304\nfile 688128\nkernel 1400832\nkernel_stack 49152\npagetables 487424\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87175168\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 415152\nslab 507072\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95495\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1786454\nuser_usec 1663250\nsystem_usec 123203\ncore_sched.force_idle_usec 0\nnr_periods 18\nnr_throttled 17\nthrottled_usec 944761\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6549, + "worker": { + "memory.current": "80568320", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 78856192\nfile 675840\nkernel 991232\nkernel_stack 49152\npagetables 430080\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79380480\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 377848\nslab 469168\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 102100\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1991706\nuser_usec 1855274\nsystem_usec 136431\ncore_sched.force_idle_usec 0\nnr_periods 20\nnr_throttled 18\nthrottled_usec 1023149\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8447, + "worker": { + "memory.current": "243773440", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241299456\nfile 675840\nkernel 1314816\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241807360\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380984\nslab 472304\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 144711\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2229110\nuser_usec 2063622\nsystem_usec 165488\ncore_sched.force_idle_usec 0\nnr_periods 22\nnr_throttled 20\nthrottled_usec 1115720\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0953, + "worker": { + "memory.current": "245940224", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243769344\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244281344\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 377432\nslab 468752\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 148181\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2414292\nuser_usec 2245251\nsystem_usec 169040\ncore_sched.force_idle_usec 0\nnr_periods 24\nnr_throttled 22\nthrottled_usec 1144849\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2575, + "worker": { + "memory.current": "244084736", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241799168\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242319360\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 382776\nslab 474296\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 151579\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2575986\nuser_usec 2403856\nsystem_usec 172130\ncore_sched.force_idle_usec 0\nnr_periods 26\nnr_throttled 24\nthrottled_usec 1263685\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.4381, + "worker": { + "memory.current": "256630784", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 254418944\nfile 684032\nkernel 1351680\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 254914560\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383016\nslab 474736\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 157542\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2764644\nuser_usec 2590486\nsystem_usec 174158\ncore_sched.force_idle_usec 0\nnr_periods 28\nnr_throttled 26\nthrottled_usec 1372197\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.6419, + "worker": { + "memory.current": "89264128", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 86876160\nfile 688128\nkernel 1388544\nkernel_stack 49152\npagetables 475136\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87425024\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 416400\nslab 508320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 162746\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2967367\nuser_usec 2782342\nsystem_usec 185025\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 28\nthrottled_usec 1471876\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.8367, + "worker": { + "memory.current": "81100800", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79085568\nfile 675840\nkernel 999424\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79597568\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 378888\nslab 470208\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 169350\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3155688\nuser_usec 2960634\nsystem_usec 195053\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 29\nthrottled_usec 1566552\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.036, + "worker": { + "memory.current": "243621888", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241451008\nfile 675840\nkernel 1318912\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241938432\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 381800\nslab 473120\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 211934\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3362147\nuser_usec 3145362\nsystem_usec 216784\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 31\nthrottled_usec 1673525\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.2337, + "worker": { + "memory.current": "246566912", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 244281344\nfile 675840\nkernel 1318912\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244797440\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 382792\nslab 474112\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 215505\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3562352\nuser_usec 3341631\nsystem_usec 220720\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 33\nthrottled_usec 1777266\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.4408, + "worker": { + "memory.current": "250142720", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247869440\nfile 684032\nkernel 1335296\nkernel_stack 49152\npagetables 770048\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 248414208\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383832\nslab 475552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 220266\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3767453\nuser_usec 3539875\nsystem_usec 227577\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 35\nthrottled_usec 1869788\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.6462, + "worker": { + "memory.current": "262807552", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 260542464\nfile 684032\nkernel 1355776\nkernel_stack 49152\npagetables 790528\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 261038080\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383832\nslab 475552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 226232\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3973493\nuser_usec 3743909\nsystem_usec 229584\ncore_sched.force_idle_usec 0\nnr_periods 40\nnr_throttled 37\nthrottled_usec 1959331\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.8385, + "worker": { + "memory.current": "76562432", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74629120\nfile 675840\nkernel 942080\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75153408\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 341424\nslab 432744\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 232394\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4137903\nuser_usec 3897014\nsystem_usec 240888\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 38\nthrottled_usec 1976863\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.9948, + "worker": { + "memory.current": "162709504", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 160632832\nfile 675840\nkernel 1200128\nkernel_stack 49152\npagetables 630784\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 161120256\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383640\nslab 474960\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 256512\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4322051\nuser_usec 4069334\nsystem_usec 252716\ncore_sched.force_idle_usec 0\nnr_periods 43\nnr_throttled 40\nthrottled_usec 2044525\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.1698, + "worker": { + "memory.current": "244023296", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241827840\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242343936\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383872\nslab 475192\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 279212\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4494084\nuser_usec 4231481\nsystem_usec 262602\ncore_sched.force_idle_usec 0\nnr_periods 45\nnr_throttled 42\nthrottled_usec 2146593\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.3559, + "worker": { + "memory.current": "242335744", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 240181248\nfile 679936\nkernel 1331200\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 240713728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 379328\nslab 470648\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 282693\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4684066\nuser_usec 4417542\nsystem_usec 266524\ncore_sched.force_idle_usec 0\nnr_periods 47\nnr_throttled 44\nthrottled_usec 2245102\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.5446, + "worker": { + "memory.current": "249327616", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247123968\nfile 684032\nkernel 1339392\nkernel_stack 49152\npagetables 774144\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 247652352\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 384896\nslab 476616\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 287265\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4871121\nuser_usec 4594759\nsystem_usec 276361\ncore_sched.force_idle_usec 0\nnr_periods 49\nnr_throttled 46\nthrottled_usec 2354663\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.744, + "worker": { + "memory.current": "261533696", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 259211264\nfile 684032\nkernel 1368064\nkernel_stack 49152\npagetables 798720\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 259756032\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 385640\nslab 477360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 293088\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5068078\nuser_usec 4786849\nsystem_usec 281228\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 48\nthrottled_usec 2457492\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.9437, + "worker": { + "memory.current": "76525568", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74838016\nfile 679936\nkernel 942080\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75317248\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 90272\nslab_unreclaimable 341192\nslab 431464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 299629\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5246669\nuser_usec 4951600\nsystem_usec 295069\ncore_sched.force_idle_usec 0\nnr_periods 53\nnr_throttled 50\nthrottled_usec 2542069\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.13, + "worker": { + "memory.current": "214814720", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 212742144\nfile 675840\nkernel 1282048\nkernel_stack 49152\npagetables 712704\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 213237760\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383696\nslab 475016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 336420\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5451473\nuser_usec 5142543\nsystem_usec 308930\ncore_sched.force_idle_usec 0\nnr_periods 55\nnr_throttled 52\nthrottled_usec 2659554\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.3342, + "worker": { + "memory.current": "245424128", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243212288\nfile 675840\nkernel 1327104\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243716096\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383432\nslab 474752\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 346733\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5660402\nuser_usec 5346487\nsystem_usec 313914\ncore_sched.force_idle_usec 0\nnr_periods 57\nnr_throttled 54\nthrottled_usec 2758211\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.5383, + "worker": { + "memory.current": "244420608", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242167808\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242708480\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 94280\nslab_unreclaimable 384272\nslab 478552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 351058\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5884173\nuser_usec 5564607\nsystem_usec 319565\ncore_sched.force_idle_usec 0\nnr_periods 59\nnr_throttled 56\nthrottled_usec 2849581\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.7466, + "worker": { + "memory.current": "254889984", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 252788736\nfile 684032\nkernel 1351680\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 253308928\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386456\nslab 478176\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 356529\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6063524\nuser_usec 5740057\nsystem_usec 323467\ncore_sched.force_idle_usec 0\nnr_periods 61\nnr_throttled 58\nthrottled_usec 2956295\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.9308, + "worker": { + "memory.current": "249012224", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 246657024\nfile 684032\nkernel 1400832\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 247148544\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 397936\nslab 489656\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 362024\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6258018\nuser_usec 5927656\nsystem_usec 330361\ncore_sched.force_idle_usec 0\nnr_periods 63\nnr_throttled 60\nthrottled_usec 3075398\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.1266, + "worker": { + "memory.current": "78884864", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 76996608\nfile 675840\nkernel 946176\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 77520896\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 352096\nslab 443416\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 368045\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6434616\nuser_usec 6088381\nsystem_usec 346235\ncore_sched.force_idle_usec 0\nnr_periods 64\nnr_throttled 61\nthrottled_usec 3077896\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.2725, + "worker": { + "memory.current": "181219328", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 179220480\nfile 675840\nkernel 1220608\nkernel_stack 49152\npagetables 655360\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 179744768\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384512\nslab 475832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 396110\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6597949\nuser_usec 6238945\nsystem_usec 359004\ncore_sched.force_idle_usec 0\nnr_periods 66\nnr_throttled 63\nthrottled_usec 3188604\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.4543, + "worker": { + "memory.current": "244588544", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242282496\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242774016\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384248\nslab 475568\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 414389\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6774241\nuser_usec 6405332\nsystem_usec 368909\ncore_sched.force_idle_usec 0\nnr_periods 68\nnr_throttled 65\nthrottled_usec 3290034\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.6434, + "worker": { + "memory.current": "243073024", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 240910336\nfile 679936\nkernel 1327104\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241434624\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384248\nslab 475568\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 417932\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6974922\nuser_usec 6603072\nsystem_usec 371850\ncore_sched.force_idle_usec 0\nnr_periods 70\nnr_throttled 67\nthrottled_usec 3392666\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.8408, + "worker": { + "memory.current": "251777024", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 249483264\nfile 684032\nkernel 1347584\nkernel_stack 49152\npagetables 778240\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 249974784\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386280\nslab 478000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 422902\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7163878\nuser_usec 6786101\nsystem_usec 377776\ncore_sched.force_idle_usec 0\nnr_periods 72\nnr_throttled 69\nthrottled_usec 3500325\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.0391, + "worker": { + "memory.current": "263426048", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 261042176\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 261578752\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386280\nslab 478000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 428597\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7359035\nuser_usec 6974274\nsystem_usec 384760\ncore_sched.force_idle_usec 0\nnr_periods 74\nnr_throttled 71\nthrottled_usec 3600855\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.2355, + "worker": { + "memory.current": "77447168", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75452416\nfile 679936\nkernel 933888\nkernel_stack 49152\npagetables 405504\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75964416\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 348040\nslab 439360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 434835\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7542243\nuser_usec 7144032\nsystem_usec 398211\ncore_sched.force_idle_usec 0\nnr_periods 76\nnr_throttled 72\nthrottled_usec 3639550\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.4362, + "worker": { + "memory.current": "236023808", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 233701376\nfile 675840\nkernel 1302528\nkernel_stack 49152\npagetables 737280\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 234217472\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 385328\nslab 476648\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 476596\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7745487\nuser_usec 7326518\nsystem_usec 418969\ncore_sched.force_idle_usec 0\nnr_periods 78\nnr_throttled 74\nthrottled_usec 3743288\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.64, + "worker": { + "memory.current": "245723136", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243539968\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244039680\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386056\nslab 477376\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 481879\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7946846\nuser_usec 7523846\nsystem_usec 422999\ncore_sched.force_idle_usec 0\nnr_periods 80\nnr_throttled 76\nthrottled_usec 3844224\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.8336, + "worker": { + "memory.current": "244654080", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242450432\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242978816\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 385864\nslab 477384\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 485496\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8146758\nuser_usec 7717825\nsystem_usec 428933\ncore_sched.force_idle_usec 0\nnr_periods 82\nnr_throttled 78\nthrottled_usec 3951186\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.0329, + "worker": { + "memory.current": "257679360", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 255352832\nfile 684032\nkernel 1355776\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 255885312\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 387096\nslab 478816\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 491523\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8340638\nuser_usec 7906612\nsystem_usec 434026\ncore_sched.force_idle_usec 0\nnr_periods 84\nnr_throttled 80\nthrottled_usec 4061201\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.2365, + "worker": { + "memory.current": "90140672", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 87625728\nfile 688128\nkernel 1331200\nkernel_stack 49152\npagetables 475136\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88174592\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 419544\nslab 511464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 496681\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8537699\nuser_usec 8092573\nsystem_usec 445126\ncore_sched.force_idle_usec 0\nnr_periods 86\nnr_throttled 82\nthrottled_usec 4168912\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.4325, + "worker": { + "memory.current": "84443136", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 82571264\nfile 675840\nkernel 1060864\nkernel_stack 49152\npagetables 487424\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 83050496\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 385464\nslab 476784\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 504041\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8730749\nuser_usec 8276353\nsystem_usec 454395\ncore_sched.force_idle_usec 0\nnr_periods 88\nnr_throttled 83\nthrottled_usec 4259590\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.6327, + "worker": { + "memory.current": "244965376", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242761728\nfile 675840\nkernel 1343488\nkernel_stack 49152\npagetables 774144\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243290112\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386704\nslab 478024\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 546031\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8946723\nuser_usec 8476417\nsystem_usec 470305\ncore_sched.force_idle_usec 0\nnr_periods 90\nnr_throttled 85\nthrottled_usec 4359771\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.8408, + "worker": { + "memory.current": "243564544", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241434624\nfile 679936\nkernel 1339392\nkernel_stack 49152\npagetables 770048\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241967104\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 387672\nslab 479192\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 549584\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9148347\nuser_usec 8671974\nsystem_usec 476373\ncore_sched.force_idle_usec 0\nnr_periods 92\nnr_throttled 87\nthrottled_usec 4464704\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.0402, + "worker": { + "memory.current": "253980672", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 251596800\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 252100608\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 387912\nslab 479632\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 554937\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9352751\nuser_usec 8874345\nsystem_usec 478406\ncore_sched.force_idle_usec 0\nnr_periods 94\nnr_throttled 89\nthrottled_usec 4563624\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.2355, + "worker": { + "memory.current": "265728000", + "memory.peak": "265875456", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 263569408\nfile 684032\nkernel 1380352\nkernel_stack 49152\npagetables 806912\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 264114176\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 388904\nslab 480624\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 560732\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9548786\nuser_usec 9064499\nsystem_usec 484286\ncore_sched.force_idle_usec 0\nnr_periods 96\nnr_throttled 91\nthrottled_usec 4670267\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.4323, + "worker": { + "memory.current": "81321984", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79179776\nfile 675840\nkernel 958464\nkernel_stack 49152\npagetables 421888\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79708160\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 352808\nslab 444128\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 567281\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9734279\nuser_usec 9240914\nsystem_usec 493364\ncore_sched.force_idle_usec 0\nnr_periods 98\nnr_throttled 92\nthrottled_usec 4766580\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.6306, + "worker": { + "memory.current": "243724288", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241229824\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241737728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386696\nslab 478016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 609959\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9927515\nuser_usec 9413373\nsystem_usec 514142\ncore_sched.force_idle_usec 0\nnr_periods 100\nnr_throttled 94\nthrottled_usec 4872153\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.833, + "worker": { + "memory.current": "246476800", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 244035584\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244535296\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386696\nslab 478016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 613516\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10123398\nuser_usec 9606209\nsystem_usec 517188\ncore_sched.force_idle_usec 0\nnr_periods 102\nnr_throttled 96\nthrottled_usec 4980716\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.0328, + "worker": { + "memory.current": "245137408", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242970624\nfile 684032\nkernel 1339392\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243474432\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 387496\nslab 479016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 617134\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10324726\nuser_usec 9804569\nsystem_usec 520157\ncore_sched.force_idle_usec 0\nnr_periods 104\nnr_throttled 98\nthrottled_usec 5087414\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.2347, + "worker": { + "memory.current": "259846144", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 257601536\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 258109440\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 388728\nslab 480448\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 623583\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10526147\nuser_usec 10001031\nsystem_usec 525116\ncore_sched.force_idle_usec 0\nnr_periods 106\nnr_throttled 100\nthrottled_usec 5192242\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.4306, + "worker": { + "memory.current": "68222976", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66523136\nfile 692224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 8192\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67072000\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 89424\nslab_unreclaimable 316848\nslab 406272\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 628260\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10721910\nuser_usec 10185814\nsystem_usec 536095\ncore_sched.force_idle_usec 0\nnr_periods 108\nnr_throttled 102\nthrottled_usec 5294934\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.6262, + "worker": { + "memory.current": "126590976", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 124612608\nfile 675840\nkernel 1126400\nkernel_stack 49152\npagetables 557056\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 125140992\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 387528\nslab 478848\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 645792\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10917765\nuser_usec 10373713\nsystem_usec 544051\ncore_sched.force_idle_usec 0\nnr_periods 110\nnr_throttled 104\nthrottled_usec 5399951\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.8329, + "worker": { + "memory.current": "245424128", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243052544\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243552256\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 387512\nslab 478832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 677586\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11124368\nuser_usec 10561448\nsystem_usec 562919\ncore_sched.force_idle_usec 0\nnr_periods 112\nnr_throttled 106\nthrottled_usec 5505169\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.0327, + "worker": { + "memory.current": "244043776", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241819648\nfile 679936\nkernel 1327104\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242327552\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 388312\nslab 479832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 681168\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11327218\nuser_usec 10761303\nsystem_usec 565914\ncore_sched.force_idle_usec 0\nnr_periods 114\nnr_throttled 108\nthrottled_usec 5611110\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.2325, + "worker": { + "memory.current": "254652416", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 252379136\nfile 684032\nkernel 1359872\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 252887040\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 389544\nslab 481264\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 686623\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11527701\nuser_usec 10958812\nsystem_usec 568889\ncore_sched.force_idle_usec 0\nnr_periods 116\nnr_throttled 110\nthrottled_usec 5715900\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.438, + "worker": { + "memory.current": "249868288", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247463936\nfile 684032\nkernel 1376256\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 248000512\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 402504\nslab 494224\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 692425\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11732200\nuser_usec 11153383\nsystem_usec 578816\ncore_sched.force_idle_usec 0\nnr_periods 118\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.6408, + "worker": { + "memory.current": "67588096", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 143360\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 83976\nslab_unreclaimable 310864\nslab 394840\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 695477\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11840293\nuser_usec 11252399\nsystem_usec 587893\ncore_sched.force_idle_usec 0\nnr_periods 119\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 0.06314554199999911, + "ready_seconds": 5.1136705419999995, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.07206512499999995, + "ready_seconds": 6.358601541999999, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.07929308300000137, + "ready_seconds": 10.723366916000002, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 0.0655274170000002, + "ready_seconds": 1.6717298339999989, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.06864495800000014, + "ready_seconds": 7.294111999999998, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.09219408299999898, + "ready_seconds": 11.659252915999998, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 0.053762415999999646, + "ready_seconds": 4.160090082999998, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.05835524999999997, + "ready_seconds": 9.452590790999999, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 0.07616920800000138, + "ready_seconds": 8.538444833000002, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.06853324999999977, + "ready_seconds": 2.9269747909999992, + "status": "ready" + } + ], + "seconds": 11.840416583 + } + ] +} diff --git a/evidence/restart-after.json b/evidence/restart-after.json new file mode 100644 index 00000000..40aba5e4 --- /dev/null +++ b/evidence/restart-after.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475559008Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.552220082Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.074661473Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:01.993890605Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.087804668Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475268543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/restart-before.json b/evidence/restart-before.json new file mode 100644 index 00000000..37b6583d --- /dev/null +++ b/evidence/restart-before.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:46.437791821Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:03:32.37188016Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:03:41.154830662Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:03:24.364562897Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:23.350954361Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T14:56:14.79412711Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:03:23.874882274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.375415684Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.375672543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:05.95436716Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.374203439Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:23.351252982Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + } + ] + } +] diff --git a/evidence/runtime.json b/evidence/runtime.json new file mode 100644 index 00000000..016174fd --- /dev/null +++ b/evidence/runtime.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.608120136Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.67064591Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:08:06.296755178Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679298033Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679577406Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/security-stderr.txt b/evidence/security-stderr.txt new file mode 100644 index 00000000..7a251ff6 --- /dev/null +++ b/evidence/security-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab5-audit-run-790369e3d6e4 Creating + Container mtusi-lab5-audit-run-790369e3d6e4 Created diff --git a/evidence/security.json b/evidence/security.json new file mode 100644 index 00000000..748b6f7a --- /dev/null +++ b/evidence/security.json @@ -0,0 +1,15 @@ +{ + "anonymous_object_and_list_403": true, + "job_fields": [ + "attachment_id", + "attempt", + "created_at", + "job_id", + "pipeline_version", + "schema_version", + "type" + ], + "job_bytes": 243, + "no_source_bytes_or_credentials_in_envelope": true, + "strict_invalid_envelope_checks": true +} diff --git a/evidence/slow-upload.json b/evidence/slow-upload.json new file mode 100644 index 00000000..9e303a32 --- /dev/null +++ b/evidence/slow-upload.json @@ -0,0 +1 @@ +{"trickle_seconds": 14.998, "http_status": 400, "absolute_deadline_enforced": true} diff --git a/evidence/startup.txt b/evidence/startup.txt new file mode 100644 index 00000000..0213ac1a --- /dev/null +++ b/evidence/startup.txt @@ -0,0 +1,80 @@ + Container mtusi-lab5-broker-1 Running + Container mtusi-lab5-s3-store-1 Running + Container mtusi-lab5-postgres-1 Running + Container mtusi-lab5-redis-1 Running + Container mtusi-lab5-s3-1 Running + Container mtusi-lab5-bucket-bootstrap-1 Recreate + Container mtusi-lab5-migrate-1 Recreate + Container mtusi-lab5-bucket-bootstrap-1 Recreated + Container mtusi-lab5-migrate-1 Recreated + Container mtusi-lab5-worker-1 Recreate + Container mtusi-lab5-api-1 Recreate + Container mtusi-lab5-dispatcher-1 Recreate + Container mtusi-lab5-api-2 Recreate + Container mtusi-lab5-dispatcher-1 Recreated + Container mtusi-lab5-api-1 Recreated + Container mtusi-lab5-api-2 Recreated + Container mtusi-lab5-lb-1 Running + Container mtusi-lab5-worker-1 Recreated + Container mtusi-lab5-s3-store-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-s3-store-1 Healthy + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-migrate-1 Starting + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-migrate-1 Started + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-bucket-bootstrap-1 Starting + Container mtusi-lab5-bucket-bootstrap-1 Started + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-redis-1 Waiting + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-redis-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-worker-1 Starting + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-api-1 Started + Container mtusi-lab5-api-2 Starting + Container mtusi-lab5-api-2 Started + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-redis-1 Waiting + Container mtusi-lab5-dispatcher-1 Waiting + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-lb-1 Waiting + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-s3-store-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-worker-1 Waiting + Container mtusi-lab5-redis-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-s3-store-1 Healthy + Container mtusi-lab5-lb-1 Healthy + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Healthy + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-worker-1 Healthy + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy diff --git a/evidence/stopped-instances.jsonl b/evidence/stopped-instances.jsonl new file mode 100644 index 00000000..80b74b5b --- /dev/null +++ b/evidence/stopped-instances.jsonl @@ -0,0 +1 @@ +{"instance": "e4901b6a601a4dd094235d4dfcb0591c", "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", "running": false, "exit_code": 0, "seconds": 0.445} diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..2f51eee4 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.184s + +OK diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..5495e1e8 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,46 @@ +global + maxconn 1024 + nbthread 1 + hard-stop-after 10s + +defaults + mode http + timeout connect 500ms + timeout check 500ms + timeout queue 1s + timeout client 30s + timeout server 30s + timeout http-request 3s + retries 2 + retry-on conn-failure + option http-server-close + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 2 + server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +listen metrics + bind :8404 + stats enable + stats uri /stats diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..eeac3c14 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,42 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY + , username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$') + , display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100) + , password_hash TEXT NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; +CREATE TABLE chats ( + id UUID PRIMARY KEY + , title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , last_message_at TIMESTAMPTZ +) +; +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) + , user_id UUID NOT NULL REFERENCES users(id) + , PRIMARY KEY (chat_id, user_id) +) +; +CREATE INDEX chat_members_user ON chat_members (user_id, chat_id) +; +CREATE TABLE messages ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , sender_id UUID NOT NULL REFERENCES users(id) + , text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000) + , created_at TIMESTAMPTZ NOT NULL + , FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX messages_history ON messages (chat_id, created_at, id) +; +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY + , user_id UUID NOT NULL REFERENCES users(id) + , expires_at TIMESTAMPTZ NOT NULL +) +; +CREATE INDEX sessions_expiry ON sessions (expires_at) +; diff --git a/migrations/002_unicode_nul.sql b/migrations/002_unicode_nul.sql new file mode 100644 index 00000000..224cda0d --- /dev/null +++ b/migrations/002_unicode_nul.sql @@ -0,0 +1,17 @@ +-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA. +-- API validation keeps the contract's code-point limits; these bounds cap bytes. +ALTER TABLE users + DROP CONSTRAINT users_display_name_check + , ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8') + , ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +; +ALTER TABLE chats + DROP CONSTRAINT chats_title_check + , ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8') + , ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +; +ALTER TABLE messages + DROP CONSTRAINT messages_text_check + , ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8') + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..3ca53a7e --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) + , chat_id UUID NOT NULL REFERENCES chats(id) + , key VARCHAR(128) NOT NULL + , request_hash CHAR(64) NOT NULL + , message_id UUID REFERENCES messages(id) + , expires_at TIMESTAMPTZ NOT NULL + , PRIMARY KEY (user_id, chat_id, key) +) +; +CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at) +; diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..bfd51898 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,30 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_bytes + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000) + , ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}' +; +CREATE TABLE attachments ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , owner_id UUID NOT NULL REFERENCES users(id) + , filename TEXT NOT NULL + , object_key TEXT NOT NULL UNIQUE + , content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')) + , size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760) + , sha256 CHAR(64) NOT NULL + , state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing')) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX attachments_chat ON attachments (chat_id, id) +; +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , PRIMARY KEY (message_id, attachment_id) +) +; +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id) +; diff --git a/migrations/005_pipeline.sql b/migrations/005_pipeline.sql new file mode 100644 index 00000000..0ef1e81d --- /dev/null +++ b/migrations/005_pipeline.sql @@ -0,0 +1,40 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_state_check + , ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing')) + , ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 + , ADD COLUMN staging_key TEXT + , ADD COLUMN variants JSONB NOT NULL DEFAULT '[]' + , ADD COLUMN error_code TEXT + , ADD COLUMN current_job_id UUID +; +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , pipeline_version INTEGER NOT NULL + , source_key TEXT NOT NULL + , state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')) + , attempt INTEGER NOT NULL DEFAULT 0 + , max_attempts INTEGER NOT NULL DEFAULT 5 + , lease_token UUID + , lease_until TIMESTAMPTZ + , next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , published_at TIMESTAMPTZ + , error_code TEXT + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , UNIQUE (attachment_id, pipeline_version) +) +; +ALTER TABLE attachments + ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id) +; +CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at) +; +CREATE TABLE job_events ( + id BIGSERIAL PRIMARY KEY + , job_id UUID NOT NULL REFERENCES attachment_jobs(id) + , event TEXT NOT NULL + , attempt INTEGER NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..d97ccc99 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Optional, only for plotting the saved evidence, Python 3.11+. +matplotlib==3.11.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..30a631ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +typing-extensions==4.16.0 +boto3==1.40.61 +Pillow==13.0.0 +botocore==1.40.76 +jmespath==1.1.0 +s3transfer==0.14.0 +python-dateutil==2.9.0.post0 +urllib3==2.8.0 +six==1.17.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..4b118cbd --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(mode=0o700, exist_ok=True) + +def save(name, value): + path = directory / name + if not path.exists(): + path.write_text(value) + path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700. + return path.read_text().strip() + +password = save('postgres_password', secrets.token_hex(24)) +save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger') +save('cache_url', 'redis://redis:6379/0') +save('broker_url', 'redis://broker:6379/0') +identities = [] +for role, actions in { + 'admin': ['Admin', 'Read', 'Write', 'List'], + 'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], + 'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], +}.items(): + key = save('s3_' + role + '_key', secrets.token_hex(12)) + secret = save('s3_' + role + '_secret', secrets.token_hex(24)) + identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions}) +configuration = directory / 's3_config' +configuration.write_text(json.dumps({'identities': identities})) +configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration. +env = root / '.env' +if not env.exists(): + env.write_text((root / '.env.example').read_text()) + env.chmod(0o600) +print('Local configuration ready; no secrets printed.') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c8e00d70 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/snapshot.py > evidence/restart-before.json +docker compose stop api lb worker dispatcher +docker compose restart postgres redis s3-store s3 broker +docker compose up -d --wait postgres redis s3-store s3 broker +docker compose up -d --wait api lb worker dispatcher +python3 scripts/snapshot.py > evidence/restart-after.json diff --git a/scripts/snapshot.py b/scripts/snapshot.py new file mode 100644 index 00000000..3d0354f3 --- /dev/null +++ b/scripts/snapshot.py @@ -0,0 +1,11 @@ +"""Sanitized process and resource evidence; never dump container environments.""" +import json +import subprocess + +ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() +records = [] +for container in ids: + raw = subprocess.check_output(['docker', 'inspect', container], text=True) + data = json.loads(raw)[0] + records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]}) +print(json.dumps(records, indent=2)) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..ce9279f2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/stop_one.py diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..bc495aa5 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,32 @@ +"""Resolve the requested process alias using each actual container, never a fake header.""" +import json +import os +from pathlib import Path +import subprocess +import time + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def instance(container): + code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])" + return docker('exec', container, 'python', '-c', code) + + +if __name__ == '__main__': + target = os.environ['TARGET_INSTANCE_ID'] + started = time.monotonic() + for container in docker('compose', 'ps', '-q', 'api').splitlines(): + if instance(container) == target: + docker('stop', '--time', '8', container) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container)) + assert not state['Running'], 'target must remain stopped' + record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)} + with Path('evidence/stopped-instances.jsonl').open('a') as out: + out.write(json.dumps(record) + '\n') + print(json.dumps(record)) + break + else: + raise SystemExit('Requested live API instance not found') diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..e635caef --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,16 @@ +import os +import time +import urllib.request + +base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087') +for attempt in range(120): + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as response: + if response.status == 200: + print('Ready through load balancer.') + break + except OSError: + pass + time.sleep(0.5) +else: + raise SystemExit('Readiness timeout') diff --git a/src/admin.py b/src/admin.py new file mode 100644 index 00000000..699bd3e9 --- /dev/null +++ b/src/admin.py @@ -0,0 +1,127 @@ +"""Explicit local operator tools; no administrative HTTP endpoint.""" +import argparse +import json +import os +import uuid + +from pipeline import connect, audit +from objects import Objects +from postgres import setting + + +def replay(job_id): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone() + if job is None or job['state'] != 'failed': + raise ValueError('Replay requires a failed job') + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone() + if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None: + raise ValueError('Current failed source is unavailable') + conn.execute(''' + UPDATE attachment_jobs + SET state = 'queued' + , max_attempts = attempt + %s + , published_at = NULL + , next_attempt_at = clock_timestamp() + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id)) + conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,)) + audit(conn, job, 'operator_replay') + return {'job_id': str(job_id), 'replayed': True} + + +def version(attachment_id, number): + if number not in (1, 2): + raise ValueError('Only implemented pipeline versions 1 and 2 are supported') + with connect() as conn, conn.transaction(): + row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() + if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'): + raise ValueError('Version must advance the current process exactly once') + old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone() + source = row['object_key'] if row['state'] == 'ready' else old['source_key'] + if Objects(setting).head(source) is None: + raise ValueError('Source unavailable') + conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],)) + audit(conn, old, 'operator_supersede') + job_id = uuid.uuid4() + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts) + VALUES (%s, %s, %s, %s, %s) + ''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))) + conn.execute(''' + UPDATE attachments + SET pipeline_version = %s + , current_job_id = %s + , state = 'queued' + , variants = '[]' + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (number, job_id, attachment_id)) + return {'job_id': str(job_id), 'version': number} + + +def cleanup(apply=False, retention=86400): + if retention < 86400: + raise ValueError('Retention must be >=24 hours') + objects = Objects(setting) + counts = {'ready_staging': 0, 'expired_staging': 0} + with connect() as conn: + # Intent rows cover every valid staging PUT. Never collect active jobs. + last = '00000000-0000-0000-0000-000000000000' + while True: + rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall() + if not rows: + break + for candidate in rows: + last = candidate['id'] + with conn.transaction(): + row = conn.execute(''' + SELECT * + FROM attachments + WHERE 1=1 + AND id = %s + AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second')) + FOR UPDATE SKIP LOCKED + ''', (last, retention)).fetchone() + if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'): + continue + counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1 + if apply: + objects.delete(row['staging_key']) + if row['state'] == 'uploading' and row['current_job_id'] is None: + conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,)) + else: + conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,)) + return dict(apply=apply, retention_seconds=retention, counts=counts) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup']) + p.add_argument('identifier', nargs='?') + p.add_argument('--version', type=int) + p.add_argument('--apply', action='store_true') + args = p.parse_args() + try: + if args.action == 'failed': + with connect() as conn: + result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall() + elif args.action == 'replay': + result = replay(uuid.UUID(args.identifier)) + elif args.action == 'version': + result = version(uuid.UUID(args.identifier), args.version) + else: + result = cleanup(args.apply) + print(json.dumps(result, default=str, indent=2)) + except Exception: + print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'})) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/bucket_bootstrap.py b/src/bucket_bootstrap.py new file mode 100644 index 00000000..a0a2cbd3 --- /dev/null +++ b/src/bucket_bootstrap.py @@ -0,0 +1,13 @@ +"""Idempotent private bucket bootstrap, executed only with administrative S3 keys.""" +from botocore.exceptions import ClientError +from objects import Objects +from postgres import setting + +objects = Objects(setting) +try: + objects.client.create_bucket(Bucket=objects.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'): + raise RuntimeError('Bucket bootstrap failed') from None +objects.ready() +print('Private bucket ready; static authenticated identities loaded from local secret config.') diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..29c87c58 --- /dev/null +++ b/src/common.py @@ -0,0 +1,98 @@ +"""Shared HTTP contract; storage is supplied by each laboratory.""" +import base64 +import datetime as dt +import json +import re +import uuid + + +class APIError(Exception): + def __init__(self, status, code, message): + self.status, self.code, self.message = status, code, message + + +def invalid(): + raise APIError(400, 'invalid_request', 'Неверные параметры запроса') + + +def unauthorized(): + raise APIError(401, 'unauthorized', 'Требуется действительная идентификация') + + +def missing(): + raise APIError(404, 'not_found', 'Объект не найден') + + +def identifier(value): + try: + if not isinstance(value, str): + invalid() + return str(uuid.UUID(value)) + except ValueError: + invalid() + + +def text(value, maximum, minimum=1): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +def fields(value, required): + if not isinstance(value, dict) or set(value) != set(required): + invalid() + + +def username(value): + if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value): + invalid() + return value + + +def now(): + return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def page_args(query, scope): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + invalid() + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]+', limit): + invalid() + try: + limit = int(limit) + except ValueError: + invalid() + if not 1 <= limit <= 100: + invalid() + cursor = None + if 'cursor' in query: + raw = query['cursor'][0] + try: + if not raw or len(raw) > 1024: + invalid() + decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True)) + if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope: + invalid() + stamp = decoded[1] + if not isinstance(stamp, str) or not stamp.endswith('Z'): + invalid() + stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z') + cursor = (stamp, identifier(decoded[2])) + except (ValueError, TypeError, UnicodeError): + invalid() + return limit, cursor + + +def page(items, limit, scope): + more = len(items) > limit + items = items[:limit] + cursor = None + if more: + last = items[-1] + cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=') + return {'items': items, 'next_cursor': cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..ebd85f70 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,13 @@ +"""One-shot migration job; replicas never execute DDL.""" +from pathlib import Path +import psycopg +from postgres import setting + +with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2301003)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)') + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,)) + print('Applied', path.name) diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 00000000..f3a634cb --- /dev/null +++ b/src/objects.py @@ -0,0 +1,156 @@ +"""Bounded S3 operations. No root credentials or filename-derived object keys.""" +import contextlib +import hashlib +import os +import tempfile +import threading +import time +import unicodedata +import warnings + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, PngImagePlugin, UnidentifiedImageError + +from common import APIError, invalid + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 +PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) +UPLOAD_SLOTS = threading.BoundedSemaphore(8) +DECODE_SLOTS = threading.BoundedSemaphore(1) + + +def unavailable(): + raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно') + + +def filename(value): + if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip(): + invalid() + if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +@contextlib.contextmanager +def bounded_upload(): + if not UPLOAD_SLOTS.acquire(blocking=False): + raise APIError(503, 'unavailable', 'Все слоты загрузки заняты') + try: + yield + finally: + UPLOAD_SLOTS.release() + + +@contextlib.contextmanager +def receive(handler): + lengths = handler.headers.get_all('Content-Length', []) + if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Максимум 10 MiB') + if length <= 0: + invalid() + mime = handler.headers.get_content_type() + if mime not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG') + deadline = time.monotonic() + 15 + digest = hashlib.sha256() + # Ephemeral bounded spool is never a durable source of a confirmed attachment. + with tempfile.TemporaryFile(mode='w+b') as source: + remaining = length + while remaining: + handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic()))) + if time.monotonic() >= deadline: + invalid() + try: + chunk = handler.rfile.read1(min(64 * 1024, remaining)) + except (OSError, TimeoutError): + invalid() + if not chunk: + invalid() + digest.update(chunk) + source.write(chunk) + remaining -= len(chunk) + if not DECODE_SLOTS.acquire(timeout=10): + raise APIError(503, 'unavailable', 'Проверка изображений перегружена') + try: + source.flush() + for decode in (False,): + source.seek(0) + # Pillow.close releases pixel buffers and may close its input file. + # A duplicate descriptor preserves the original for the S3 PUT. + with os.fdopen(os.dup(source.fileno()), 'rb') as image_file: + image = Image.open(image_file, formats=['PNG', 'JPEG']) + try: + if image.width * image.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') + if Image.MIME[image.format] != mime: + raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению') + if decode: + image.load() + else: + image.verify() + finally: + image.close() # Free pixels BEFORE releasing the decoder semaphore. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None + finally: + DECODE_SLOTS.release() + source.seek(0) + yield source, length, mime, digest.hexdigest() + + +class Objects: + def __init__(self, setting): + self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required')) + + def head(self, key): + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'): + return None + unavailable() + except BotoCoreError: + unavailable() + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + except (ClientError, BotoCoreError): + unavailable() + + def put(self, key, source, length, mime, digest): + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest}) + except (ClientError, BotoCoreError): + unavailable() + + def get(self, key): + try: + return self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() + + def delete(self, key): + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() diff --git a/src/pipeline.py b/src/pipeline.py new file mode 100644 index 00000000..9086ec9c --- /dev/null +++ b/src/pipeline.py @@ -0,0 +1,332 @@ +"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits.""" +import argparse +import concurrent.futures +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import random +import re +import signal +import subprocess +import sys +import tempfile +import threading +import time +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +import redis +from botocore.exceptions import ClientError, BotoCoreError +from common import APIError +from objects import Objects, MAX_BYTES +from postgres import setting + +STREAM, GROUP = 'attachment-jobs', 'image-workers' +LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60')) +STOP = threading.Event() + + +def connect(): + return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500') + + +def broker(): + client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3) + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): + raise + return client + + +def event(job): + return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z')} + + +def validate(message): + required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message): + return False + try: + uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id']) + stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')) + if stamp.tzinfo is None or not message['created_at'].endswith('Z'): + return False + if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])): + return False + if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512): + return False + return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt']) + except (ValueError, TypeError, AttributeError): + return False + + +def audit(conn, job, name): + conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt'])) + + +def retry(conn, job, code, permanent=False): + failed = permanent or job['attempt'] >= job['max_attempts'] + state = 'failed' if failed else 'queued' + code = code if permanent or not failed else 'retry_exhausted' + delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5) + conn.execute(''' + UPDATE attachment_jobs + SET state = %s + , error_code = %s + , lease_token = NULL + , lease_until = NULL + , published_at = NULL + , next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (state, code, delay, job['id'])) + conn.execute(''' + UPDATE attachments + SET state = %s + , error_code = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + ''', (state, code if failed else None, job['id'])) + audit(conn, job, 'failed:' + code if failed else 'retry:' + code) + + +def dispatch_once(client, fault=None): + with connect() as conn: + with conn.transaction(): + expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall() + for job in expired: + retry(conn, job, 'lease_expired') + with conn.transaction(): + jobs = conn.execute(''' + SELECT j.* + FROM attachment_jobs j + JOIN attachments a ON a.current_job_id = j.id + WHERE 1=1 + AND j.state = 'queued' + AND j.next_attempt_at <= clock_timestamp() + AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second') + ORDER BY j.created_at + LIMIT 10 + FOR UPDATE OF j SKIP LOCKED + ''', (LEASE,)).fetchall() + for job in jobs: + client.xadd(STREAM, {'event': json.dumps(event(job), separators=(',', ':'))}) + if fault: + fault('after_publish', job) + conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],)) + audit(conn, job, 'published') + return len(jobs) + + +def claim(message): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone() + if job is None: + return None + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone() + if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']: + return None + if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']: + audit(conn, job, 'stale_noop') + return None + if job['state'] in ('ready', 'failed', 'superseded'): + audit(conn, job, 'terminal_noop') + return None + if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1: + audit(conn, job, 'duplicate_or_early_noop') + return None + token = uuid.uuid4() + job = conn.execute(''' + UPDATE attachment_jobs + SET state = 'processing' + , attempt = attempt + 1 + , lease_token = %s + , lease_until = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (token, LEASE, job['id'])).fetchone() + conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],)) + audit(conn, job, 'claimed') + return job, attachment + + +class Permanent(Exception): + pass + + +def immutable(objects, key, path, mime, sha): + head = objects.head(key) + if head is not None: + if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha: + raise Permanent('result_conflict') + return + try: + with path.open('rb') as source: + objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*') + except ClientError as error: + if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412): + head = objects.head(key) + if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha: + return + raise + + +def process(message, fault=None): + claimed = claim(message) + if claimed is None: + return 'noop' + job, attachment = claimed + objects = Objects(setting) + try: + if fault: + fault('before_side_effect', job) + with tempfile.TemporaryDirectory(prefix='job-') as temporary: + directory = Path(temporary) + source = directory / 'original' + head = objects.head(job['source_key']) + if head is None: + raise Permanent('source_missing') + response = objects.get(job['source_key']) + digest, size = hashlib.sha256(), 0 + deadline = time.monotonic() + 15 + with response['Body'] as stream, source.open('wb') as out: + for chunk in iter(lambda: stream.read(64 * 1024), b''): + size += len(chunk) + if size > MAX_BYTES or time.monotonic() > deadline: + raise Permanent('invalid_image') + digest.update(chunk); out.write(chunk) + if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']: + raise Permanent('checksum_mismatch') + result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20'))) + if result.returncode == 2: + raise Permanent('invalid_image') + if result.returncode: + raise RuntimeError('render_failed') + variants = json.loads(result.stdout) + prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/' + original_key = prefix + 'original' + immutable(objects, original_key, source, attachment['content_type'], attachment['sha256']) + if fault: + fault('after_original', job) + for variant in variants: + variant['object_key'] = prefix + variant['name'] + immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256']) + if fault: + fault('after_objects', job) + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + changed = conn.execute(''' + UPDATE attachments + SET state = 'ready' + , object_key = %s + , variants = %s + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + AND pipeline_version = %s + AND %s + RETURNING id + ''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone() + if changed: + conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],)) + audit(conn, job, 'ready') + else: + audit(conn, job, 'fenced_noop') + if fault: + fault('after_commit', job) + if changed and job['source_key'].startswith('staging/'): + objects.delete(job['source_key']) + return 'ready' if changed else 'fenced' + except Exception as error: + code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed' + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + if current['state'] == 'processing' and current['lease_token'] == job['lease_token']: + retry(conn, current, code, isinstance(error, Permanent)) + return 'failed_or_retry' + finally: + objects.client.close() + + +def ack(client, identifier): + # Atomic ack+delete; PostgreSQL is the durable completion/replay registry. + client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier) + + +def consume(once=False, fault=None): + client = broker() + consumer = uuid.uuid4().hex + while not STOP.is_set(): + messages = [] + try: + pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1] + if pending: + messages = pending + else: + batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500) + messages = batches[0][1] if batches else [] + for identifier, fields in messages: + raw = fields.get('event', '') + try: + message = json.loads(raw) if len(raw) <= 4096 else None + except ValueError: + message = None + if validate(message): + process(message, fault) + else: + client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False) + ack(client, identifier) + if once: + return + except (redis.RedisError, psycopg.Error): + STOP.wait(.5) + if once and not messages: + return + client.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['dispatch', 'work']) + parser.add_argument('--once', action='store_true') + parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish']) + args = parser.parse_args() + # Operator-only opt-in injection; no request header or public endpoint controls it. + def fault(point, job): + if point == args.fault_point: + os._exit(77) + signal.signal(signal.SIGTERM, lambda *_: STOP.set()) + signal.signal(signal.SIGINT, lambda *_: STOP.set()) + if args.mode == 'dispatch': + client = broker() + while not STOP.is_set(): + try: + dispatch_once(client, fault) + except (redis.RedisError, psycopg.Error): + pass + if args.once: + break + STOP.wait(.2) + else: + count = int(os.environ.get('WORKER_CONCURRENCY', '1')) + if not 1 <= count <= 2: + raise ValueError('WORKER_CONCURRENCY must be 1..2') + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + futures = [pool.submit(consume, args.once, fault) for _ in range(count)] + for result in futures: + result.result() + + +if __name__ == '__main__': + main() diff --git a/src/postgres.py b/src/postgres.py new file mode 100644 index 00000000..0ee57fa2 --- /dev/null +++ b/src/postgres.py @@ -0,0 +1,409 @@ +import base64 +import contextlib +import datetime as dt +import hashlib +import hmac +import json +import os +import re +from pathlib import Path +import secrets +import threading +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +import redis + +from objects import Objects, unavailable + +from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username + +KDF_SLOTS = threading.BoundedSemaphore(2) + + +def setting(name): + path = os.environ.get(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def public(row): + return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'} + + +def password_hash(password, salt=None): + salt = salt or secrets.token_bytes(16) + if not KDF_SLOTS.acquire(timeout=1.5): + raise APIError(503, 'unavailable', 'Аутентификация временно перегружена') + try: + digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024) + finally: + KDF_SLOTS.release() + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +class Store: + persistent = True + + def __init__(self): + self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600')) + if not 1 <= self.ttl <= 86400: + raise ValueError('SESSION_TTL_SECONDS must be 1..86400') + self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized') + if self.access_mode not in ('baseline', 'optimized'): + raise ValueError('DB_ACCESS_MODE must be baseline or optimized') + pool_size = int(os.environ.get('DB_POOL_MAX', '8')) + if not 1 <= pool_size <= 32: + raise ValueError('DB_POOL_MAX must be 1..32') + self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True) + self.pool.wait(timeout=10) + self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True) + self.objects = Objects(setting) + self.dummy_hash = password_hash('dummy-password-for-unknown-users') + with self.connection() as conn: + if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '005_pipeline.sql'").fetchone(): + raise RuntimeError('Run the separate migration service first') + + @contextlib.contextmanager + def connection(self): + try: + with self.pool.connection() as conn: + yield conn + except psycopg.errors.UniqueViolation: + raise APIError(409, 'conflict', 'Имя пользователя занято') from None + except (psycopg.Error, PoolTimeout): + raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None + + def close(self): + self.pool.close() + self.cache.close() + self.objects.client.close() + + def ready(self): + self.objects.ready() + with self.connection() as conn: + conn.execute('SELECT 1') + + def register(self, body): + fields(body, ('username', 'display_name', 'password')) + name, display = username(body['username']), text(body['display_name'], 100) + # Passwords may contain spaces and colons; only length/UTF-8 are restricted. + password = body['password'] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid() + try: + password.encode('utf-8') + except UnicodeError: + invalid() + hashed = password_hash(password) + with self.connection() as conn: + row = conn.execute(''' + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + ''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone() + return public(row) + + def login(self, headers): + authorization = headers.get('Authorization', '') + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic': + unauthorized() + name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError): + unauthorized() + if not re_username(name) or not 12 <= len(password) <= 128: + unauthorized() + with self.connection() as conn: + row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone() + hashed = row['password_hash'] if row else self.dummy_hash + verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed) + if row is None or not verified: + unauthorized() + token = secrets.token_urlsafe(32) + with self.connection() as conn: + expiry = conn.execute(''' + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second') + RETURNING expires_at + ''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at'] + return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)} + + def token_hash(self, headers): + parts = headers.get('Authorization', '').split(' ') + if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii(): + unauthorized() + return hashlib.sha256(parts[1].encode()).hexdigest() + + def authenticate(self, headers): + token_hash = self.token_hash(headers) + with self.connection() as conn: + session = conn.execute(''' + SELECT user_id + FROM sessions + WHERE 1=1 + AND token_hash = %s + AND expires_at > clock_timestamp() + ''', (token_hash,)).fetchone() + if session is None: + unauthorized() + key = 'user:' + str(session['user_id']) + try: + cached = self.cache.get(key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError): + pass + row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone() + if row is None: + unauthorized() + user = public(row) + try: + self.cache.setex(key, 60, json.dumps(user)) + except redis.RedisError: + pass + return user + + def logout(self, headers): + with self.connection() as conn: + conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),)) + + def chat_in(self, conn, actor, chat_id): + row = conn.execute(''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND c.id = %s + AND cm.user_id = %s + ''', (chat_id, actor['id'])).fetchone() + if row is None: + missing() + members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall() + return dict(public(row), member_ids=[str(v['user_id']) for v in members]) + + def chat(self, actor, chat_id): + with self.connection() as conn: + return self.chat_in(conn, actor, chat_id) + + def create_chat(self, actor, body): + fields(body, ('title', 'member_ids')) + title = text(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + invalid() + members = [identifier(v) for v in members] + if len(set(members)) != len(members) or actor['id'] not in members: + invalid() + with self.connection() as conn: + chat_id = uuid.uuid4() + conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8'))) + # Checking after INSERT exercises rollback of the whole composite operation. + found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall() + if len(found) != len(members): + invalid() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members]) + return self.chat_in(conn, actor, chat_id) + + def list_chats(self, actor, limit, cursor, scope): + with self.connection() as conn: + where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [actor['id']] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT c.id + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND cm.user_id = %s + ''' + where + ''' + ORDER BY c.created_at, c.id + LIMIT %s + ''', params).fetchall() + return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope) + + def require_member(self, conn, actor, chat_id): + if self.access_mode == 'baseline': + return self.chat_in(conn, actor, chat_id) + if not conn.execute(''' + SELECT 1 + FROM chat_members + WHERE 1=1 + AND chat_id = %s + AND user_id = %s + ''', (chat_id, actor['id'])).fetchone(): + missing() + + def send(self, actor, chat_id, body, key): + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body: + invalid() + attachments = body.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [identifier(v) for v in attachments] + if len(set(attachments)) != len(attachments): + invalid() + content = body['text'] + if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()): + invalid() + try: + content.encode('utf-8') + except UnicodeError: + invalid() + canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + # The unique key arbitrates concurrent processes; everything commits together. + claimed = conn.execute(''' + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at) + VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours') + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash + , message_id = NULL + , expires_at = EXCLUDED.expires_at + WHERE 1=1 + AND idempotency_keys.expires_at <= clock_timestamp() + RETURNING request_hash + ''', (actor['id'], chat_id, key, digest)).fetchone() + if claimed is None: + previous = conn.execute(''' + SELECT ik.request_hash + , m.id + , m.chat_id + , m.sender_id + , m.text + , m.created_at + , m.attachment_ids + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE 1=1 + AND ik.user_id = %s + AND ik.chat_id = %s + AND ik.key = %s + ''', (actor['id'], chat_id, key)).fetchone() + if previous['request_hash'] != digest: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + previous.pop('request_hash') + return public(previous) + if attachments: + rows = conn.execute(''' + SELECT id, state + FROM attachments + WHERE 1=1 + AND id = ANY(%s::uuid[]) + AND chat_id = %s + ORDER BY id + FOR SHARE + ''', (attachments, chat_id)).fetchall() + if len(rows) != len(attachments): + missing() + if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows): + raise APIError(409, 'conflict', 'Вложение ещё не готово') + # Per-chat row lock preserves committed history order across replicas/clocks. + stamp = conn.execute(''' + UPDATE chats + SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond') + WHERE 1=1 + AND id = %s + RETURNING last_message_at + ''', (chat_id,)).fetchone()['last_message_at'] + row = conn.execute(''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (%s, %s, %s, %s, %s, %s::uuid[]) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids + ''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone() + if attachments: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments]) + conn.execute(''' + UPDATE idempotency_keys + SET message_id = %s + WHERE 1=1 + AND user_id = %s + AND chat_id = %s + AND key = %s + ''', (row['id'], actor['id'], chat_id, key)) + return public(row) + + def history(self, actor, chat_id, limit, cursor, scope): + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [chat_id] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE 1=1 + AND chat_id = %s + ''' + where + ''' + ORDER BY created_at, id + LIMIT %s + ''', params).fetchall() + return page([public(v) for v in rows], limit, scope) + + def upload(self, actor, chat_id, name, source, length, mime, digest): + attachment_id, job_id = uuid.uuid4(), uuid.uuid4() + key = 'staging/' + str(attachment_id) + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + conn.execute('SELECT pg_advisory_xact_lock(2301005)') + pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n'] + if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')): + raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена') + conn.execute(''' + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading') + ''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest)) + with self.connection() as conn: + if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None: + unavailable() + self.objects.put(key, source, length, mime, digest) + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts) + VALUES (%s, %s, 1, %s, %s) + ''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))) + row = conn.execute(''' + UPDATE attachments + SET state = 'queued' + , current_job_id = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (job_id, attachment_id)).fetchone() + # Capture the queued snapshot, even if the worker finishes before HTTP reply. + return self.attachment_public(row) + + def attachment_public(self, row): + keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at') + result = public({k: row[k] for k in keys}) + variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']] + return dict(result, status=row['state'], variants=variants, error_code=row['error_code']) + + def attachment(self, actor, attachment_id): + with self.connection() as conn: + row = conn.execute(''' + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE 1=1 + AND a.id = %s + AND cm.user_id = %s + ''', (attachment_id, actor['id'])).fetchone() + if row is None: + missing() + if row['state'] == 'storage_missing': + unavailable() + if row['state'] == 'uploading': + raise APIError(409, 'conflict', 'Приём не завершён') + return row + + +def re_username(value): + import re + return re.fullmatch(r'[a-z0-9_]{3,32}', value) diff --git a/src/render.py b/src/render.py new file mode 100644 index 00000000..a9093347 --- /dev/null +++ b/src/render.py @@ -0,0 +1,64 @@ +"""Isolated image decoder/resizer; no network, database or credentials.""" +import hashlib +import json +import math +from pathlib import Path +import resource +import sys +import warnings + +resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) +resource.setrlimit(resource.RLIMIT_CPU, (15, 16)) +from PIL import Image, ImageOps, PngImagePlugin +Image.MAX_IMAGE_PIXELS = 40_000_000 +PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) + + +def render(source, directory, version): + output = [] + with Image.open(source, formats=['PNG', 'JPEG']) as image: + if image.width * image.height > 40_000_000: + raise ValueError('pixels') + image.load() + ImageOps.exif_transpose(image, in_place=True) + rgba = None + if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info): + rgba = image if image.mode == 'RGBA' else image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + alpha = rgba.getchannel('A') + rgb.paste(rgba, mask=alpha) + alpha.close() + else: + rgb = image if image.mode == 'RGB' else image.convert('RGB') + try: + for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]: + if name == 'crop': + result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5)) + else: + scale = min(1, bound / max(rgb.size)) + size = tuple(max(1, math.floor(v * scale)) for v in rgb.size) + result = rgb.resize(size, Image.Resampling.LANCZOS) + try: + path = directory / name + options = {'quality': quality, 'exif': b'', 'icc_profile': None} + if fmt == 'WEBP': + options['method'] = 4 + result.save(path, format=fmt, **options) + raw = path.read_bytes() + output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()}) + finally: + result.close() + finally: + if rgb is not image: + rgb.close() + if rgba is not None and rgba is not image: + rgba.close() + return output + + +if __name__ == '__main__': + try: + print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3])))) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning): + raise SystemExit(2) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..0b43d181 --- /dev/null +++ b/src/server.py @@ -0,0 +1,296 @@ +"""Bounded threaded API; persistent correctness state is shared across replicas.""" +import concurrent.futures +import http.server +import json +import logging +import os +import time +import signal +import socket +import threading +import uuid +from urllib.parse import parse_qs, urlsplit + +from objects import bounded_upload, receive, filename, unavailable + +from common import APIError, identifier, invalid, missing, page_args + +LOG = logging.getLogger('messenger') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + invalid() + result[key] = value + return result + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def setup(self): + super().setup() + self.connection.settimeout(3) + + def send_error(self, code, message=None, explain=None): + # BaseHTTPRequestHandler otherwise emits HTML without contract headers. + self.request_id = uuid.uuid4().hex + self.route = '' + self.close_connection = True + status = 413 if code == 413 else 400 + self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}}) + + def log_message(self, *args): + pass + + def reply(self, status, body): + if status == 200 and isinstance(body, tuple): + return self.download(*body) + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + if status >= 400: + self.close_connection = True + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + if self.close_connection: + self.send_header('Connection', 'close') + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + if status == 401 and self.server.store.persistent: + self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(raw) + + def download(self, row, response, chunk): + stream = response['Body'] + try: + self.send_response(200) + self.send_header('Content-Type', row['content_type']) + self.send_header('Content-Length', str(row['size_bytes'])) + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cache-Control', 'private, no-store') + self.end_headers() + deadline = time.monotonic() + 15 + while chunk: + self.wfile.write(chunk) + if time.monotonic() > deadline: + raise TimeoutError() + chunk = stream.read(64 * 1024) + finally: + stream.close() + + def body(self): + lengths = self.headers.get_all('Content-Length', []) + if len(lengths) != 1 or self.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > 32768: + raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт') + if length <= 0: + invalid() + if self.headers.get_content_type() != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Требуется application/json') + try: + raw = self.rfile.read(length) + if len(raw) != length: + invalid() + return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid()) + except (ValueError, UnicodeError, RecursionError): + invalid() + + def dispatch(self): + self.request_id = uuid.uuid4().hex + self.route = urlsplit(self.path).path + status = 500 + try: + status, result = self.route_request() + except APIError as exc: + status = exc.status + result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}} + except Exception: + # No bodies, credentials, tokens, SQL or exception details in logs/responses. + LOG.error('request_failed request_id=%s', self.request_id) + result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}} + try: + self.reply(status, result) + except Exception: + self.close_connection = True + LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id) + + def route_request(self): + method, path, store = self.command, self.route, self.server.store + if method == 'GET' and path in ('/health/live', '/health/ready'): + if path.endswith('ready'): + store.ready() + return 200, {'status': 'ok'} + if method == 'POST' and path == '/api/v1/users': + return 201, store.register(self.body()) + if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + invalid() + return 201, store.login(self.headers) + actor = store.authenticate(self.headers) + if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.logout(self.headers) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, actor + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + if path == '/api/v1/chats': + if method == 'POST': + return 201, store.create_chat(actor, self.body()) + if method == 'GET': + scope = 'chats:' + actor['id'] + limit, cursor = page_args(query, scope) + return 200, store.list_chats(actor, limit, cursor, scope) + parts = path.split('/') + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET': + row = store.attachment(actor, identifier(parts[4])) + if len(parts) == 5: + if query: + invalid() + return 200, store.attachment_public(row) + if parts[5] == 'content': + if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()): + invalid() + if row['state'] != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + variant = query.get('variant', ['original'])[0] + if variant != 'original': + item = next((v for v in row['variants'] if v['name'] == variant), None) + if item is None: + missing() + row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes']) + response = store.objects.get(row['object_key']) + try: + chunk = response['Body'].read(64 * 1024) + if response['ContentLength'] != row['size_bytes'] or not chunk: + unavailable() + except Exception: + response['Body'].close() + unavailable() + return 200, (row, response, chunk) + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']: + chat_id = identifier(parts[4]) + if store.access_mode == 'baseline': + store.chat(actor, chat_id) + if len(parts) == 5 and method == 'GET': + return 200, store.chat(actor, chat_id) + if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST': + store.chat(actor, chat_id) + if set(query) != {'filename'} or len(query['filename']) != 1: + invalid() + name = filename(query['filename'][0]) + with bounded_upload(): + with receive(self) as (source, length, mime, digest): + return 202, store.upload(actor, chat_id, name, source, length, mime, digest) + if len(parts) == 6 and parts[5] == 'messages': + if method == 'POST': + keys = self.headers.get_all('Idempotency-Key', []) + return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None) + if method == 'GET': + scope = 'messages:' + chat_id + limit, cursor = page_args(query, scope) + return 200, store.history(actor, chat_id, limit, cursor, scope) + missing() + + do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch + + +class Server(http.server.HTTPServer): + request_queue_size = 64 + allow_reuse_address = True + + def __init__(self, address, store): + self.store = store + self.instance_id = uuid.uuid4().hex + workers = int(os.environ.get('HTTP_WORKERS', '16')) + if not 1 <= workers <= 128: + raise ValueError('HTTP_WORKERS must be 1..128') + self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) + self.slots = threading.BoundedSemaphore(workers * 2) + self.connections = set() + self.connections_lock = threading.Lock() + super().__init__(address, Handler) + + def process_request(self, request, address): + if not self.slots.acquire(blocking=False): + rid = uuid.uuid4().hex + raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode() + try: + request.settimeout(0.2) + request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw) + except OSError: + pass + self.shutdown_request(request) + return + with self.connections_lock: + self.connections.add(request) + self.executor.submit(self.worker, request, address) + + def worker(self, request, address): + try: + self.finish_request(request, address) + except OSError: + pass + finally: + self.shutdown_request(request) + with self.connections_lock: + self.connections.discard(request) + self.slots.release() + + def close_connections(self): + with self.connections_lock: + connections = list(self.connections) + for connection in connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def server_close(self): + super().server_close() + # Bound draining even when a slow client keeps an incomplete request alive. + deadline = threading.Timer(5, self.close_connections) + deadline.start() + try: + self.executor.shutdown(wait=True) + finally: + deadline.cancel() + + +def main(): + logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s') + from postgres import Store + store = Store() + server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store) + stopping = threading.Event() + + def stop(signum, frame): + if not stopping.is_set(): + stopping.set() + LOG.info('shutdown_started signal=%s', signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats()))) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + LOG.info('started port=%s', server.server_address[1]) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + store.close() + LOG.info('shutdown_complete') + + +if __name__ == '__main__': + main() diff --git a/student-tests/durability.py b/student-tests/durability.py new file mode 100644 index 00000000..3470de0d --- /dev/null +++ b/student-tests/durability.py @@ -0,0 +1,52 @@ +"""Stop worker, accept queued, restart API/broker, then complete the same attachment.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client,Failure + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap();record={} + before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + try: + docker('compose','stop','worker','dispatcher') + queued,raw=c.attachment(alice,chat) + assert queued['status']=='queued' + message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json'] + c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob) + killed=docker('compose','ps','-q','api').splitlines() + for container in killed:docker('kill','--signal=KILL',container) + record['killed_api_containers']=killed + docker('compose','up','-d','--wait','--no-deps','api') + time.sleep(2) + assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued' + docker('compose','up','-d','--no-deps','dispatcher') + deadline=time.monotonic()+10 + while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1: + assert time.monotonic()=length_before>=1 + record['stream_entries_before_after_broker_restart']=[length_before,length_after] + docker('compose','up','-d','--no-deps','worker','dispatcher') + ready=c.wait_ready(bob,queued) + assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw + record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']]) + finally:docker('compose','up','-d','--no-deps','worker','dispatcher') + after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + record['before']=before;record['after']=after + print(json.dumps(record,indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 00000000..906356fa --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,24 @@ +"""Small deterministic PNG helpers; generation happens outside measurement.""" +import random +import struct +import zlib + + +def chunk(name, data): + return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff) + + +def png(width, height, noise=False, rgb=True): + channels = 3 if rgb else 1 + rand = random.Random(2301) + encoded = zlib.compressobj(1) + parts = [] + for _ in range(height): + line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels) + parts.append(encoded.compress(b'\0' + line)) + parts.append(encoded.flush()) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(): + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'') diff --git a/student-tests/pipeline_checks.py b/student-tests/pipeline_checks.py new file mode 100644 index 00000000..c4d36ba1 --- /dev/null +++ b/student-tests/pipeline_checks.py @@ -0,0 +1,188 @@ +"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first.""" +import concurrent.futures +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from PIL import Image +from client import Client +from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack +from objects import Objects +from postgres import setting +from admin import replay,version,cleanup + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap() + bus=broker(); objects=Objects(setting) + report={'invariants':{},'faults':[]} + def fetch(aid): + with connect() as conn: + a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone() + j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone() + return a,j + def upload(raw=None,mime='image/png'): + if raw is None: a,_=c.attachment(alice,chat) + else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json'] + return a,fetch(a['id'])[1] + def run_once(fault=None): + args=[sys.executable,'/app/src/pipeline.py','work','--once'] + if fault:args+=['--fault-point',fault] + return subprocess.run(args,capture_output=True,timeout=30).returncode + def drain(): + for _ in range(200): + batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1) + if not batch:break + for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid) + def expire(job): + with connect() as conn: + conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],)) + def recover(aid): + until=time.monotonic()+20 + while time.monotonic() no publisher yet: one queued row, no Redis message, then automatic outbox pass. + a,j=upload() + assert fetch(a['id'])[0]['state']=='queued' + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob) + c.message(bob,chat,'queued is linkable',attachments=[a['id']]) + dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['commit_before_publish_recovered']=True + # Broker accepted XADD but publisher died before marking outbox: true exit(77). + a,j=upload() + code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode + assert code==77 + dispatch_once(bus);drain() + assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['publish_before_marker_duplicate_safe']=True + for point in ['before_side_effect','after_original','after_objects','after_commit']: + a,j=upload();dispatch_once(bus) + code=run_once(point);assert code==77,(point,code) + before,job=fetch(a['id']) + if point not in ('after_commit','before_side_effect'):expire(job) + result,job=recover(a['id']) + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes() + reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1] + for mid,data in reclaimed: + process(json.loads(data['event']));ack(bus,mid) + assert bus.xpending(STREAM,GROUP)['pending']==0 + keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[]) + assert len(keys)==4,(point,len(keys)) + report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)}) + report['invariants']['crashes_before_after_side_effect_and_commit']=True + bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})}) + assert run_once()==0 + assert bus.xlen('rejected-jobs')>=1 + report['invariants']['invalid_envelope_observable_dlq']=True + # Duplicate/reorder through broker on a completed job: no extra attempt/effect. + a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id']) + old=event(j) + for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))}) + drain();after,done2=fetch(a['id']) + assert before['variants']==after['variants'] and done2['attempt']==done['attempt'] + report['invariants']['duplicate_and_reorder_no_second_effect']=True + # Race two consumers with the SAME logical job, not just one serial duplicate. + a,j=upload();e=event(j) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2))) + assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready'] + report['invariants']['concurrent_duplicate_single_claim']=True + # Pause actual old process after all immutable side effects, advance version and finish v2. + a,j=upload();e=event(j) + directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release' + code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))" + child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + try: + deadline=time.monotonic()+20 + while not marker.exists(): + assert child.poll() is None + assert time.monotonic()=250 + transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())}) + report['transforms']=transforms + pending,pending_job=upload() + dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True) + assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None + report['invariants']['cleanup_preserves_current_and_fresh_failed']=True + assert repeat['counts']['ready_staging']==0 + report['invariants']['ready_staging_cleanup_idempotent']=True + report['cleanup']=[dry,fixed,repeat] + print(json.dumps(report,indent=2)) + + +if __name__=='__main__': main() diff --git a/student-tests/plot.py b/student-tests/plot.py new file mode 100644 index 00000000..157828de --- /dev/null +++ b/student-tests/plot.py @@ -0,0 +1,28 @@ +"""Figures from real accepted/ready and cgroup samples, no synthetic results.""" +import json +from pathlib import Path +import statistics +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +r=Path(__file__).resolve().parents[1]/'evidence' +x=json.loads((r/'resources.json').read_text())['experiments'] +fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained') +labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB'] +a=[max(v['accepted_seconds'] for v in e['replies']) for e in x] +b=[max(v['ready_seconds'] for v in e['replies']) for e in x] +axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202') +axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready') +axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1') +axes[0].legend() +peak=[int(e['after']['memory.peak'])/1048576 for e in x] +anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x] +axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak') +axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon') +axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker') +axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер') +axes[1].legend(fontsize=8) +for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True) +fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1') +fig.savefig(r/'pipeline.png',dpi=150) +fig.savefig(r/'pipeline.svg') diff --git a/student-tests/resources.py b/student-tests/resources.py new file mode 100644 index 00000000..a9c409d3 --- /dev/null +++ b/student-tests/resources.py @@ -0,0 +1,52 @@ +"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS.""" +import concurrent.futures +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client +from images import png + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def memory(container): + code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))" + return json.loads(docker('exec',container,'python','-c',code)) + + +def main(): + c=Client(timeout=30);a,b,e,chat=c.bootstrap() + profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)] + result=[] + for w,h,noise in profiles: + content=png(w,h,noise=noise) + docker('compose','restart','worker');container=docker('compose','ps','-q','worker') + before=memory(container);samples=[];done=threading.Event();start=time.monotonic() + def sample(): + while not done.is_set(): + samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)}) + done.wait(.05) + sampler=threading.Thread(target=sample);sampler.start() + barrier=threading.Barrier(10) + def upload(i): + barrier.wait();t=time.monotonic() + r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') + accepted=time.monotonic()-t + ready=Client(timeout=30).wait_ready(b,r['json']) + assert ready['sha256']==hashlib.sha256(content).hexdigest() + return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']} + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10))) + finally:done.set();sampler.join() + after=memory(container);assert 'oom_kill 0' in after['memory.events'] + result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start}) + print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/security.py b/student-tests/security.py new file mode 100644 index 00000000..9538f7ea --- /dev/null +++ b/student-tests/security.py @@ -0,0 +1,27 @@ +"""Private S3 and job envelopes do not expose source bytes or credentials.""" +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from client import Client +from pipeline import connect,validate,event +from objects import Objects +from postgres import setting + +c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att) +with connect() as conn: + row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone() + job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone() +message=event(job);message['attempt']=max(1,job['attempt']) +assert validate(message) +assert set(message)=={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} +assert len(json.dumps(message))<1024 +for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'): + try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access') + except urllib.error.HTTPError as error:assert error.code==403 +for key in ['schema_version','pipeline_version','attempt']: + bad=dict(message);bad[key]=True;assert not validate(bad) +for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body) +print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2)) diff --git a/student-tests/slow_upload.py b/student-tests/slow_upload.py new file mode 100644 index 00000000..c17d2b01 --- /dev/null +++ b/student-tests/slow_upload.py @@ -0,0 +1,32 @@ +"""A continuously trickling client still reaches the absolute upload deadline.""" +import json +from pathlib import Path +import select +import socket +import sys +import time +import uuid +from urllib.parse import urlsplit +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +c = Client() +alice, bob, eve, chat = c.bootstrap() +target = urlsplit(c.base) +connection = socket.create_connection((target.hostname, target.port), timeout=3) +header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' +started = time.monotonic() +connection.sendall(header.encode()) +response = b'' +while time.monotonic() - started < 20: + if select.select([connection], [], [], .15)[0]: + response += connection.recv(4096) + if b'\r\n\r\n' in response: + break + connection.sendall(b'x') +connection.close() +elapsed = time.monotonic() - started +assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100] +assert 14 <= elapsed < 19 +assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower() +print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True})) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..45f0d800 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,177 @@ +"""Extra black-box tests, without changing the course's public tests.""" +import concurrent.futures +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client as PublicClient, LAB + + +class Client(PublicClient): + def call(self, method, path, *args, headers=None, **kwargs): + headers = dict(headers or {}) + if method == 'POST' and path.endswith('/messages'): + headers.setdefault('Idempotency-Key', uuid.uuid4().hex) + return super().call(method, path, *args, headers=headers, **kwargs) + + +class APIContract(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_malformed_json_and_extra_fields(self): + for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body) + deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}' + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'}) + + def test_unicode_limits(self): + result = self.client.message(self.alice, self.chat, '👩' * 4000)['json'] + self.assertEqual(result['text'], '👩' * 4000) + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001}) + + def test_unicode_nul_round_trip(self): + import base64 + value = 'A\x00B\\u0000👩' + registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value) + user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json'] + self.assertEqual(user['display_name'], value) + credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(session['user'], user) + actor = {'id': user['id'], 'token': session['token']} + # Exercise the database path and then the Redis profile cache. + for _ in range(2): + self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user) + chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json'] + self.assertEqual(chat['title'], value) + self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat) + contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00'] + for content in contents: + self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content) + history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json'] + self.assertEqual([message['text'] for message in history['items']], contents) + + def test_foreign_chat(self): + for suffix in ('', '/messages'): + self.client.call('GET', self.path + suffix, 404, actor=self.eve) + self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'}) + self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice) + + def test_registration_race(self): + body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20]) + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + statuses = list(executor.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + def test_invalid_chat_creation(self): + members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True]) + for member_ids in members: + self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids}) + + def test_cursor_scope_and_validation(self): + for i in range(3): + self.client.message(self.alice, self.chat, str(i)) + cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice) + for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'): + self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice) + + def test_concurrent_messages_and_pagination(self): + def send(i): + return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + sent = list(executor.map(send, range(100))) + found = [] + cursor = None + while True: + url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '') + page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json'] + found.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(len(found), 100) + self.assertEqual(len({v['id'] for v in found}), 100) + self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent}) + self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id']))) + + def test_idempotency_concurrent_replicas(self): + key = uuid.uuid4().hex + def send(_): + return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2) + self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies)) + page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(page['items'], [replies[0]['json']]) + self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice) + self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key}) + + def test_protocol_error_headers(self): + self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice) + + def test_idempotency_scope(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'same', key=key)['json'] + two = self.client.message(self.bob, self.chat, 'same', key=key)['json'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + three = self.client.message(self.alice, other, 'same', key=key)['json'] + self.assertEqual(len({one['id'], two['id'], three['id']}), 3) + + def test_idempotency_semantic_body(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json'] + two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + self.assertEqual(one, two) + + def test_idempotency_invalid_keys(self): + for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'): + headers = {} if key is None else {'Idempotency-Key': key} + PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers) + for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + + def test_idempotency_conflicting_race(self): + key = uuid.uuid4().hex + def send(i): + return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertEqual(sum(v['status'] == 201 for v in replies), 16) + self.assertEqual(sum(v['status'] == 409 for v in replies), 16) + page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(len(page['items']), 1) + + @unittest.skipUnless(LAB >= 2, 'sessions start in lab2') + def test_unicode_password_basic_and_logout(self): + import base64 + registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'} + self.client.call('POST', '/api/v1/users', 201, body=registration) + encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + headers = {'Authorization': 'Basic ' + encoded} + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json'] + actor = {'id': session['user']['id'], 'token': session['token']} + self.client.call('GET', '/api/v1/users/me', actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, headers=headers) + for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'): + self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 00000000..f27c0393 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,52 @@ +"""Independent tests for async image metadata, transformations and access.""" +import io +from pathlib import Path +import sys +import unittest +import uuid +from urllib.parse import quote +sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')] +from client import Client +from images import bomb + + +class Images(unittest.TestCase): + def setUp(self): + self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap() + self.path='/api/v1/chats/'+self.chat['id'] + self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + + def test_async_variants_and_replay(self): + queued,_=self.c.attachment(self.alice,self.chat) + self.assertEqual(queued['status'],'queued') + key=uuid.uuid4().hex + one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'] + self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one) + ready=self.c.wait_ready(self.alice,queued) + self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'}) + dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']} + self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}) + for v in ready['variants']: + import hashlib + r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob) + self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256']) + self.assertEqual(len(r['bytes']),v['size_bytes']) + self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob) + + def test_mime_bytes_pixels_and_filename(self): + path=self.path+'/attachments?filename=test.png' + for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]: + self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime}) + for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''): + self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'}) + + def test_private_and_cross_chat(self): + att,_=self.c.attachment(self.alice,self.chat) + for suffix in ('','/content'): + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve) + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401) + other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json'] + self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + + +if __name__=='__main__': unittest.main(verbosity=2) -- 2.54.0