diff --git a/.dockerignore b/.dockerignore index bc16e556..8eb0c085 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store + +build diff --git a/.env.example b/.env.example index 119384b5..bc195d6d 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,14 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +APP_PORT=18091 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +API_CPUS=1 +API_MEMORY=256m +HTTP_WORKERS=16 +DB_POOL_MAX=8 +DB_ACCESS_MODE=optimized +POSTGRES_CPUS=1 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_TIMEOUT_SECONDS=20 +JOB_LEASE_SECONDS=60 +MAX_PENDING_JOBS=100 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..1220e717 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,26 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + BASE_URL: http://127.0.0.1:18091 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: | + python scripts/bootstrap.py + docker compose build api + docker compose up -d --wait + python scripts/wait_ready.py + make check test test-student BASE_URL=$BASE_URL + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL + docker compose up -d --wait --no-deps api + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL + - if: always() + run: docker compose down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5e287bd9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2 +WORKDIR /app +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY src/ ./src/ +COPY migrations/ ./migrations/ +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] +STOPSIGNAL SIGTERM +CMD ["python", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..9bda7e0c 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,8 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-student + +test-student: + BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v diff --git a/REPORT.md b/REPORT.md index e98f75e9..f217f83c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,210 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5 — очереди и асинхронные изображения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Майорова Софья, БВТ2301. Целевая оценка: 5. Трек 2 — доказанная идемпотентность versioned pipeline.** Выполнено 27 октября 2026 года. Оценку подтверждает преподаватель. + +Продолжено [моё решение lab4, PR 18](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/18). Сохранены две API за LB, сессии PostgreSQL, Redis-кэш, keyset-пагинация, идемпотентность сообщений и приватные S3-вложения. Upload изменён по контракту: 202 queued после устойчивого staging и сохранения задания; resize выполняется отдельным worker. Исходные README/COURSE/contracts/tests не менялись. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine/Compose v2 и Python 3.9+ для публичных тестов на хосте. Сервер/worker — Python 3.12.8 внутри Docker. На этом Mac: -## Архитектура и решения +```sh +colima start --profile mtusi-labs +export DOCKER_CONTEXT=colima-mtusi-labs +cd "/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab5" +python3 scripts/bootstrap.py +docker compose build api +docker compose up -d --wait +export BASE_URL=http://127.0.0.1:18091 +python3 scripts/wait_ready.py +make check test test-student BASE_URL=$BASE_URL +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +На другой машине клонировать публичный форк lab5 и выбрать `solution/BVT2301-Mayorova`, затем выполнить те же команды из корня clone. `build api` идёт первым: migrate, worker, dispatcher и tools используют тот же построенный образ. Одноразовый migrate под advisory lock применяет 001–005; API DDL не выполняют. Bucket-bootstrap идемпотентно создаёт приватный bucket. После них поднимаются API, dispatcher и worker. -## Доказательства критериев +Bootstrap создаёт случайные локальные пароли и ключи, сохраняет существующие значения и `.env`, пересобирает только генерируемую конфигурацию identities из этих ключей. Каталог secrets — 0700; полный s3_config — **0600**, монтируется только root S3 gateway. Отдельные ключи процесса читаются UID 10001 внутри контейнера; их файлы находятся в закрытом 0700-каталоге хоста. API не получает worker/admin credentials. `.env`, secrets, build и данные исключены из Git и build context. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Остановка с сохранением данных: `docker compose down`. Только для намеренного удаления ненужных **учебных данных данного проекта** — `docker compose down -v`. В restart/failover volumes не удаляются. Изолированный clean-clone стенд использует другое имя проекта, порт и временные volumes, которые удаляются после его проверки. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура и границы подтверждения -## Эксперименты и отказы +```text +Клиент → LB → API × 2 → PostgreSQL (сессии, чаты, сообщения, attachments + jobs/outbox) + ├→ Redis cache профилей (восстанавливаемый) + └→ S3 staging (полные исходные bytes) + ↑ +PostgreSQL jobs/outbox → dispatcher → Redis Streams broker (AOF always) + ↓ XREADGROUP / XAUTOCLAIM + worker → изолированный render subprocess + ├→ конечный S3 results//vN/... + └→ атомарный ready + варианты в PostgreSQL → XACK + XDEL +``` -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +S3 — SeaweedFS 4.51: отдельный gateway и mini master/volume/filer на сети storage. И байты, и filer metadata устойчиво находятся в `s3_data`; API не имеет доступа к filer, административный UI/WebDAV отключены. PostgreSQL — другой volume. Брокер — **отдельный Redis 7.4.1**, не кэш: AOF, `appendfsync always`, `noeviction`, persistent `broker_data`, без публикации порта на хост. Кэш остаётся disposable Redis без AOF. Единственная публичная точка — LB на **127.0.0.1:18091**. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Образы Python 3.12.8, PostgreSQL 17.2-bookworm, Redis 7.4.1-bookworm, HAProxy 3.0.5-alpine и SeaweedFS 4.51 закреплены digest в Dockerfile/Compose. Например, SeaweedFS: `sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8`. Pillow 13.0.0, boto3 1.40.61, psycopg/redis и транзитивные зависимости закреплены в requirements.txt. Это проверенный набор версий, не обещание отсутствия будущих уязвимостей. -## Ограничения и следующие шаги +| Процесс | CPU | Память | Состояние | +| --- | ---: | ---: | --- | +| API, каждая из двух | 1 | 256 MiB | Без устойчивых локальных originals | +| Worker, concurrency=1 | 1 | 768 MiB | Временный tmpfs; источник/результат в S3 | +| Dispatcher | 1 | 256 MiB | Устойчивый outbox в PostgreSQL | +| PostgreSQL | 1 | 512 MiB | postgres_data | +| Broker | 0,5 | 192 MiB, maxmemory=96mb/noeviction | broker_data/AOF | +| S3 store / gateway | 1 / 0,5 | 512 / 512 MiB | s3_data / без собственных originals | +| LB / cache | 0,5 / 0,25 | 64 / 64 MiB | Нет обязательных устойчивых данных | -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сумма CPU-лимитов превышает 4 vCPU локальной VM: это overcommit, не выделенные физические ядра. Измерения проводились на macOS arm64, Colima/Linux с 4 vCPU и примерно 4 GiB RAM. Ранее запущенные учебные стенды не останавливались. + +### Что сохранено при 202 + +API проверяет Bearer/доступ к чату, Content-Length ≤10 MiB, допустимый filename/MIME, реальный PNG/JPEG header, размеры ≤40 млн pixels и `verify()`. Исходник читается кусками 64 KiB в ограниченный временный файл, SHA-256 считается потоково. Полного декодирования/resize в API нет. Поэтому JPEG с корректным заголовком, но повреждённым scan data может быть принят и затем завершиться **failed invalid_image** в worker; заведомо чужой формат/несовпадение MIME отклоняется 415, лимиты — 413, опасное имя — 400. + +Перед PUT фиксируется internal intent uploading со случайным UUID, server-generated staging key и ожидаемой checksum. Под row lock выполняется полный PUT в приватный `staging/`. В одной следующей транзакционной границе с этим lock создаётся queued job/outbox и меняется Attachment на queued с current_job_id. **COMMIT выполнен до 202.** API возвращает сохранённый queued snapshot даже если worker успел закончить раньше HTTP-ответа. Bytes в PostgreSQL и broker не помещаются. Временный файл API не является долговременным staging. + +Уже принятая работа не зависит от RAM API или доступности dispatcher/broker в этот момент. Readiness API проверяет PostgreSQL и S3; остановленный worker/broker не запускает resize в HTTP. При `MAX_PENDING_JOBS=100` новые uploading/queued/processing ограничены общим счётчиком PostgreSQL под advisory lock; заполнение даёт 503. Это предел количества, не расчёт disk quota. Неподтверждённый upload не следует прозрачно повторять: контракт идемпотентности относится к POST messages. + +## Transactional outbox и at-least-once + +Таблица `attachment_jobs` совмещает job registry и transactional outbox. Создание job и queued Attachment атомарно. Dispatcher выбирает due rows `FOR UPDATE SKIP LOCKED`, выполняет XADD с envelope по jobs.schema.json, затем отмечает published_at. При сбое до XADD строка остаётся; после XADD до фиксации marker возможен дубль. XADD response с AOF always подтверждает приём/сохранение брокером согласно его политике. Это ещё не подтверждение обработки изображения. + +Worker читает consumer group `image-workers`, **count/prefetch=1 на slot**. XAUTOCLAIM возвращает зависшую pending delivery после lease. Dispatcher также восстанавливает processing с истёкшей lease и повторяет долго не захваченную queued publication. DB-инварианты, а не предположение о единственности сообщения брокера, определяют допустимость обработки. + +Broker message содержит только schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at; допускается контрактный trace carrier. Пример фактически проверенного envelope — 243 bytes. Ключи хранилища и credentials берутся worker из доверенных настроек/БД, произвольные URL/filename-пути из сообщения не исполняются. Неверный envelope наблюдаем в ограниченном `rejected-jobs` stream: только причина, исходный message ID и hash, без сохранения произвольного payload. Подтверждение ошибочного envelope происходит после записи этой DLQ-записи. + +XACK + XDEL выполняются одной Redis Lua-операцией **после** устойчивого ready/failed либо безопасного no-op. При потере ack доставка повторяется. Ошибка БД, не позволившая сохранить результат/ошибку, не подтверждает delivery. Доставка остаётся at-least-once; сокращение числа логических эффектов не превращает её в exactly-once. + +## Worker, варианты, retry и lease + +Worker — отдельный контейнер/процесс. Default concurrency=1, конфигурация допускает 1–2; при увеличении нужно отдельно увеличить/измерить WORKER_MEMORY, в отчёте проверен только 1. Производственная lease — 60 s; дочерний renderer ограничен wall timeout 20 s, CPU 15 s (hard 16), virtual address space 512 MiB. Worker контейнер ограничен 768 MiB, tmpfs 128 MiB. Исходник ≤10 MiB скачивается кусками с SHA-256/size и deadline 15 s; S3 connect/read timeouts — 1/3 s, SDK total attempts=1. Самостоятельные retry контролирует job registry. + +Pillow полностью декодирует исходник в **изолированном subprocess**, проверяет pixels до load, применяет EXIF orientation, alpha композитит на белом. Размеры рассчитываются с округлением вниз, min 1 px. Для thumbnail/optimized маленький исходник не увеличивается. Варианты сохраняются без EXIF/ICC: + +- thumbnail: JPEG quality=85, вписан в 256×256; fixture 320×200 → 256×160; +- crop: центрированный квадрат 128×128, Lanczos; маленький исходник допускает увеличение; +- optimized: WebP, вписан в 1280×1280, method=4, quality=82 для pipeline v1, quality=70 для v2. Уменьшение каждого маленького файла не обещается. + +Оригинал публикуется побайтно в конечный S3-prefix отдельно от staging. Ready фиксируется только после original + **всех трёх** вариантов с checksums/размерами. GET content для queued/processing/failed даёт 409, для чужого attachment — 404. Сообщение может ссылаться на queued/processing своего чата, failed прикреплять нельзя. Download остаётся авторизованным proxy через API без redirect. + +Attempt увеличивается только при успешном claim; job_id при redelivery/retry не меняется. По умолчанию максимум **5 попыток** на job. Временные storage/timeouts/processing failures возвращают queued с backoff `min(8, 0.5*2^(attempt-1)) + U(0,0.5)` секунд; постоянные invalid_image/source_missing/checksum_mismatch/result_conflict сразу дают failed. Исчерпание бюджета — failed retry_exhausted. В публичной ошибке нет exception details. Реестр failed и job_events доступны оператору, не клиенту. + +Повторная delivery уже ready/failed — no-op. Busy/early duplicate не отбирает чужой lease. Истёкший lease снимает ownership и создаёт следующую управляемую попытку. Final COMMIT проверяет current_job_id, pipeline_version, state, lease_token **и срок lease**; потерявший аренду worker не может опубликовать старый результат. + +## Версионирование и инварианты трека 2 + +Ключ эффекта: `(attachment_id, pipeline_version, variant)`. Ключи объектов: `results//vN/original|thumbnail|crop|optimized`. PUT результата условный `If-None-Match: *`; готовый объект с совпадающими size/hash повторно не пишется, конфликтующий не затирается. Результаты детерминированы закреплённым renderer и параметрами версии. + +Операторский переход на следующую реализованную версию атомарно помечает старую job superseded, создаёт новую queued job с отдельным job_id и обновляет current_job_id/version. Источник новой job — устойчивый исходник прежней версии либо staging, а не локальный файл API. Сейчас реализованы только v1 и v2; публичный upload создаёт v1. + +Проверяемые инварианты: + +1. Подтверждённая queued задача сохраняется после смерти API и рестарта broker. +2. Одновременный дубль допускает единственный claim; serial duplicate/reorder не увеличивает число успешно опубликованных вариантов. +3. Падение до/после S3 side effect и до/после результата БД не теряет исходник и не создаёт второй набор объектов той же версии. +4. Устаревшая delivery и уже выполняющийся старый worker не возвращают ready в processing и не затирают результат новой версии. +5. Failed наблюдаем, retries конечны; replay допускается только для текущей failed job с доступным source и оставляет audit. +6. Cleanup не удаляет staging текущей queued/processing работы или свежей failed. + +`student-tests/pipeline_checks.py` выполняет настоящие операции PostgreSQL/S3/Redis и завершает отдельные worker/publisher-процессы через **os._exit(77)** в явных операторских fault points. Это не публичный HTTP hook. Проверены: COMMIT без publisher; publisher после XADD до outbox marker; worker before_side_effect, after_original, after_objects, after_commit; повтор/перестановка envelope; два concurrent consumer; pause старого subprocess после записи всех объектов → завершение v2 → продолжение v1, получающее fenced no-op; поздняя v1 delivery после ready v2. + +Для первого crash действительно ожидается истечение тестовой lease=5 s; в двух следующих случаях deadline той же тестовой строки ускоряется SQL-изменением lease_until, чтобы не тратить по 60 s. Это воздействие теста на **новые собственные job IDs**, не подмена результата. Production lease=60 s не изменяется. После каждого crash выполняется реальный Redis pending reclaim/ack; в соответствующем prefix остаётся ровно четыре объекта. После after_commit новая delivery не увеличивает attempt. + +Transient failure воспроизводится настоящим S3 отказом подписи с заведомо неверным тестовым ключом: budget=2 только для этой job, затем queued → failed retry_exhausted, после явного replay с правильными credentials — ready на attempt 3. Повреждённый JPEG доходит до постоянного failed invalid_image. Проверены EXIF-поворот, прозрачность на белом, отсутствие EXIF в thumbnail и запрет увеличения маленького thumbnail. + +Команда исследования, **только на синтетическом стенде** (останавливает фоновую обработку, не API): + +```sh +docker compose stop worker dispatcher +docker compose run --rm -T --no-deps -v "$PWD:/work:ro" -w /work audit python student-tests/pipeline_checks.py +docker compose up -d --no-deps worker dispatcher +``` + +`audit` — tools profile, не запускается обычным up. Только он получает дополнительные локальные секреты для операторских/негативных проверок; API их не получает. Исследование автоматически создаёт новые synthetic users/chats/jobs, не изменяет чужие. + +## Операторские команды и очистка + +```sh +# Наблюдаемый реестр failed: +docker compose exec worker python src/admin.py failed +# Явный replay текущей failed job; UUID взять из предыдущей команды: +docker compose exec worker python src/admin.py replay JOB_UUID +# Новый pipeline для конкретного вложения: +docker compose exec worker python src/admin.py version ATTACHMENT_UUID --version 2 +# Dry-run и явное применение cleanup: +docker compose exec worker python src/admin.py cleanup +docker compose exec worker python src/admin.py cleanup --apply +``` + +Replay не сбрасывает счётчик, не повторяет ready и не создаёт новую копию успешной job: явно добавляет новый бюджет попыток к тому же job_id, сохраняя историю. Необходим доступный source; после истечения retention может понадобиться внешнее восстановление исходника. + +Staging удаляется только **после COMMIT ready**. Crash после COMMIT до очистки безопасен: операторский cleanup идемпотентно удаляет оставшийся staging готовой работы. Failed/незавершённый uploading хранится минимум **24 h**; queued/processing никогда не собираются. Cleanup использует row locks/SKIP LOCKED и фильтр managed staging prefix; dry-run без --apply ничего не меняет. Счётчики cleanup отражают кандидаты по метаданным: часть staging objects уже удалена worker, повторная очистка безопасна. Реестр и сообщения не удаляются вместе с исходником. Неактуальные immutable outputs старых версий пока сохраняются — автоматическая уборка old-version results не заявляется. + +## Результаты и воспроизводимость + +| Критерий | Проверка и наблюдение | Evidence | +| --- | --- | --- | +| Публичный минимум lab5 | Smoke: 202, ready, исходный SHA-256 и настоящий JPEG thumbnail | final-tests.txt | +| Прежний API и собственные изображения | 18 собственных тестов PASS: Unicode/NUL, auth, гонки/idempotency, variants, ACL, MIME/bytes/pixels/filename | student-tests.txt, final-tests.txt | +| Устойчивый staging/очередь | Worker/dispatcher остановлены, upload 202 и message; обе API убиты SIGKILL до publish; после рестарта со старым Bearer всё сохранено | durability.json | +| Persistence broker | Настоящий Redis Streams XADD; до/после restart с volume — 1/1 запись; затем та же job ready | durability.json | +| Полный restart | API/LB/worker/dispatcher остановлены, PostgreSQL/cache/оба S3/broker перезапущены с volumes; старые сессия и bytes сохранены | persistence.txt, restart-before/after.json | +| Отказ API | 2 реплики, 114 подтверждённых writes, 2 переходные ошибки, восстановление 2,534 s | failover.txt | +| Все уровни retry/final/replay | Реальные S3 signature failures, конечный budget=2, audit/replay и готовый результат на attempt 3 | pipeline.json | +| Outbox и worker crash windows | Отдельные процессы exit 77, реальные redelivery и четыре объекта на версию | pipeline.json | +| Reorder, concurrent duplicate, stale version | 13 групп assertions/invariants; old running process fenced, new ready не изменился | pipeline.json | +| Изображения и лимиты worker | EXIF/alpha/small-image, malformed JPEG failed, 40 Мп RGB ready, subprocess/container limits | pipeline.json, resources.json | +| Cleanup | Dry-run/apply/repeat, защита активного source и свежих failed | pipeline.json | +| Приватность | Anonymous known-key GET/list →403; в envelope только 7 служебных полей, проверены неверные типы/форматы | security.json | +| Полная валидация схемы | OpenAPI 3.1 / JSON Schema PASS | openapi.txt | +| Чистый clone | Отдельные новые секреты, проект, порт и пустые volumes | PASS: новые секреты и пустые PG/S3/broker volumes, 18 тестов, SIGKILL API с queued work и полный restart; clean-clone.txt | + +Все сценарии сохраняют tokens только в памяти теста. Snapshot содержит разрешённые поля state/started_at/mounts/resources, не environment или secret values. Fault injection и операторские команды воспроизводимы из репозитория. Дополнительные команды: + +```sh +export BASE_URL=http://127.0.0.1:18091 +python3 student-tests/durability.py +python3 student-tests/resources.py +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=$BASE_URL +docker compose up -d --wait --no-deps api +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL +``` + +### Измерение ресурсов и задержек + +Это дополнительная проверка bounded worker, **не заявка на трек 1**. Concurrency/prefetch=1, один прогон на профиль, по 10 одновременно начинающих клиентов. Генерация PNG до измерения; отправляется реальное тело, без скрытых retry. Worker перезапускается перед профилем, cgroup peak относится к этому запуску. Считается время HTTP-accept и время до наблюдаемого ready; polling добавляет до примерно 0,3 s. Для N=10 nearest-rank p95/p99 равны максимуму. Сэмплер docker exec также вносит overhead, closed-loop не даёт верхней границы production throughput. + +| Pixels / bytes | 202: p95 | До ready: p50 / p95=p99 | Worker cgroup peak / resident anon | Успех | +| --- | ---: | ---: | ---: | --- | +| 64 000 / 1 294 | 0,080 s | 1,834 / 2,923 s | 78,81 / 76,82 MiB | 10/10 | +| 2 520 000 / 7 563 763 | 1,102 s | 4,155 / 7,025 s | 123,46 / 112,50 MiB | 10/10 | +| 40 000 000 / 533 859 | 0,092 s | 6,826 / 11,659 s | 253,67 / 251,36 MiB | 10/10 | + +После исследования и восстановления очередь полностью опустела: 85 ready, 3 намеренно failed из-за повреждённых JPEG, 0 queued/processing/uploading, 0 stream entries/pending deliveries; у всех ready по три варианта (jobs-summary.json). + +Все 30 приняли 202 и достигли ready; OOM kill=0. `memory.peak` — kernel cgroup максимум контейнера, включая renderer, tmpfs/cache/kernel; resident anon взят как максимум `memory.stat anon` среди samples, это не точный максимальный RSS каждого отдельного PID. CPU counters также сохранены в raw samples. Маленький сжатый файл 40 Мп занимает больше RAM/CPU при декодировании, чем более крупный файл с меньшим количеством pixels. Быстрый 202 не означает мгновенный thumbnail: работа проходит ограниченный worker и ждёт очередь. + +![Задержка и память отдельного worker](evidence/pipeline.png) + +Основной исследовательский результат трека 2: все внедрённые crash/duplicate/reorder/version сценарии соблюли заявленные инварианты; умирающий после COMMIT worker не заставил повторно публиковать эффект, а живой, но устаревший worker получил fencing вместо перезаписи новой версии. Это измеренная функциональная проверка, не статистическое доказательство отсутствия всех гонок. + +Для графиков: Python 3.11+, `pip install -r requirements-research.txt`, `python student-tests/plot.py`. PNG/SVG уже приложены. Workflow Application добавляет сборку/стенд/smoke/own tests/failover/persistence; hosted CI отдельно не объявляется прошедшим без результата платформы. + +## Ограничения + +Один host/PostgreSQL/S3-store/broker/LB остаются точками отказа. Доказаны процессные сбои и restart с volumes, не потеря диска/всего хоста или гарантия power-loss durability. HTTP и неподключённые к хосту backend-порты допускаются только для локального синтетического стенда; удалённой публикации без TLS нет. Docker operator имеет доступ к секретам. + +Существующий immutable результат проверяется по HEAD size/hash metadata, не повторным чтением всего объекта; защита от произвольной подмены привилегированным S3-оператором не заявляется. + +Нет распределённой ACID-транзакции. Outbox и периодическая republish закрывают разрывы повтором, поэтому сообщения/CPU могут дублироваться; fences и immutable versioned keys защищают опубликованный результат. Потерявшийся worker может ещё некоторое время тратить CPU до timeout/завершения, но не публиковать старый ready. Нормальные bounded render заканчиваются до lease; выбирая меньшую lease, надо отдельно измерять нагрузку. + +Состояния jobs/events и неактуальные versions накапливаются. Cleanup запускает оператор, не скрытый scheduler. Retention failed sources ограничивает возможность replay без внешней копии. Принятая задача может закончиться failed из-за постоянной ошибки или исчерпания retry; это наблюдаемое завершение, а не исчезновение работы. Image parser ограничивает декодированные PNG-текстовые метаданные 1 MiB. Антивирус и UI не входят в работу. + +## Источники и ИИ + +Собственное решение lab4 (Майорова Софья, PR 18); чужие студенческие решения не копировались. Задание и contracts текущего skeleton — основание обязательного поведения. + +- [Redis persistence/AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [XAUTOCLAIM](https://redis.io/docs/latest/commands/xautoclaim/), [Streams](https://redis.io/docs/latest/develop/data-types/streams/). +- [Pillow ImageOps/EXIF/fit](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html). +- [SeaweedFS S3 credentials](https://github.com/seaweedfs/seaweedfs/wiki/S3-Credentials), [PostgreSQL explicit locks](https://www.postgresql.org/docs/17/explicit-locking.html). + +Codex использован для реализации, тестов, fault-injection, запуска измерений и оформления. Все PASS/числа получены реальными командами, границы их применимости приведены выше. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..25366f0e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,234 @@ +name: mtusi-lab5 +x-application: &application + image: mtusi-messenger:lab5 + networks: [backend] + environment: &environment + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + HTTP_WORKERS: ${HTTP_WORKERS:-16} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + DB_ACCESS_MODE: ${DB_ACCESS_MODE:-optimized} + LOG_LEVEL: WARNING + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-100} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret + secrets: [database_url, cache_url, s3_api_key, s3_api_secret] + tmpfs: ["/tmp:size=128m,mode=1777"] + mem_limit: ${API_MEMORY:-512m} + cpus: ${API_CPUS:-1} + stop_grace_period: 10s +services: + migrate: + <<: *application + command: [python, src/migrate.py] + healthcheck: + disable: true + deploy: + replicas: 1 + depends_on: + postgres: + condition: service_healthy + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + bucket-bootstrap: + condition: service_completed_successfully + bucket-bootstrap: + <<: *application + command: [python, src/bucket_bootstrap.py] + environment: + <<: *environment + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: [s3_admin_key, s3_admin_secret] + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + audit: + <<: *application + profiles: [tools] + environment: + <<: *environment + BASE_URL: http://lb:8080 + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: 5 + secrets: [database_url, cache_url, s3_api_key, s3_api_secret, s3_admin_key, s3_admin_secret, s3_worker_key, s3_worker_secret, broker_url] + healthcheck: + disable: true + deploy: + replicas: 1 + dispatcher: + <<: *application + command: [python, src/pipeline.py, dispatch] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + secrets: [database_url, broker_url] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + worker: + <<: *application + command: [python, src/pipeline.py, work] + environment: + <<: *environment + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-60} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + secrets: [database_url, broker_url, s3_worker_key, s3_worker_secret] + mem_limit: ${WORKER_MEMORY:-768m} + tmpfs: ["/tmp:size=128m,mode=1777"] + cpus: 1 + deploy: + replicas: 1 + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + bucket-bootstrap: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + broker: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --appendonly, "yes", --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction, --save, ""] + volumes: [broker_data:/data] + networks: [backend] + mem_limit: 192m + cpus: 0.5 + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + s3-store: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [mini, -dir=/data, -ip=s3-store, -s3=false, -admin.ui=false, -webdav=false, -master.telemetry=false, -master.volumeSizeLimitMB=128, -volume.max=16, -volume.fileSizeLimitMB=11, -volume.concurrentUploadLimitMB=64, -volume.concurrentDownloadLimitMB=64] + networks: [storage] + volumes: [s3_data:/data] + mem_limit: 512m + cpus: 1 + stop_grace_period: 20s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8888/] + interval: 2s + timeout: 2s + retries: 45 + s3: + image: chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8 + command: [s3, -filer=s3-store:8888, -port=8333, -config=/run/secrets/s3_config, -iam=false, -port.iceberg=0, -port.lance=0, -concurrentFileUploadLimit=16, -concurrentUploadLimitMB=32, -readerCacheSizeMB=16] + secrets: [s3_config] + networks: [backend, storage] + mem_limit: 512m + environment: + GOMEMLIMIT: 384MiB + cpus: 0.5 + depends_on: + s3-store: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'nc -z 127.0.0.1 8333'] + interval: 2s + timeout: 1s + retries: 30 + lb: + image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949 + ports: ["127.0.0.1:${APP_PORT:-18091}:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + mem_limit: 64m + cpus: 0.5 + stop_grace_period: 12s + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 2s + retries: 10 + postgres: + image: postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [backend] + healthcheck: + test: [CMD, pg_isready, -h, 127.0.0.1, -U, messenger, -d, messenger] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1} + redis: + image: redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47 + command: [redis-server, --maxmemory, 32mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 1s + retries: 30 + mem_limit: 64m + cpus: 0.25 +volumes: + postgres_data: + broker_data: + s3_data: +networks: + edge: + storage: + internal: true + backend: + internal: true +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_api_key: + file: ./secrets/s3_api_key + s3_api_secret: + file: ./secrets/s3_api_secret + s3_config: + file: ./secrets/s3_config + broker_url: + file: ./secrets/broker_url + s3_worker_key: + file: ./secrets/s3_worker_key + s3_worker_secret: + file: ./secrets/s3_worker_secret diff --git a/evidence/build.txt b/evidence/build.txt new file mode 100644 index 00000000..336722dd --- /dev/null +++ b/evidence/build.txt @@ -0,0 +1,55 @@ + Image mtusi-messenger:lab5 Building +#1 [internal] load local bake definitions +#1 reading from stdin 650B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 15.97kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [2/7] WORKDIR /app +#8 CACHED + +#9 [4/7] COPY requirements.txt ./ +#9 CACHED + +#10 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#10 CACHED + +#11 [6/7] COPY src/ ./src/ +#11 DONE 0.0s + +#12 [7/7] COPY migrations/ ./migrations/ +#12 DONE 0.0s + +#13 exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:7e99f0c6a7ac94b85161817ff88967106e59e98d4e7dd9de3938b1135e0b12cd done +#13 exporting config sha256:0e7c28f3d856890d8caee1ac1cdc0b496010b55a4e1f60fd6b733acc575f7f2e done +#13 exporting attestation manifest sha256:f9c54687679fe921dd39dde4f93898025e671dd4d91f692ffb6ec14f50df16c6 done +#13 exporting manifest list sha256:14afbe72a221379a2e0438f88629275c862f8347151aee0a33ac60708793d1c2 done +#13 naming to docker.io/library/mtusi-messenger:lab5 done +#13 unpacking to docker.io/library/mtusi-messenger:lab5 0.0s done +#13 DONE 0.1s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image mtusi-messenger:lab5 Built diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..fbb76a17 --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,1054 @@ +Cloning into '/Users/ssmayorova/Documents/Учеба/Высоконагруженные приложения/lab5/build/lab5-clean-7668fd68/repo'... +done. +Local configuration ready; no secrets printed. + Image mtusi-messenger:lab5 Building +#1 [internal] load local bake definitions +#1 reading from stdin 721B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 739B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 1.1s + +#4 [internal] load .dockerignore +#4 transferring context: 203B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 73.48kB done +#5 DONE 0.0s + +#6 [1/7] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#7 CACHED + +#8 [4/7] COPY requirements.txt ./ +#8 CACHED + +#9 [5/7] RUN pip install --no-cache-dir -r requirements.txt +#9 CACHED + +#10 [6/7] COPY src/ ./src/ +#10 CACHED + +#11 [2/7] WORKDIR /app +#11 CACHED + +#12 [7/7] COPY migrations/ ./migrations/ +#12 CACHED + +#13 resolving provenance for metadata file +#13 DONE 0.0s + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:e06ce20ee092cb4997cdd7c8020b9659af43ac04b1b2dcbed69ea0ce95100049 done +#14 exporting config sha256:6b11dd9f29f79206ad928a7577a62cc834ad7502c3231f032e29534c803d81b3 done +#14 exporting attestation manifest sha256:861ed44d72400f7f11a9e27680aa5ff785a038924413c6784a4976384aabc2b5 done +#14 exporting manifest list sha256:8dd5aba56cbd0954c950d890bda5b57986bd0fd89535e5d140387bacbb97e65d done +#14 naming to docker.io/library/mtusi-messenger:lab5 done +#14 unpacking to docker.io/library/mtusi-messenger:lab5 done +#14 DONE 0.0s + Image mtusi-messenger:lab5 Built + Network lab5-clean-7668fd68_storage Creating + Network lab5-clean-7668fd68_storage Created + Network lab5-clean-7668fd68_edge Creating + Network lab5-clean-7668fd68_edge Created + Network lab5-clean-7668fd68_backend Creating + Network lab5-clean-7668fd68_backend Created + Volume lab5-clean-7668fd68_broker_data Creating + Volume lab5-clean-7668fd68_broker_data Created + Volume lab5-clean-7668fd68_postgres_data Creating + Volume lab5-clean-7668fd68_postgres_data Created + Volume lab5-clean-7668fd68_s3_data Creating + Volume lab5-clean-7668fd68_s3_data Created + Container lab5-clean-7668fd68-postgres-1 Creating + Container lab5-clean-7668fd68-redis-1 Creating + Container lab5-clean-7668fd68-broker-1 Creating + Container lab5-clean-7668fd68-s3-store-1 Creating + Container lab5-clean-7668fd68-broker-1 Created + Container lab5-clean-7668fd68-postgres-1 Created + Container lab5-clean-7668fd68-migrate-1 Creating + Container lab5-clean-7668fd68-redis-1 Created + Container lab5-clean-7668fd68-s3-store-1 Created + Container lab5-clean-7668fd68-s3-1 Creating + Container lab5-clean-7668fd68-migrate-1 Created + Container lab5-clean-7668fd68-dispatcher-1 Creating + Container lab5-clean-7668fd68-s3-1 Created + Container lab5-clean-7668fd68-bucket-bootstrap-1 Creating + Container lab5-clean-7668fd68-dispatcher-1 Created + Container lab5-clean-7668fd68-bucket-bootstrap-1 Created + Container lab5-clean-7668fd68-worker-1 Creating + Container lab5-clean-7668fd68-api-2 Creating + Container lab5-clean-7668fd68-api-1 Creating + Container lab5-clean-7668fd68-api-1 Created + Container lab5-clean-7668fd68-api-2 Created + Container lab5-clean-7668fd68-worker-1 Created + Container lab5-clean-7668fd68-lb-1 Creating + Container lab5-clean-7668fd68-lb-1 Created + Container lab5-clean-7668fd68-postgres-1 Starting + Container lab5-clean-7668fd68-s3-store-1 Starting + Container lab5-clean-7668fd68-broker-1 Starting + Container lab5-clean-7668fd68-redis-1 Starting + Container lab5-clean-7668fd68-redis-1 Started + Container lab5-clean-7668fd68-broker-1 Started + Container lab5-clean-7668fd68-postgres-1 Started + Container lab5-clean-7668fd68-postgres-1 Waiting + Container lab5-clean-7668fd68-s3-store-1 Started + Container lab5-clean-7668fd68-s3-store-1 Waiting + Container lab5-clean-7668fd68-postgres-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Starting + Container lab5-clean-7668fd68-migrate-1 Started + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-s3-store-1 Healthy + Container lab5-clean-7668fd68-s3-1 Starting + Container lab5-clean-7668fd68-s3-1 Started + Container lab5-clean-7668fd68-s3-1 Waiting + Container lab5-clean-7668fd68-s3-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Starting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Started + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-redis-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-redis-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-api-2 Starting + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-worker-1 Starting + Container lab5-clean-7668fd68-worker-1 Started + Container lab5-clean-7668fd68-api-2 Started + Container lab5-clean-7668fd68-api-1 Starting + Container lab5-clean-7668fd68-api-1 Started + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-lb-1 Starting + Container lab5-clean-7668fd68-lb-1 Started + Container lab5-clean-7668fd68-lb-1 Waiting + Container lab5-clean-7668fd68-dispatcher-1 Waiting + Container lab5-clean-7668fd68-redis-1 Waiting + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-postgres-1 Waiting + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-bucket-bootstrap-1 Waiting + Container lab5-clean-7668fd68-worker-1 Waiting + Container lab5-clean-7668fd68-s3-store-1 Waiting + Container lab5-clean-7668fd68-migrate-1 Waiting + Container lab5-clean-7668fd68-s3-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-s3-1 Healthy + Container lab5-clean-7668fd68-bucket-bootstrap-1 Exited + Container lab5-clean-7668fd68-dispatcher-1 Healthy + Container lab5-clean-7668fd68-migrate-1 Exited + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-redis-1 Healthy + Container lab5-clean-7668fd68-postgres-1 Healthy + Container lab5-clean-7668fd68-s3-store-1 Healthy + Container lab5-clean-7668fd68-worker-1 Healthy + Container lab5-clean-7668fd68-lb-1 Healthy +Ready through load balancer. +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18092" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18092" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.131s + +OK + Container lab5-clean-7668fd68-worker-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-worker-1 Stopped + Container lab5-clean-7668fd68-api-2 Starting + Container lab5-clean-7668fd68-api-2 Started + Container lab5-clean-7668fd68-api-1 Starting + Container lab5-clean-7668fd68-api-1 Started + Container lab5-clean-7668fd68-api-2 Waiting + Container lab5-clean-7668fd68-api-1 Waiting + Container lab5-clean-7668fd68-api-2 Healthy + Container lab5-clean-7668fd68-api-1 Healthy + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-broker-1 Restarting + Container lab5-clean-7668fd68-broker-1 Started + Container lab5-clean-7668fd68-broker-1 Running + Container lab5-clean-7668fd68-broker-1 Waiting + Container lab5-clean-7668fd68-broker-1 Healthy + Container lab5-clean-7668fd68-dispatcher-1 Starting + Container lab5-clean-7668fd68-worker-1 Starting + Container lab5-clean-7668fd68-dispatcher-1 Started + Container lab5-clean-7668fd68-worker-1 Started + Container lab5-clean-7668fd68-worker-1 Running + Container lab5-clean-7668fd68-dispatcher-1 Running +{ + "killed_api_containers": [ + "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56" + ], + "stream_entries_before_after_broker_restart": [ + 1, + 1 + ], + "attachment_id": "c087ac91-80ee-4954-a2c1-4c7f01787cd4", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "status": "ready", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "old_bearer_survived": true, + "message_attachment_retained": true, + "before": [ + { + "container": "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "name": "/lab5-clean-7668fd68-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.797245354Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56", + "name": "/lab5-clean-7668fd68-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.73913582Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "5567835c3bab8e2f4b997cb67968c2155a0922fe19240cd60c73e5e090b2760a", + "name": "/lab5-clean-7668fd68-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.927126872Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_broker_data" + } + ] + }, + { + "container": "65eed1838fce2bcb378be8d14628313d7c14fead8280bb63d211015767a70cd5", + "name": "/lab5-clean-7668fd68-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:34.154321143Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "c506b79db3e60b71e314e78e4abda0256838675bb9e9daba579efbb5b8f2e744", + "name": "/lab5-clean-7668fd68-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:30.132057402Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "791d5c6b17a6fbf020c38648a40d7c0fc0ae2db9cedd11aecb85b604da9fc6c6", + "name": "/lab5-clean-7668fd68-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:14:40.374957025Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18092" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "7378d8f662313d4f5475dc6b6be1f82e9e60736564a5d94c4e939a75ea87d9f4", + "name": "/lab5-clean-7668fd68-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:29.547624841Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "12dd22014a45a0658b2fa3c5117723c471c2f0c438a245e381fd7904eee04080", + "name": "/lab5-clean-7668fd68-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926579904Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab5-clean-7668fd68_postgres_data" + } + ] + }, + { + "container": "e5bcddc6ef40108e9af7bce37e209ea1524097dab06b232bf9126a0e0d41e303", + "name": "/lab5-clean-7668fd68-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926291214Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "310d1b7f9d4700a40971a9d27e5c49bfc7f3d818af445ccf985fc3ca5bab4a5b" + } + ] + }, + { + "container": "35e9daaeef21c5bfc0cfa46bfe9054b23236c1ad76e62bab70b8602d44e82c00", + "name": "/lab5-clean-7668fd68-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:31.546652936Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "a9677d401ea04710c1220e23073bd3d05c5f6955399379842d5357a3f6d8a5c4" + } + ] + }, + { + "container": "0847dd77bc06ba2fe81753689f8f46ff09b0dd1a6851d8109762c9552bb10f98", + "name": "/lab5-clean-7668fd68-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.925042463Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_s3_data" + } + ] + }, + { + "container": "e3343c9d17d09d716d03066cbac7bc24030fa1061164c7efc75eda9413954c0b", + "name": "/lab5-clean-7668fd68-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:34.738831925Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ], + "after": [ + { + "container": "c6bf17afaf48a316d3bd139d819ef527551f14f3677ef898c0a2c8c4698d5d11", + "name": "/lab5-clean-7668fd68-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:50.908562499Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "6384b285c9055f141caa6f03f2e5cfd69e1d3a4f65a3db4b9af949b03215ca56", + "name": "/lab5-clean-7668fd68-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:14:50.859526761Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "5567835c3bab8e2f4b997cb67968c2155a0922fe19240cd60c73e5e090b2760a", + "name": "/lab5-clean-7668fd68-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:59.65492626Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_broker_data" + } + ] + }, + { + "container": "65eed1838fce2bcb378be8d14628313d7c14fead8280bb63d211015767a70cd5", + "name": "/lab5-clean-7668fd68-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:34.154321143Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "c506b79db3e60b71e314e78e4abda0256838675bb9e9daba579efbb5b8f2e744", + "name": "/lab5-clean-7668fd68-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:15:02.066887519Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "791d5c6b17a6fbf020c38648a40d7c0fc0ae2db9cedd11aecb85b604da9fc6c6", + "name": "/lab5-clean-7668fd68-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:14:40.374957025Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18092" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "7378d8f662313d4f5475dc6b6be1f82e9e60736564a5d94c4e939a75ea87d9f4", + "name": "/lab5-clean-7668fd68-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:14:29.547624841Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "12dd22014a45a0658b2fa3c5117723c471c2f0c438a245e381fd7904eee04080", + "name": "/lab5-clean-7668fd68-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926579904Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab5-clean-7668fd68_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "e5bcddc6ef40108e9af7bce37e209ea1524097dab06b232bf9126a0e0d41e303", + "name": "/lab5-clean-7668fd68-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.926291214Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "310d1b7f9d4700a40971a9d27e5c49bfc7f3d818af445ccf985fc3ca5bab4a5b" + } + ] + }, + { + "container": "35e9daaeef21c5bfc0cfa46bfe9054b23236c1ad76e62bab70b8602d44e82c00", + "name": "/lab5-clean-7668fd68-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:31.546652936Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "a9677d401ea04710c1220e23073bd3d05c5f6955399379842d5357a3f6d8a5c4" + } + ] + }, + { + "container": "0847dd77bc06ba2fe81753689f8f46ff09b0dd1a6851d8109762c9552bb10f98", + "name": "/lab5-clean-7668fd68-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:14:26.925042463Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab5-clean-7668fd68_s3_data" + } + ] + }, + { + "container": "e3343c9d17d09d716d03066cbac7bc24030fa1061164c7efc75eda9413954c0b", + "name": "/lab5-clean-7668fd68-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:15:02.065936755Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ] +} +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18092" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab5-clean-7668fd68-worker-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopping + Container lab5-clean-7668fd68-lb-1 Stopping + Container lab5-clean-7668fd68-dispatcher-1 Stopped + Container lab5-clean-7668fd68-dispatcher-1 Removing + Container lab5-clean-7668fd68-dispatcher-1 Removed + Container lab5-clean-7668fd68-lb-1 Stopped + Container lab5-clean-7668fd68-lb-1 Removing + Container lab5-clean-7668fd68-lb-1 Removed + Container lab5-clean-7668fd68-api-1 Stopping + Container lab5-clean-7668fd68-api-2 Stopping + Container lab5-clean-7668fd68-api-2 Stopped + Container lab5-clean-7668fd68-api-2 Removing + Container lab5-clean-7668fd68-api-2 Removed + Container lab5-clean-7668fd68-api-1 Stopped + Container lab5-clean-7668fd68-api-1 Removing + Container lab5-clean-7668fd68-worker-1 Stopped + Container lab5-clean-7668fd68-worker-1 Removing + Container lab5-clean-7668fd68-api-1 Removed + Container lab5-clean-7668fd68-redis-1 Stopping + Container lab5-clean-7668fd68-worker-1 Removed + Container lab5-clean-7668fd68-broker-1 Stopping + Container lab5-clean-7668fd68-bucket-bootstrap-1 Stopping + Container lab5-clean-7668fd68-migrate-1 Stopping + Container lab5-clean-7668fd68-bucket-bootstrap-1 Stopped + Container lab5-clean-7668fd68-bucket-bootstrap-1 Removing + Container lab5-clean-7668fd68-migrate-1 Stopped + Container lab5-clean-7668fd68-migrate-1 Removing + Container lab5-clean-7668fd68-bucket-bootstrap-1 Removed + Container lab5-clean-7668fd68-s3-1 Stopping + Container lab5-clean-7668fd68-migrate-1 Removed + Container lab5-clean-7668fd68-postgres-1 Stopping + Container lab5-clean-7668fd68-postgres-1 Stopped + Container lab5-clean-7668fd68-postgres-1 Removing + Container lab5-clean-7668fd68-postgres-1 Removed + Container lab5-clean-7668fd68-s3-1 Stopped + Container lab5-clean-7668fd68-s3-1 Removing + Container lab5-clean-7668fd68-s3-1 Removed + Container lab5-clean-7668fd68-s3-store-1 Stopping + Container lab5-clean-7668fd68-redis-1 Stopped + Container lab5-clean-7668fd68-redis-1 Removing + Container lab5-clean-7668fd68-redis-1 Removed + Container lab5-clean-7668fd68-broker-1 Stopped + Container lab5-clean-7668fd68-broker-1 Removing + Container lab5-clean-7668fd68-broker-1 Removed + Container lab5-clean-7668fd68-s3-store-1 Stopped + Container lab5-clean-7668fd68-s3-store-1 Removing + Container lab5-clean-7668fd68-s3-store-1 Removed + Volume lab5-clean-7668fd68_broker_data Removing + Volume lab5-clean-7668fd68_s3_data Removing + Volume lab5-clean-7668fd68_postgres_data Removing + Network lab5-clean-7668fd68_edge Removing + Network lab5-clean-7668fd68_storage Removing + Network lab5-clean-7668fd68_backend Removing + Volume lab5-clean-7668fd68_broker_data Removed + Volume lab5-clean-7668fd68_postgres_data Removed + Volume lab5-clean-7668fd68_s3_data Removed + Network lab5-clean-7668fd68_storage Removed + Network lab5-clean-7668fd68_edge Removed + Network lab5-clean-7668fd68_backend Removed +PASS: clean clone, new secrets, empty PostgreSQL/S3/broker volumes; 18 own tests, API SIGKILL and broker restart with queued work; full persistence. diff --git a/evidence/durability-stderr.txt b/evidence/durability-stderr.txt new file mode 100644 index 00000000..f13820d4 --- /dev/null +++ b/evidence/durability-stderr.txt @@ -0,0 +1,27 @@ + Container mtusi-lab5-worker-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopped + Container mtusi-lab5-worker-1 Stopped + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-api-1 Started + Container mtusi-lab5-api-2 Starting + Container mtusi-lab5-api-2 Started + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-dispatcher-1 Stopping + Container mtusi-lab5-dispatcher-1 Stopped + Container mtusi-lab5-broker-1 Restarting + Container mtusi-lab5-broker-1 Started + Container mtusi-lab5-broker-1 Running + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-worker-1 Starting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-dispatcher-1 Running + Container mtusi-lab5-worker-1 Running diff --git a/evidence/durability.json b/evidence/durability.json new file mode 100644 index 00000000..6a1fd1df --- /dev/null +++ b/evidence/durability.json @@ -0,0 +1,766 @@ +{ + "killed_api_containers": [ + "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3" + ], + "stream_entries_before_after_broker_restart": [ + 1, + 1 + ], + "attachment_id": "06570cf0-e36c-4403-b10d-c216f92d9696", + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "status": "ready", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "old_bearer_survived": true, + "message_attachment_retained": true, + "before": [ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475559008Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.552220082Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.074661473Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:01.993890605Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475268543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ], + "after": [ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.608120136Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.67064591Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:08:06.296755178Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679298033Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679577406Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } + ] +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..4a4f4707 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18091" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 2, "first_recovery_seconds": 2.534} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-tests.txt b/evidence/final-tests.txt new file mode 100644 index 00000000..a03043e3 --- /dev/null +++ b/evidence/final-tests.txt @@ -0,0 +1,28 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.344s + +OK diff --git a/evidence/jobs-summary.json b/evidence/jobs-summary.json new file mode 100644 index 00000000..f0c552ef --- /dev/null +++ b/evidence/jobs-summary.json @@ -0,0 +1,73 @@ +{ + "attachments_by_state": [ + { + "state": "failed", + "count": 3 + }, + { + "state": "ready", + "count": 85 + } + ], + "ready_without_three_variants": 0, + "stream_length": 0, + "pending_deliveries": 0, + "job_events": [ + { + "event": "claimed", + "count": 107 + }, + { + "event": "duplicate_or_early_noop", + "count": 3 + }, + { + "event": "failed:invalid_image", + "count": 3 + }, + { + "event": "failed:retry_exhausted", + "count": 2 + }, + { + "event": "failed:source_missing", + "count": 3 + }, + { + "event": "fenced_noop", + "count": 3 + }, + { + "event": "operator_replay", + "count": 5 + }, + { + "event": "operator_supersede", + "count": 3 + }, + { + "event": "published", + "count": 74 + }, + { + "event": "ready", + "count": 85 + }, + { + "event": "retry:lease_expired", + "count": 9 + }, + { + "event": "retry:storage_unavailable", + "count": 2 + }, + { + "event": "stale_noop", + "count": 3 + }, + { + "event": "terminal_noop", + "count": 30 + } + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..7de89ff9 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18091" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline-stderr.txt b/evidence/pipeline-stderr.txt new file mode 100644 index 00000000..3c0d05a0 --- /dev/null +++ b/evidence/pipeline-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab5-audit-run-b338f8a5a02a Creating + Container mtusi-lab5-audit-run-b338f8a5a02a Created diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..e56c8156 --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,119 @@ +{ + "invariants": { + "commit_before_publish_recovered": true, + "publish_before_marker_duplicate_safe": true, + "crashes_before_after_side_effect_and_commit": true, + "invalid_envelope_observable_dlq": true, + "duplicate_and_reorder_no_second_effect": true, + "concurrent_duplicate_single_claim": true, + "stale_running_version_fenced": true, + "old_delivery_after_new_ready_no_overwrite": true, + "failed_registry_explicit_replay_and_ready_refusal": true, + "transient_backoff_exhaustion_then_replay": true, + "invalid_decode_permanent_failed": true, + "cleanup_preserves_current_and_fresh_failed": true, + "ready_staging_cleanup_idempotent": true + }, + "faults": [ + { + "point": "before_side_effect", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_original", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_objects", + "exit_code": 77, + "state_at_crash": "processing", + "recovered": "ready", + "attempts": 2, + "result_objects": 4 + }, + { + "point": "after_commit", + "exit_code": 77, + "state_at_crash": "ready", + "recovered": "ready", + "attempts": 1, + "result_objects": 4 + } + ], + "retry": { + "transitions": [ + { + "state": "queued", + "attempt": 1, + "error_code": "storage_unavailable" + }, + { + "state": "failed", + "attempt": 2, + "error_code": "retry_exhausted" + } + ], + "seconds": 1.941, + "replayed_attempt": 3 + }, + "transforms": [ + { + "kind": "alpha", + "size": [ + 20, + 10 + ], + "exif_removed": true + }, + { + "kind": "exif", + "size": [ + 256, + 160 + ], + "exif_removed": true + }, + { + "kind": "small", + "size": [ + 5, + 3 + ], + "exif_removed": true + } + ], + "cleanup": [ + { + "apply": false, + "retention_seconds": 86400, + "counts": { + "ready_staging": 46, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 46, + "expired_staging": 0 + } + }, + { + "apply": true, + "retention_seconds": 86400, + "counts": { + "ready_staging": 0, + "expired_staging": 0 + } + } + ] +} diff --git a/evidence/pipeline.png b/evidence/pipeline.png new file mode 100644 index 00000000..ea12eb3b Binary files /dev/null and b/evidence/pipeline.png differ diff --git a/evidence/pipeline.svg b/evidence/pipeline.svg new file mode 100644 index 00000000..56bd1015 --- /dev/null +++ b/evidence/pipeline.svg @@ -0,0 +1,2374 @@ + + + + + + + + 2026-10-27T18:09:20.663716 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/public-smoke.txt b/evidence/public-smoke.txt new file mode 100644 index 00000000..33326b12 --- /dev/null +++ b/evidence/public-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/resources-stderr.txt b/evidence/resources-stderr.txt new file mode 100644 index 00000000..55c9f9e5 --- /dev/null +++ b/evidence/resources-stderr.txt @@ -0,0 +1,6 @@ + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-worker-1 Restarting + Container mtusi-lab5-worker-1 Started diff --git a/evidence/resources.json b/evidence/resources.json new file mode 100644 index 00000000..8fa4ee6e --- /dev/null +++ b/evidence/resources.json @@ -0,0 +1,1410 @@ +{ + "method": "10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache", + "experiments": [ + { + "width": 320, + "height": 200, + "bytes": 1294, + "clients": 10, + "before": { + "memory.current": "25993216", + "memory.peak": "26140672", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 25546752\nfile 0\nkernel 540672\nkernel_stack 16384\npagetables 196608\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 25509888\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 78624\nslab_unreclaimable 200616\nslab 279240\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 9455\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 217755\nuser_usec 202908\nsystem_usec 14846\ncore_sched.force_idle_usec 0\nnr_periods 2\nnr_throttled 1\nthrottled_usec 27668\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "67489792", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 12288\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 309872\nslab 392928\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 115649\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3037727\nuser_usec 2854014\nsystem_usec 183713\ncore_sched.force_idle_usec 0\nnr_periods 33\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0001, + "worker": { + "memory.current": "30593024", + "memory.peak": "30740480", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 29769728\nfile 0\nkernel 548864\nkernel_stack 16384\npagetables 200704\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 29741056\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 210896\nslab 292912\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13515\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 361161\nuser_usec 337348\nsystem_usec 23812\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 3\nthrottled_usec 189684\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.222, + "worker": { + "memory.current": "46673920", + "memory.peak": "46821376", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 45789184\nfile 0\nkernel 667648\nkernel_stack 16384\npagetables 270336\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 45752320\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 254584\nslab 336600\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 21899\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 551236\nuser_usec 513426\nsystem_usec 37810\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 5\nthrottled_usec 314604\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.4257, + "worker": { + "memory.current": "76713984", + "memory.peak": "76713984", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75374592\nfile 16384\nkernel 937984\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75313152\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 343120\nslab 431152\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 33898\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 762649\nuser_usec 705972\nsystem_usec 56676\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 7\nthrottled_usec 412358\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.6332, + "worker": { + "memory.current": "75657216", + "memory.peak": "76713984", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74448896\nfile 20480\nkernel 933888\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74432512\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 340424\nslab 428456\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 40861\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 940967\nuser_usec 874394\nsystem_usec 66573\ncore_sched.force_idle_usec 0\nnr_periods 10\nnr_throttled 9\nthrottled_usec 498775\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8211, + "worker": { + "memory.current": "66772992", + "memory.peak": "80965632", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65732608\nfile 16384\nkernel 794624\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65708032\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 305592\nslab 389888\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 45711\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1139170\nuser_usec 1062496\nsystem_usec 76674\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 11\nthrottled_usec 553897\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.024, + "worker": { + "memory.current": "66785280", + "memory.peak": "81715200", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65732608\nfile 16384\nkernel 802816\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65732608\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 87376\nslab_unreclaimable 305824\nslab 393200\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 52903\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1343743\nuser_usec 1254425\nsystem_usec 89318\ncore_sched.force_idle_usec 0\nnr_periods 14\nnr_throttled 13\nthrottled_usec 643571\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.2249, + "worker": { + "memory.current": "66695168", + "memory.peak": "81715200", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65744896\nfile 32768\nkernel 806912\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 8192\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65761280\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 87576\nslab_unreclaimable 316592\nslab 404168\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 60107\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1548475\nuser_usec 1449214\nsystem_usec 99261\ncore_sched.force_idle_usec 0\nnr_periods 16\nnr_throttled 15\nthrottled_usec 693176\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4268, + "worker": { + "memory.current": "66973696", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65957888\nfile 28672\nkernel 942080\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65961984\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88632\nslab_unreclaimable 322992\nslab 411624\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 67356\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1750126\nuser_usec 1637151\nsystem_usec 112975\ncore_sched.force_idle_usec 0\nnr_periods 18\nnr_throttled 16\nthrottled_usec 791006\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6289, + "worker": { + "memory.current": "81682432", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80556032\nfile 24576\nkernel 1040384\nkernel_stack 49152\npagetables 450560\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 12288\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 80568320\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 90792\nslab_unreclaimable 405952\nslab 496744\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 74920\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1943185\nuser_usec 1818774\nsystem_usec 124411\ncore_sched.force_idle_usec 0\nnr_periods 20\nnr_throttled 17\nthrottled_usec 838763\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8231, + "worker": { + "memory.current": "80900096", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79634432\nfile 16384\nkernel 999424\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79622144\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 380472\nslab 468504\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 81785\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2131478\nuser_usec 1996963\nsystem_usec 134515\ncore_sched.force_idle_usec 0\nnr_periods 22\nnr_throttled 18\nthrottled_usec 936012\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0172, + "worker": { + "memory.current": "81068032", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79790080\nfile 16384\nkernel 1015808\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79736832\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 397872\nslab 485904\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 89023\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2327586\nuser_usec 2184030\nsystem_usec 143555\ncore_sched.force_idle_usec 0\nnr_periods 23\nnr_throttled 18\nthrottled_usec 936012\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2159, + "worker": { + "memory.current": "80629760", + "memory.peak": "82018304", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79171584\nfile 20480\nkernel 966656\nkernel_stack 49152\npagetables 430080\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79151104\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 88032\nslab_unreclaimable 355496\nslab 443528\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95908\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2514361\nuser_usec 2360865\nsystem_usec 153495\ncore_sched.force_idle_usec 0\nnr_periods 25\nnr_throttled 19\nthrottled_usec 1031276\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.3631, + "worker": { + "memory.current": "71213056", + "memory.peak": "82165760", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 69812224\nfile 20480\nkernel 925696\nkernel_stack 49152\npagetables 397312\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 4096\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 69742592\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 86192\nslab_unreclaimable 343336\nslab 429528\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 100672\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2666143\nuser_usec 2502667\nsystem_usec 163475\ncore_sched.force_idle_usec 0\nnr_periods 27\nnr_throttled 21\nthrottled_usec 1126801\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.5456, + "worker": { + "memory.current": "67538944", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66646016\nfile 28672\nkernel 806912\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 16384\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66662400\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 85096\nslab_unreclaimable 317720\nslab 402816\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 107013\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2860171\nuser_usec 2686677\nsystem_usec 173493\ncore_sched.force_idle_usec 0\nnr_periods 29\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.7395, + "worker": { + "memory.current": "67612672", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66646016\nfile 12288\nkernel 794624\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66646016\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 309872\nslab 392928\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 109894\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2920115\nuser_usec 2742447\nsystem_usec 177668\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.9027, + "worker": { + "memory.current": "67510272", + "memory.peak": "82640896", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 12288\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 306320\nslab 389376\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 112771\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2978023\nuser_usec 2797357\nsystem_usec 180666\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 23\nthrottled_usec 1221745\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 0.05672016600000007, + "ready_seconds": 1.678756416, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.0797383330000001, + "ready_seconds": 1.9898555410000003, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.06636754199999984, + "ready_seconds": 2.3032809579999998, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 0.07966337499999998, + "ready_seconds": 2.923389875, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.07643879200000003, + "ready_seconds": 2.608790333, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.04658795799999993, + "ready_seconds": 1.019897792, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 0.06349704200000006, + "ready_seconds": 0.717395875, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.06630558300000011, + "ready_seconds": 1.661077125, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 0.06494524999999984, + "ready_seconds": 1.346463375, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.06040191699999986, + "ready_seconds": 2.305937583, + "status": "ready" + } + ], + "seconds": 3.093689791 + }, + { + "width": 1800, + "height": 1400, + "bytes": 7563763, + "clients": 10, + "before": { + "memory.current": "23658496", + "memory.peak": "23810048", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 22904832\nfile 0\nkernel 512000\nkernel_stack 16384\npagetables 188416\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 22904832\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 78624\nslab_unreclaimable 188336\nslab 266960\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 8545\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 195466\nuser_usec 180783\nsystem_usec 14682\ncore_sched.force_idle_usec 0\nnr_periods 3\nnr_throttled 1\nthrottled_usec 45259\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "68448256", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67538944\nfile 28672\nkernel 798720\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67538944\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311312\nslab 394368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 298507\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6584973\nuser_usec 6143467\nsystem_usec 441506\ncore_sched.force_idle_usec 0\nnr_periods 75\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0002, + "worker": { + "memory.current": "29175808", + "memory.peak": "29544448", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 28377088\nfile 0\nkernel 544768\nkernel_stack 16384\npagetables 200704\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 28368896\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82016\nslab_unreclaimable 199184\nslab 281200\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13036\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 372577\nuser_usec 350883\nsystem_usec 21694\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 2\nthrottled_usec 100680\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.2164, + "worker": { + "memory.current": "45006848", + "memory.peak": "45006848", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 43810816\nfile 0\nkernel 663552\nkernel_stack 16384\npagetables 266240\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 43757568\ninactive_file 0\nactive_file 0\nunevictable 0\nslab_reclaimable 82856\nslab_unreclaimable 258800\nslab 341656\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 21272\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 573424\nuser_usec 538003\nsystem_usec 35420\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 4\nthrottled_usec 139589\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.3946, + "worker": { + "memory.current": "50413568", + "memory.peak": "50561024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 49602560\nfile 4096\nkernel 745472\nkernel_stack 32768\npagetables 294912\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 49602560\ninactive_file 4096\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 296960\nslab 380016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 25614\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 683616\nuser_usec 637335\nsystem_usec 46281\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 5\nthrottled_usec 183385\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.5611, + "worker": { + "memory.current": "50548736", + "memory.peak": "50696192", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 49602560\nfile 4096\nkernel 741376\nkernel_stack 32768\npagetables 290816\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 49602560\ninactive_file 4096\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 295968\nslab 379024\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 28491\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 743694\nuser_usec 694372\nsystem_usec 49321\ncore_sched.force_idle_usec 0\nnr_periods 9\nnr_throttled 5\nthrottled_usec 183385\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.7123, + "worker": { + "memory.current": "66285568", + "memory.peak": "66433024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65290240\nfile 16384\nkernel 786432\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65290240\ninactive_file 16384\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 306256\nslab 390552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 36810\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 890617\nuser_usec 813340\nsystem_usec 77277\ncore_sched.force_idle_usec 0\nnr_periods 11\nnr_throttled 6\nthrottled_usec 206722\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8867, + "worker": { + "memory.current": "66142208", + "memory.peak": "66433024", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 65302528\nfile 12288\nkernel 786432\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 65290240\ninactive_file 12288\nactive_file 0\nunevictable 0\nslab_reclaimable 84296\nslab_unreclaimable 301712\nslab 386008\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 39693\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 952467\nuser_usec 872127\nsystem_usec 80340\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 6\nthrottled_usec 206722\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.0403, + "worker": { + "memory.current": "84230144", + "memory.peak": "84418560", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75423744\nfile 7593984\nkernel 962560\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 82972672\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 344560\nslab 450352\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 45282\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1087134\nuser_usec 992095\nsystem_usec 95039\ncore_sched.force_idle_usec 0\nnr_periods 15\nnr_throttled 7\nthrottled_usec 293202\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.253, + "worker": { + "memory.current": "113782784", + "memory.peak": "113782784", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 104636416\nfile 7610368\nkernel 1118208\nkernel_stack 49152\npagetables 507904\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 112173056\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 414480\nslab 521464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 57704\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1298250\nuser_usec 1183670\nsystem_usec 114580\ncore_sched.force_idle_usec 0\nnr_periods 17\nnr_throttled 9\nthrottled_usec 398918\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4618, + "worker": { + "memory.current": "75698176", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66265088\nfile 8372224\nkernel 806912\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74604544\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 308728\nslab 414544\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 64522\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1495060\nuser_usec 1363487\nsystem_usec 131573\ncore_sched.force_idle_usec 0\nnr_periods 19\nnr_throttled 11\nthrottled_usec 458319\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6743, + "worker": { + "memory.current": "90038272", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81072128\nfile 7593984\nkernel 1019904\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88625152\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379064\nslab 484856\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 71423\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1677684\nuser_usec 1529797\nsystem_usec 147886\ncore_sched.force_idle_usec 0\nnr_periods 21\nnr_throttled 12\nthrottled_usec 536499\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8627, + "worker": { + "memory.current": "121225216", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 112369664\nfile 7610368\nkernel 1138688\nkernel_stack 49152\npagetables 524288\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 119922688\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 415728\nslab 522712\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 84185\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1870527\nuser_usec 1709663\nsystem_usec 160863\ncore_sched.force_idle_usec 0\nnr_periods 23\nnr_throttled 14\nthrottled_usec 650482\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0596, + "worker": { + "memory.current": "75735040", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66523136\nfile 8372224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74850304\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 309976\nslab 415792\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 89179\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2032115\nuser_usec 1860152\nsystem_usec 171963\ncore_sched.force_idle_usec 0\nnr_periods 24\nnr_throttled 15\nthrottled_usec 658121\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2097, + "worker": { + "memory.current": "88981504", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79937536\nfile 7598080\nkernel 1028096\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87494656\ninactive_file 32768\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 392328\nslab 498120\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95737\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2209175\nuser_usec 2025323\nsystem_usec 183852\ncore_sched.force_idle_usec 0\nnr_periods 26\nnr_throttled 16\nthrottled_usec 696184\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.383, + "worker": { + "memory.current": "115290112", + "memory.peak": "127328256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 106123264\nfile 7610368\nkernel 1122304\nkernel_stack 49152\npagetables 507904\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 113680384\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 419272\nslab 525064\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 107250\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2388567\nuser_usec 2192880\nsystem_usec 195687\ncore_sched.force_idle_usec 0\nnr_periods 28\nnr_throttled 18\nthrottled_usec 798513\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.5736, + "worker": { + "memory.current": "78794752", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66629632\nfile 8372224\nkernel 827392\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 2797568\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74973184\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 109352\nslab_unreclaimable 312008\nslab 421360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 113800\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2580929\nuser_usec 2375202\nsystem_usec 205726\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 20\nthrottled_usec 903938\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.7743, + "worker": { + "memory.current": "88862720", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79921152\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87429120\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 374264\nslab 480056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 120326\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2743632\nuser_usec 2528947\nsystem_usec 214684\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 21\nthrottled_usec 991261\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.9742, + "worker": { + "memory.current": "117268480", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 108314624\nfile 7610368\nkernel 1134592\nkernel_stack 49152\npagetables 520192\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 115888128\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 418352\nslab 525336\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 132381\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2940734\nuser_usec 2702402\nsystem_usec 238331\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 23\nthrottled_usec 1088687\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.1634, + "worker": { + "memory.current": "76079104", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66633728\nfile 8372224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 4096\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 74977280\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 311360\nslab 417176\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 138400\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3117143\nuser_usec 2867732\nsystem_usec 249410\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 25\nthrottled_usec 1141971\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.3522, + "worker": { + "memory.current": "90132480", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81235968\nfile 7593984\nkernel 1028096\nkernel_stack 49152\npagetables 442368\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88784896\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379552\nslab 485344\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 145311\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3299749\nuser_usec 3039554\nsystem_usec 260195\ncore_sched.force_idle_usec 0\nnr_periods 37\nnr_throttled 25\nthrottled_usec 1141971\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.5539, + "worker": { + "memory.current": "126955520", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 117960704\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 532480\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 125509632\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 414424\nslab 521408\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 159342\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3517509\nuser_usec 3247462\nsystem_usec 270046\ncore_sched.force_idle_usec 0\nnr_periods 40\nnr_throttled 28\nthrottled_usec 1316301\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.7603, + "worker": { + "memory.current": "72503296", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66854912\nfile 3964928\nkernel 806912\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 1036288\nvmalloc 16384\nshmem 3932160\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 70770688\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 94464\nslab_unreclaimable 310936\nslab 405400\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 163062\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3674848\nuser_usec 3388522\nsystem_usec 286326\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 28\nthrottled_usec 1316301\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.9159, + "worker": { + "memory.current": "101822464", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 92762112\nfile 7593984\nkernel 1052672\nkernel_stack 49152\npagetables 466944\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 100274176\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 386520\nslab 492312\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 172742\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3863955\nuser_usec 3568778\nsystem_usec 295176\ncore_sched.force_idle_usec 0\nnr_periods 43\nnr_throttled 30\nthrottled_usec 1379485\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.0905, + "worker": { + "memory.current": "122384384", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 113397760\nfile 7610368\nkernel 1150976\nkernel_stack 49152\npagetables 528384\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120975360\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 418992\nslab 525976\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 182834\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4035822\nuser_usec 3727955\nsystem_usec 307867\ncore_sched.force_idle_usec 0\nnr_periods 45\nnr_throttled 32\nthrottled_usec 1480906\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.2686, + "worker": { + "memory.current": "76140544", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66908160\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75251712\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 313240\nslab 419056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 187677\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4195036\nuser_usec 3874239\nsystem_usec 320796\ncore_sched.force_idle_usec 0\nnr_periods 46\nnr_throttled 33\nthrottled_usec 1495617\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.4545, + "worker": { + "memory.current": "90464256", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 81584128\nfile 7593984\nkernel 1019904\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 89149440\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 383696\nslab 489488\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 194607\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4378271\nuser_usec 4044745\nsystem_usec 333526\ncore_sched.force_idle_usec 0\nnr_periods 48\nnr_throttled 34\nthrottled_usec 1549546\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.659, + "worker": { + "memory.current": "125095936", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 116039680\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 528384\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 123584512\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 419808\nslab 526792\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 208081\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4587405\nuser_usec 4242955\nsystem_usec 344450\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 37\nthrottled_usec 1749887\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.856, + "worker": { + "memory.current": "76509184", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67264512\nfile 8372224\nkernel 819200\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75608064\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 314056\nslab 419872\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 212364\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4735651\nuser_usec 4383984\nsystem_usec 351667\ncore_sched.force_idle_usec 0\nnr_periods 52\nnr_throttled 38\nthrottled_usec 1754778\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.0193, + "worker": { + "memory.current": "76652544", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67260416\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75603968\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 314056\nslab 419872\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 215245\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4796252\nuser_usec 4441669\nsystem_usec 354582\ncore_sched.force_idle_usec 0\nnr_periods 54\nnr_throttled 38\nthrottled_usec 1754778\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.1811, + "worker": { + "memory.current": "89493504", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80687104\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88195072\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 381256\nslab 487048\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 221804\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4957854\nuser_usec 4593277\nsystem_usec 364577\ncore_sched.force_idle_usec 0\nnr_periods 56\nnr_throttled 39\nthrottled_usec 1829478\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.3667, + "worker": { + "memory.current": "118333440", + "memory.peak": "128864256", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 109322240\nfile 7610368\nkernel 1134592\nkernel_stack 49152\npagetables 516096\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 116891648\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 420624\nslab 527608\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 233922\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5153684\nuser_usec 4778296\nsystem_usec 375388\ncore_sched.force_idle_usec 0\nnr_periods 58\nnr_throttled 41\nthrottled_usec 1938995\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.5561, + "worker": { + "memory.current": "76836864", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67391488\nfile 8372224\nkernel 819200\nkernel_stack 32768\npagetables 323584\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75735040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 110416\nslab_unreclaimable 314912\nslab 425328\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 240645\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5335204\nuser_usec 4950658\nsystem_usec 384545\ncore_sched.force_idle_usec 0\nnr_periods 59\nnr_throttled 42\nthrottled_usec 1950630\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.7535, + "worker": { + "memory.current": "89817088", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 80695296\nfile 7593984\nkernel 1015808\nkernel_stack 49152\npagetables 438272\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88252416\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105792\nslab_unreclaimable 379344\nslab 485136\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 247172\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5502620\nuser_usec 5107103\nsystem_usec 395516\ncore_sched.force_idle_usec 0\nnr_periods 62\nnr_throttled 44\nthrottled_usec 2082147\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.9541, + "worker": { + "memory.current": "121753600", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 112652288\nfile 7610368\nkernel 1138688\nkernel_stack 49152\npagetables 520192\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120221696\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 106984\nslab_unreclaimable 421440\nslab 528424\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 260098\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5706772\nuser_usec 5302254\nsystem_usec 404518\ncore_sched.force_idle_usec 0\nnr_periods 64\nnr_throttled 46\nthrottled_usec 2189973\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.1564, + "worker": { + "memory.current": "76931072", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67411968\nfile 8372224\nkernel 827392\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75735040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 315688\nslab 421504\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 265246\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5876598\nuser_usec 5464049\nsystem_usec 412549\ncore_sched.force_idle_usec 0\nnr_periods 65\nnr_throttled 47\nthrottled_usec 2198351\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.3131, + "worker": { + "memory.current": "101044224", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 92147712\nfile 7593984\nkernel 1052672\nkernel_stack 49152\npagetables 462848\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7565312\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 99667968\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105584\nslab_unreclaimable 389784\nslab 495368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 274641\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6068527\nuser_usec 5641347\nsystem_usec 427180\ncore_sched.force_idle_usec 0\nnr_periods 67\nnr_throttled 49\nthrottled_usec 2248335\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.4887, + "worker": { + "memory.current": "122351616", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 113147904\nfile 7610368\nkernel 1146880\nkernel_stack 49152\npagetables 524288\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 7581696\nzswap 0\nzswapped 0\nfile_mapped 12288\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 120672256\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 107184\nslab_unreclaimable 424736\nslab 531920\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 284818\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6244347\nuser_usec 5814145\nsystem_usec 430201\ncore_sched.force_idle_usec 0\nnr_periods 69\nnr_throttled 51\nthrottled_usec 2354429\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.6681, + "worker": { + "memory.current": "77053952", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67534848\nfile 8372224\nkernel 823296\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 8343552\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75878400\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 105816\nslab_unreclaimable 316504\nslab 422320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 289874\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6411394\nuser_usec 5976052\nsystem_usec 435341\ncore_sched.force_idle_usec 0\nnr_periods 71\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.8605, + "worker": { + "memory.current": "68456448", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67538944\nfile 28672\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67538944\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311088\nslab 394144\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 292753\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6471910\nuser_usec 6034578\nsystem_usec 437331\ncore_sched.force_idle_usec 0\nnr_periods 72\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.0149, + "worker": { + "memory.current": "68554752", + "memory.peak": "129454080", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 67543040\nfile 28672\nkernel 794624\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67543040\ninactive_file 28672\nactive_file 0\nunevictable 0\nslab_reclaimable 83056\nslab_unreclaimable 311312\nslab 394368\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 295630\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6528449\nuser_usec 6089129\nsystem_usec 439319\ncore_sched.force_idle_usec 0\nnr_periods 74\nnr_throttled 53\nthrottled_usec 2419781\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 1.0932762500000006, + "ready_seconds": 6.383178083, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.744891333, + "ready_seconds": 2.9478767920000006, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.7652272499999997, + "ready_seconds": 3.889252625, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 1.0959794999999994, + "ready_seconds": 5.7605763340000005, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.747986042, + "ready_seconds": 2.326131459, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.978114916, + "ready_seconds": 4.420152333000001, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 1.102462333, + "ready_seconds": 7.024895291999999, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.6511005000000001, + "ready_seconds": 1.919602084, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 1.0380668750000002, + "ready_seconds": 5.422940084, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.7656660830000002, + "ready_seconds": 3.5762043749999997, + "status": "ready" + } + ], + "seconds": 7.2014601659999995 + }, + { + "width": 10000, + "height": 4000, + "bytes": 533859, + "clients": 10, + "before": { + "memory.current": "26193920", + "memory.peak": "26193920", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 25034752\nfile 126976\nkernel 516096\nkernel_stack 16384\npagetables 184320\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 25026560\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 79544\nslab_unreclaimable 196456\nslab 276000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 9312\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 213801\nuser_usec 195818\nsystem_usec 17983\ncore_sched.force_idle_usec 0\nnr_periods 2\nnr_throttled 1\nthrottled_usec 47256\nnr_bursts 0\nburst_usec 0" + }, + "after": { + "memory.current": "67629056", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 139264\nkernel 794624\nkernel_stack 32768\npagetables 331776\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 83976\nslab_unreclaimable 310864\nslab 394840\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 698354\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11899130\nuser_usec 11309189\nsystem_usec 589941\ncore_sched.force_idle_usec 0\nnr_periods 120\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + }, + "samples": [ + { + "seconds": 0.0001, + "worker": { + "memory.current": "30728192", + "memory.peak": "30875648", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 29769728\nfile 126976\nkernel 552960\nkernel_stack 16384\npagetables 204800\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 29741056\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 82936\nslab_unreclaimable 203176\nslab 286112\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 13510\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 368020\nuser_usec 346137\nsystem_usec 21882\ncore_sched.force_idle_usec 0\nnr_periods 4\nnr_throttled 3\nthrottled_usec 209770\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.2346, + "worker": { + "memory.current": "48369664", + "memory.peak": "48517120", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 47390720\nfile 126976\nkernel 700416\nkernel_stack 16384\npagetables 278528\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 47357952\ninactive_file 126976\nactive_file 0\nunevictable 0\nslab_reclaimable 82936\nslab_unreclaimable 280032\nslab 362968\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 22319\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 568126\nuser_usec 531312\nsystem_usec 36813\ncore_sched.force_idle_usec 0\nnr_periods 6\nnr_throttled 5\nthrottled_usec 326949\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.4368, + "worker": { + "memory.current": "78430208", + "memory.peak": "78430208", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 76587008\nfile 675840\nkernel 954368\nkernel_stack 49152\npagetables 421888\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 77107200\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 343600\nslab 434920\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 34179\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 765407\nuser_usec 704850\nsystem_usec 60557\ncore_sched.force_idle_usec 0\nnr_periods 8\nnr_throttled 7\nthrottled_usec 430066\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.6374, + "worker": { + "memory.current": "242069504", + "memory.peak": "242216960", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 239763456\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 240287744\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380000\nslab 471320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 77128\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 967740\nuser_usec 874248\nsystem_usec 93491\ncore_sched.force_idle_usec 0\nnr_periods 10\nnr_throttled 9\nthrottled_usec 535278\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 0.8379, + "worker": { + "memory.current": "244908032", + "memory.peak": "245055488", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242790400\nfile 675840\nkernel 1327104\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243273728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380728\nslab 472048\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 80744\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1171239\nuser_usec 1072807\nsystem_usec 98431\ncore_sched.force_idle_usec 0\nnr_periods 12\nnr_throttled 11\nthrottled_usec 642609\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.0389, + "worker": { + "memory.current": "244006912", + "memory.peak": "245055488", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241872896\nfile 684032\nkernel 1331200\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242372608\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 380536\nslab 472056\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 84399\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1382200\nuser_usec 1280845\nsystem_usec 101354\ncore_sched.force_idle_usec 0\nnr_periods 14\nnr_throttled 13\nthrottled_usec 734464\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.2484, + "worker": { + "memory.current": "258646016", + "memory.peak": "258793472", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 256446464\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 798720\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 256978944\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 382760\nslab 474480\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 90841\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1594891\nuser_usec 1486645\nsystem_usec 108246\ncore_sched.force_idle_usec 0\nnr_periods 16\nnr_throttled 15\nthrottled_usec 831199\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.4458, + "worker": { + "memory.current": "89042944", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 86626304\nfile 688128\nkernel 1400832\nkernel_stack 49152\npagetables 487424\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87175168\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 415152\nslab 507072\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 95495\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1786454\nuser_usec 1663250\nsystem_usec 123203\ncore_sched.force_idle_usec 0\nnr_periods 18\nnr_throttled 17\nthrottled_usec 944761\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.6549, + "worker": { + "memory.current": "80568320", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 78856192\nfile 675840\nkernel 991232\nkernel_stack 49152\npagetables 430080\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79380480\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 377848\nslab 469168\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 102100\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 1991706\nuser_usec 1855274\nsystem_usec 136431\ncore_sched.force_idle_usec 0\nnr_periods 20\nnr_throttled 18\nthrottled_usec 1023149\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 1.8447, + "worker": { + "memory.current": "243773440", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241299456\nfile 675840\nkernel 1314816\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241807360\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 380984\nslab 472304\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 144711\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2229110\nuser_usec 2063622\nsystem_usec 165488\ncore_sched.force_idle_usec 0\nnr_periods 22\nnr_throttled 20\nthrottled_usec 1115720\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.0953, + "worker": { + "memory.current": "245940224", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243769344\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244281344\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 377432\nslab 468752\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 148181\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2414292\nuser_usec 2245251\nsystem_usec 169040\ncore_sched.force_idle_usec 0\nnr_periods 24\nnr_throttled 22\nthrottled_usec 1144849\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.2575, + "worker": { + "memory.current": "244084736", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241799168\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242319360\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 382776\nslab 474296\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 151579\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2575986\nuser_usec 2403856\nsystem_usec 172130\ncore_sched.force_idle_usec 0\nnr_periods 26\nnr_throttled 24\nthrottled_usec 1263685\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.4381, + "worker": { + "memory.current": "256630784", + "memory.peak": "264101888", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 254418944\nfile 684032\nkernel 1351680\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 254914560\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383016\nslab 474736\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 157542\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2764644\nuser_usec 2590486\nsystem_usec 174158\ncore_sched.force_idle_usec 0\nnr_periods 28\nnr_throttled 26\nthrottled_usec 1372197\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.6419, + "worker": { + "memory.current": "89264128", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 86876160\nfile 688128\nkernel 1388544\nkernel_stack 49152\npagetables 475136\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 87425024\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 416400\nslab 508320\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 162746\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 2967367\nuser_usec 2782342\nsystem_usec 185025\ncore_sched.force_idle_usec 0\nnr_periods 30\nnr_throttled 28\nthrottled_usec 1471876\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 2.8367, + "worker": { + "memory.current": "81100800", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79085568\nfile 675840\nkernel 999424\nkernel_stack 49152\npagetables 434176\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79597568\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 378888\nslab 470208\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 169350\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3155688\nuser_usec 2960634\nsystem_usec 195053\ncore_sched.force_idle_usec 0\nnr_periods 32\nnr_throttled 29\nthrottled_usec 1566552\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.036, + "worker": { + "memory.current": "243621888", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241451008\nfile 675840\nkernel 1318912\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241938432\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 381800\nslab 473120\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 211934\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3362147\nuser_usec 3145362\nsystem_usec 216784\ncore_sched.force_idle_usec 0\nnr_periods 34\nnr_throttled 31\nthrottled_usec 1673525\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.2337, + "worker": { + "memory.current": "246566912", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 244281344\nfile 675840\nkernel 1318912\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244797440\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 382792\nslab 474112\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 215505\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3562352\nuser_usec 3341631\nsystem_usec 220720\ncore_sched.force_idle_usec 0\nnr_periods 36\nnr_throttled 33\nthrottled_usec 1777266\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.4408, + "worker": { + "memory.current": "250142720", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247869440\nfile 684032\nkernel 1335296\nkernel_stack 49152\npagetables 770048\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 248414208\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383832\nslab 475552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 220266\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3767453\nuser_usec 3539875\nsystem_usec 227577\ncore_sched.force_idle_usec 0\nnr_periods 38\nnr_throttled 35\nthrottled_usec 1869788\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.6462, + "worker": { + "memory.current": "262807552", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 260542464\nfile 684032\nkernel 1355776\nkernel_stack 49152\npagetables 790528\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 261038080\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 383832\nslab 475552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 226232\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 3973493\nuser_usec 3743909\nsystem_usec 229584\ncore_sched.force_idle_usec 0\nnr_periods 40\nnr_throttled 37\nthrottled_usec 1959331\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.8385, + "worker": { + "memory.current": "76562432", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74629120\nfile 675840\nkernel 942080\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75153408\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 341424\nslab 432744\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 232394\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4137903\nuser_usec 3897014\nsystem_usec 240888\ncore_sched.force_idle_usec 0\nnr_periods 41\nnr_throttled 38\nthrottled_usec 1976863\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 3.9948, + "worker": { + "memory.current": "162709504", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 160632832\nfile 675840\nkernel 1200128\nkernel_stack 49152\npagetables 630784\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 161120256\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383640\nslab 474960\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 256512\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4322051\nuser_usec 4069334\nsystem_usec 252716\ncore_sched.force_idle_usec 0\nnr_periods 43\nnr_throttled 40\nthrottled_usec 2044525\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.1698, + "worker": { + "memory.current": "244023296", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241827840\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242343936\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383872\nslab 475192\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 279212\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4494084\nuser_usec 4231481\nsystem_usec 262602\ncore_sched.force_idle_usec 0\nnr_periods 45\nnr_throttled 42\nthrottled_usec 2146593\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.3559, + "worker": { + "memory.current": "242335744", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 240181248\nfile 679936\nkernel 1331200\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 240713728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 379328\nslab 470648\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 282693\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4684066\nuser_usec 4417542\nsystem_usec 266524\ncore_sched.force_idle_usec 0\nnr_periods 47\nnr_throttled 44\nthrottled_usec 2245102\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.5446, + "worker": { + "memory.current": "249327616", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247123968\nfile 684032\nkernel 1339392\nkernel_stack 49152\npagetables 774144\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 247652352\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 384896\nslab 476616\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 287265\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 4871121\nuser_usec 4594759\nsystem_usec 276361\ncore_sched.force_idle_usec 0\nnr_periods 49\nnr_throttled 46\nthrottled_usec 2354663\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.744, + "worker": { + "memory.current": "261533696", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 259211264\nfile 684032\nkernel 1368064\nkernel_stack 49152\npagetables 798720\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 259756032\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 385640\nslab 477360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 293088\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5068078\nuser_usec 4786849\nsystem_usec 281228\ncore_sched.force_idle_usec 0\nnr_periods 51\nnr_throttled 48\nthrottled_usec 2457492\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 4.9437, + "worker": { + "memory.current": "76525568", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 74838016\nfile 679936\nkernel 942080\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75317248\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 90272\nslab_unreclaimable 341192\nslab 431464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 299629\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5246669\nuser_usec 4951600\nsystem_usec 295069\ncore_sched.force_idle_usec 0\nnr_periods 53\nnr_throttled 50\nthrottled_usec 2542069\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.13, + "worker": { + "memory.current": "214814720", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 212742144\nfile 675840\nkernel 1282048\nkernel_stack 49152\npagetables 712704\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 213237760\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383696\nslab 475016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 336420\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5451473\nuser_usec 5142543\nsystem_usec 308930\ncore_sched.force_idle_usec 0\nnr_periods 55\nnr_throttled 52\nthrottled_usec 2659554\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.3342, + "worker": { + "memory.current": "245424128", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243212288\nfile 675840\nkernel 1327104\nkernel_stack 49152\npagetables 761856\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243716096\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 383432\nslab 474752\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 346733\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5660402\nuser_usec 5346487\nsystem_usec 313914\ncore_sched.force_idle_usec 0\nnr_periods 57\nnr_throttled 54\nthrottled_usec 2758211\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.5383, + "worker": { + "memory.current": "244420608", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242167808\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242708480\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 94280\nslab_unreclaimable 384272\nslab 478552\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 351058\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 5884173\nuser_usec 5564607\nsystem_usec 319565\ncore_sched.force_idle_usec 0\nnr_periods 59\nnr_throttled 56\nthrottled_usec 2849581\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.7466, + "worker": { + "memory.current": "254889984", + "memory.peak": "264249344", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 252788736\nfile 684032\nkernel 1351680\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 253308928\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386456\nslab 478176\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 356529\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6063524\nuser_usec 5740057\nsystem_usec 323467\ncore_sched.force_idle_usec 0\nnr_periods 61\nnr_throttled 58\nthrottled_usec 2956295\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 5.9308, + "worker": { + "memory.current": "249012224", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 246657024\nfile 684032\nkernel 1400832\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 247148544\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 397936\nslab 489656\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 362024\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6258018\nuser_usec 5927656\nsystem_usec 330361\ncore_sched.force_idle_usec 0\nnr_periods 63\nnr_throttled 60\nthrottled_usec 3075398\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.1266, + "worker": { + "memory.current": "78884864", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 76996608\nfile 675840\nkernel 946176\nkernel_stack 49152\npagetables 413696\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 77520896\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 352096\nslab 443416\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 368045\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6434616\nuser_usec 6088381\nsystem_usec 346235\ncore_sched.force_idle_usec 0\nnr_periods 64\nnr_throttled 61\nthrottled_usec 3077896\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.2725, + "worker": { + "memory.current": "181219328", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 179220480\nfile 675840\nkernel 1220608\nkernel_stack 49152\npagetables 655360\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 179744768\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384512\nslab 475832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 396110\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6597949\nuser_usec 6238945\nsystem_usec 359004\ncore_sched.force_idle_usec 0\nnr_periods 66\nnr_throttled 63\nthrottled_usec 3188604\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.4543, + "worker": { + "memory.current": "244588544", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242282496\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242774016\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384248\nslab 475568\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 414389\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6774241\nuser_usec 6405332\nsystem_usec 368909\ncore_sched.force_idle_usec 0\nnr_periods 68\nnr_throttled 65\nthrottled_usec 3290034\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.6434, + "worker": { + "memory.current": "243073024", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 240910336\nfile 679936\nkernel 1327104\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241434624\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 384248\nslab 475568\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 417932\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 6974922\nuser_usec 6603072\nsystem_usec 371850\ncore_sched.force_idle_usec 0\nnr_periods 70\nnr_throttled 67\nthrottled_usec 3392666\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 6.8408, + "worker": { + "memory.current": "251777024", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 249483264\nfile 684032\nkernel 1347584\nkernel_stack 49152\npagetables 778240\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 249974784\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386280\nslab 478000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 422902\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7163878\nuser_usec 6786101\nsystem_usec 377776\ncore_sched.force_idle_usec 0\nnr_periods 72\nnr_throttled 69\nthrottled_usec 3500325\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.0391, + "worker": { + "memory.current": "263426048", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 261042176\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 261578752\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 386280\nslab 478000\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 428597\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7359035\nuser_usec 6974274\nsystem_usec 384760\ncore_sched.force_idle_usec 0\nnr_periods 74\nnr_throttled 71\nthrottled_usec 3600855\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.2355, + "worker": { + "memory.current": "77447168", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 75452416\nfile 679936\nkernel 933888\nkernel_stack 49152\npagetables 405504\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 75964416\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 348040\nslab 439360\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 434835\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7542243\nuser_usec 7144032\nsystem_usec 398211\ncore_sched.force_idle_usec 0\nnr_periods 76\nnr_throttled 72\nthrottled_usec 3639550\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.4362, + "worker": { + "memory.current": "236023808", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 233701376\nfile 675840\nkernel 1302528\nkernel_stack 49152\npagetables 737280\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 234217472\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 385328\nslab 476648\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 476596\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7745487\nuser_usec 7326518\nsystem_usec 418969\ncore_sched.force_idle_usec 0\nnr_periods 78\nnr_throttled 74\nthrottled_usec 3743288\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.64, + "worker": { + "memory.current": "245723136", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243539968\nfile 675840\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244039680\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386056\nslab 477376\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 481879\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 7946846\nuser_usec 7523846\nsystem_usec 422999\ncore_sched.force_idle_usec 0\nnr_periods 80\nnr_throttled 76\nthrottled_usec 3844224\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 7.8336, + "worker": { + "memory.current": "244654080", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242450432\nfile 679936\nkernel 1323008\nkernel_stack 49152\npagetables 753664\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242978816\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 385864\nslab 477384\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 485496\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8146758\nuser_usec 7717825\nsystem_usec 428933\ncore_sched.force_idle_usec 0\nnr_periods 82\nnr_throttled 78\nthrottled_usec 3951186\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.0329, + "worker": { + "memory.current": "257679360", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 255352832\nfile 684032\nkernel 1355776\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 255885312\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 387096\nslab 478816\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 491523\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8340638\nuser_usec 7906612\nsystem_usec 434026\ncore_sched.force_idle_usec 0\nnr_periods 84\nnr_throttled 80\nthrottled_usec 4061201\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.2365, + "worker": { + "memory.current": "90140672", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 87625728\nfile 688128\nkernel 1331200\nkernel_stack 49152\npagetables 475136\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 88174592\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91920\nslab_unreclaimable 419544\nslab 511464\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 496681\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8537699\nuser_usec 8092573\nsystem_usec 445126\ncore_sched.force_idle_usec 0\nnr_periods 86\nnr_throttled 82\nthrottled_usec 4168912\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.4325, + "worker": { + "memory.current": "84443136", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 82571264\nfile 675840\nkernel 1060864\nkernel_stack 49152\npagetables 487424\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 83050496\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 385464\nslab 476784\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 504041\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8730749\nuser_usec 8276353\nsystem_usec 454395\ncore_sched.force_idle_usec 0\nnr_periods 88\nnr_throttled 83\nthrottled_usec 4259590\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.6327, + "worker": { + "memory.current": "244965376", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242761728\nfile 675840\nkernel 1343488\nkernel_stack 49152\npagetables 774144\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243290112\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386704\nslab 478024\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 546031\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 8946723\nuser_usec 8476417\nsystem_usec 470305\ncore_sched.force_idle_usec 0\nnr_periods 90\nnr_throttled 85\nthrottled_usec 4359771\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 8.8408, + "worker": { + "memory.current": "243564544", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241434624\nfile 679936\nkernel 1339392\nkernel_stack 49152\npagetables 770048\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241967104\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 387672\nslab 479192\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 549584\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9148347\nuser_usec 8671974\nsystem_usec 476373\ncore_sched.force_idle_usec 0\nnr_periods 92\nnr_throttled 87\nthrottled_usec 4464704\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.0402, + "worker": { + "memory.current": "253980672", + "memory.peak": "265674752", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 251596800\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 252100608\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 387912\nslab 479632\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 554937\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9352751\nuser_usec 8874345\nsystem_usec 478406\ncore_sched.force_idle_usec 0\nnr_periods 94\nnr_throttled 89\nthrottled_usec 4563624\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.2355, + "worker": { + "memory.current": "265728000", + "memory.peak": "265875456", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 263569408\nfile 684032\nkernel 1380352\nkernel_stack 49152\npagetables 806912\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 264114176\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 388904\nslab 480624\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 560732\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9548786\nuser_usec 9064499\nsystem_usec 484286\ncore_sched.force_idle_usec 0\nnr_periods 96\nnr_throttled 91\nthrottled_usec 4670267\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.4323, + "worker": { + "memory.current": "81321984", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 79179776\nfile 675840\nkernel 958464\nkernel_stack 49152\npagetables 421888\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 79708160\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 352808\nslab 444128\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 567281\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9734279\nuser_usec 9240914\nsystem_usec 493364\ncore_sched.force_idle_usec 0\nnr_periods 98\nnr_throttled 92\nthrottled_usec 4766580\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.6306, + "worker": { + "memory.current": "243724288", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241229824\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 241737728\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386696\nslab 478016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 609959\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 9927515\nuser_usec 9413373\nsystem_usec 514142\ncore_sched.force_idle_usec 0\nnr_periods 100\nnr_throttled 94\nthrottled_usec 4872153\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 9.833, + "worker": { + "memory.current": "246476800", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 244035584\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 244535296\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 386696\nslab 478016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 613516\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10123398\nuser_usec 9606209\nsystem_usec 517188\ncore_sched.force_idle_usec 0\nnr_periods 102\nnr_throttled 96\nthrottled_usec 4980716\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.0328, + "worker": { + "memory.current": "245137408", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 242970624\nfile 684032\nkernel 1339392\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243474432\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 387496\nslab 479016\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 617134\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10324726\nuser_usec 9804569\nsystem_usec 520157\ncore_sched.force_idle_usec 0\nnr_periods 104\nnr_throttled 98\nthrottled_usec 5087414\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.2347, + "worker": { + "memory.current": "259846144", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 257601536\nfile 684032\nkernel 1363968\nkernel_stack 49152\npagetables 794624\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 258109440\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 388728\nslab 480448\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 623583\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10526147\nuser_usec 10001031\nsystem_usec 525116\ncore_sched.force_idle_usec 0\nnr_periods 106\nnr_throttled 100\nthrottled_usec 5192242\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.4306, + "worker": { + "memory.current": "68222976", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66523136\nfile 692224\nkernel 815104\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 8192\nvmalloc 16384\nshmem 548864\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 67072000\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 89424\nslab_unreclaimable 316848\nslab 406272\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 628260\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10721910\nuser_usec 10185814\nsystem_usec 536095\ncore_sched.force_idle_usec 0\nnr_periods 108\nnr_throttled 102\nthrottled_usec 5294934\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.6262, + "worker": { + "memory.current": "126590976", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 124612608\nfile 675840\nkernel 1126400\nkernel_stack 49152\npagetables 557056\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 125140992\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 387528\nslab 478848\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 645792\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 10917765\nuser_usec 10373713\nsystem_usec 544051\ncore_sched.force_idle_usec 0\nnr_periods 110\nnr_throttled 104\nthrottled_usec 5399951\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 10.8329, + "worker": { + "memory.current": "245424128", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 243052544\nfile 675840\nkernel 1335296\nkernel_stack 49152\npagetables 765952\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 536576\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 243552256\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91320\nslab_unreclaimable 387512\nslab 478832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 677586\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11124368\nuser_usec 10561448\nsystem_usec 562919\ncore_sched.force_idle_usec 0\nnr_periods 112\nnr_throttled 106\nthrottled_usec 5505169\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.0327, + "worker": { + "memory.current": "244043776", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 241819648\nfile 679936\nkernel 1327104\nkernel_stack 49152\npagetables 757760\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 540672\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 242327552\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91520\nslab_unreclaimable 388312\nslab 479832\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 681168\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11327218\nuser_usec 10761303\nsystem_usec 565914\ncore_sched.force_idle_usec 0\nnr_periods 114\nnr_throttled 108\nthrottled_usec 5611110\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.2325, + "worker": { + "memory.current": "254652416", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 252379136\nfile 684032\nkernel 1359872\nkernel_stack 49152\npagetables 786432\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 252887040\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 389544\nslab 481264\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 686623\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11527701\nuser_usec 10958812\nsystem_usec 568889\ncore_sched.force_idle_usec 0\nnr_periods 116\nnr_throttled 110\nthrottled_usec 5715900\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.438, + "worker": { + "memory.current": "249868288", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 247463936\nfile 684032\nkernel 1376256\nkernel_stack 49152\npagetables 782336\nsec_pagetables 0\npercpu 2544\nsock 0\nvmalloc 16384\nshmem 544768\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 248000512\ninactive_file 139264\nactive_file 0\nunevictable 0\nslab_reclaimable 91720\nslab_unreclaimable 402504\nslab 494224\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 692425\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11732200\nuser_usec 11153383\nsystem_usec 578816\ncore_sched.force_idle_usec 0\nnr_periods 118\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + } + }, + { + "seconds": 11.6408, + "worker": { + "memory.current": "67588096", + "memory.peak": "265990144", + "memory.events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "memory.stat": "anon 66650112\nfile 143360\nkernel 790528\nkernel_stack 32768\npagetables 327680\nsec_pagetables 0\npercpu 2448\nsock 0\nvmalloc 16384\nshmem 0\nzswap 0\nzswapped 0\nfile_mapped 0\nfile_dirty 0\nfile_writeback 0\nswapcached 0\nanon_thp 0\nfile_thp 0\nshmem_thp 0\ninactive_anon 0\nactive_anon 66650112\ninactive_file 143360\nactive_file 0\nunevictable 0\nslab_reclaimable 83976\nslab_unreclaimable 310864\nslab 394840\nworkingset_refault_anon 0\nworkingset_refault_file 0\nworkingset_activate_anon 0\nworkingset_activate_file 0\nworkingset_restore_anon 0\nworkingset_restore_file 0\nworkingset_nodereclaim 0\npgscan 0\npgsteal 0\npgscan_kswapd 0\npgscan_direct 0\npgscan_khugepaged 0\npgsteal_kswapd 0\npgsteal_direct 0\npgsteal_khugepaged 0\npgfault 695477\npgmajfault 0\npgrefill 0\npgactivate 0\npgdeactivate 0\npglazyfree 0\npglazyfreed 0\nzswpin 0\nzswpout 0\nzswpwb 0\nthp_fault_alloc 0\nthp_collapse_alloc 0\nthp_swpout 0\nthp_swpout_fallback 0", + "cpu.stat": "usage_usec 11840293\nuser_usec 11252399\nsystem_usec 587893\ncore_sched.force_idle_usec 0\nnr_periods 119\nnr_throttled 112\nthrottled_usec 5818863\nnr_bursts 0\nburst_usec 0" + } + } + ], + "replies": [ + { + "index": 0, + "accepted_seconds": 0.06314554199999911, + "ready_seconds": 5.1136705419999995, + "status": "ready" + }, + { + "index": 1, + "accepted_seconds": 0.07206512499999995, + "ready_seconds": 6.358601541999999, + "status": "ready" + }, + { + "index": 2, + "accepted_seconds": 0.07929308300000137, + "ready_seconds": 10.723366916000002, + "status": "ready" + }, + { + "index": 3, + "accepted_seconds": 0.0655274170000002, + "ready_seconds": 1.6717298339999989, + "status": "ready" + }, + { + "index": 4, + "accepted_seconds": 0.06864495800000014, + "ready_seconds": 7.294111999999998, + "status": "ready" + }, + { + "index": 5, + "accepted_seconds": 0.09219408299999898, + "ready_seconds": 11.659252915999998, + "status": "ready" + }, + { + "index": 6, + "accepted_seconds": 0.053762415999999646, + "ready_seconds": 4.160090082999998, + "status": "ready" + }, + { + "index": 7, + "accepted_seconds": 0.05835524999999997, + "ready_seconds": 9.452590790999999, + "status": "ready" + }, + { + "index": 8, + "accepted_seconds": 0.07616920800000138, + "ready_seconds": 8.538444833000002, + "status": "ready" + }, + { + "index": 9, + "accepted_seconds": 0.06853324999999977, + "ready_seconds": 2.9269747909999992, + "status": "ready" + } + ], + "seconds": 11.840416583 + } + ] +} diff --git a/evidence/restart-after.json b/evidence/restart-after.json new file mode 100644 index 00000000..40aba5e4 --- /dev/null +++ b/evidence/restart-after.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475559008Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.552220082Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.074661473Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:01.993890605Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.087804668Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:06:02.475268543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/restart-before.json b/evidence/restart-before.json new file mode 100644 index 00000000..37b6583d --- /dev/null +++ b/evidence/restart-before.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:46.437791821Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:03:32.37188016Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:03:41.154830662Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:03:24.364562897Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:23.350954361Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T14:56:14.79412711Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:03:23.874882274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.375415684Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.375672543Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "b8d84d132c21a7d1d44afe270bce5beef0c9f2ecf28e4fa9e88efe83c04660d3", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:05.95436716Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T14:56:01.374203439Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:05:23.351252982Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + } + ] + } +] diff --git a/evidence/runtime.json b/evidence/runtime.json new file mode 100644 index 00000000..016174fd --- /dev/null +++ b/evidence/runtime.json @@ -0,0 +1,373 @@ +[ + { + "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", + "name": "/mtusi-lab5-api-1", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.608120136Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "e2f2b66d9b090b1b7f131e218514621b3e76cc35a1d93e39faacd01d0a609df3", + "name": "/mtusi-lab5-api-2", + "service": "api", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:07:57.67064591Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + } + ] + }, + { + "container": "13917ee79bcae97792ff30f5bedba828db9b14c50717c4234a201a2ecfd31c97", + "name": "/mtusi-lab5-broker-1", + "service": "broker", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:08:06.296755178Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 201326592, + "nano_cpus": 500000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_broker_data" + } + ] + }, + { + "container": "468675e8e8d12872e5ea7cc119ab039c32b512ca21c468c1df036f4b73e3cf86", + "name": "/mtusi-lab5-bucket-bootstrap-1", + "service": "bucket-bootstrap", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.893110991Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_admin_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_admin_secret", + "name": null + } + ] + }, + { + "container": "6575f9ad27a284222d9b0766ff1be89f488896c84734d1c5743750707b8bae30", + "name": "/mtusi-lab5-dispatcher-1", + "service": "dispatcher", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679298033Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "container": "f0aa92b6b1dba892cab16f26d319c1950477ec0070bd7b873b7e9b748cb9b86d", + "name": "/mtusi-lab5-lb-1", + "service": "lb", + "image": "haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949", + "user": "haproxy", + "running": true, + "started_at": "2026-10-27T15:06:08.145248456Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18091" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "container": "ccc5e4696faed219bd5ba8c93d1aa014be96aa67a7261aded25b1d1a859076d4", + "name": "/mtusi-lab5-migrate-1", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": false, + "started_at": "2026-10-27T15:06:01.411588564Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nano_cpus": 1000000000, + "ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_api_secret", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_api_key", + "name": null + } + ] + }, + { + "container": "b8c6c2aa716d884daad1773dec95f7e401ec39be531423a88b87b173c86e67e3", + "name": "/mtusi-lab5-postgres-1", + "service": "postgres", + "image": "postgres:17.2-bookworm@sha256:3267c505060a0052e5aa6e5175a7b41ab6b04da2f8c4540fc6e98a37210aa2d3", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.049832274Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "5432/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "mtusi-lab5_postgres_data" + } + ] + }, + { + "container": "9cc0501477a884cf8a8640e7c1bf586044408032f540ead3a79ada1a37b3f010", + "name": "/mtusi-lab5-redis-1", + "service": "redis", + "image": "redis:7.4.1-bookworm@sha256:bb142a9c18ac18a16713c1491d779697b4e107c22a97266616099d288237ef47", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:54.007675374Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 67108864, + "nano_cpus": 250000000, + "ports": { + "6379/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f7abb66b7f4b85950f0cf389d52ad8ef0d6ba4596c5e96f1b356359d0e0ab011" + } + ] + }, + { + "container": "99e7c7f261870f281f7b9185e5a12c752e4e77d9890288e73f66628db5967106", + "name": "/mtusi-lab5-s3-1", + "service": "s3", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:07:19.59007169Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 500000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "f9ce12d80744156c70300c64c852f61c875e0e39bfa06be3f8aeb18c42d4fd21" + }, + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + } + ] + }, + { + "container": "99b00792fe5c4fcdfc13198c140b2aface58a7d5715cef89f6065a2b336f07a7", + "name": "/mtusi-lab5-s3-store-1", + "service": "s3-store", + "image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8", + "user": "", + "running": true, + "started_at": "2026-10-27T15:05:55.491841572Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "ports": { + "18080/tcp": null, + "18888/tcp": null, + "19333/tcp": null, + "7333/tcp": null, + "8080/tcp": null, + "8333/tcp": null, + "8888/tcp": null, + "9333/tcp": null + }, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "mtusi-lab5_s3_data" + } + ] + }, + { + "container": "8253cc88993d62bf5f7d20c9f7d7291933cb12a1b08d1b4416ee9729901deaed", + "name": "/mtusi-lab5-worker-1", + "service": "worker", + "image": "mtusi-messenger:lab5", + "user": "10001:10001", + "running": true, + "started_at": "2026-10-27T15:08:08.679577406Z", + "exit_code": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nano_cpus": 1000000000, + "ports": { + "8080/tcp": null + }, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_worker_secret", + "name": null + } + ] + } +] diff --git a/evidence/security-stderr.txt b/evidence/security-stderr.txt new file mode 100644 index 00000000..7a251ff6 --- /dev/null +++ b/evidence/security-stderr.txt @@ -0,0 +1,2 @@ + Container mtusi-lab5-audit-run-790369e3d6e4 Creating + Container mtusi-lab5-audit-run-790369e3d6e4 Created diff --git a/evidence/security.json b/evidence/security.json new file mode 100644 index 00000000..748b6f7a --- /dev/null +++ b/evidence/security.json @@ -0,0 +1,15 @@ +{ + "anonymous_object_and_list_403": true, + "job_fields": [ + "attachment_id", + "attempt", + "created_at", + "job_id", + "pipeline_version", + "schema_version", + "type" + ], + "job_bytes": 243, + "no_source_bytes_or_credentials_in_envelope": true, + "strict_invalid_envelope_checks": true +} diff --git a/evidence/slow-upload.json b/evidence/slow-upload.json new file mode 100644 index 00000000..9e303a32 --- /dev/null +++ b/evidence/slow-upload.json @@ -0,0 +1 @@ +{"trickle_seconds": 14.998, "http_status": 400, "absolute_deadline_enforced": true} diff --git a/evidence/startup.txt b/evidence/startup.txt new file mode 100644 index 00000000..0213ac1a --- /dev/null +++ b/evidence/startup.txt @@ -0,0 +1,80 @@ + Container mtusi-lab5-broker-1 Running + Container mtusi-lab5-s3-store-1 Running + Container mtusi-lab5-postgres-1 Running + Container mtusi-lab5-redis-1 Running + Container mtusi-lab5-s3-1 Running + Container mtusi-lab5-bucket-bootstrap-1 Recreate + Container mtusi-lab5-migrate-1 Recreate + Container mtusi-lab5-bucket-bootstrap-1 Recreated + Container mtusi-lab5-migrate-1 Recreated + Container mtusi-lab5-worker-1 Recreate + Container mtusi-lab5-api-1 Recreate + Container mtusi-lab5-dispatcher-1 Recreate + Container mtusi-lab5-api-2 Recreate + Container mtusi-lab5-dispatcher-1 Recreated + Container mtusi-lab5-api-1 Recreated + Container mtusi-lab5-api-2 Recreated + Container mtusi-lab5-lb-1 Running + Container mtusi-lab5-worker-1 Recreated + Container mtusi-lab5-s3-store-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-s3-store-1 Healthy + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-migrate-1 Starting + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-migrate-1 Started + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-bucket-bootstrap-1 Starting + Container mtusi-lab5-bucket-bootstrap-1 Started + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-redis-1 Waiting + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Starting + Container mtusi-lab5-dispatcher-1 Started + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-redis-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-worker-1 Starting + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-worker-1 Started + Container mtusi-lab5-api-1 Started + Container mtusi-lab5-api-2 Starting + Container mtusi-lab5-api-2 Started + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-redis-1 Waiting + Container mtusi-lab5-dispatcher-1 Waiting + Container mtusi-lab5-api-1 Waiting + Container mtusi-lab5-api-2 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-lb-1 Waiting + Container mtusi-lab5-bucket-bootstrap-1 Waiting + Container mtusi-lab5-s3-store-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-worker-1 Waiting + Container mtusi-lab5-redis-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-s3-store-1 Healthy + Container mtusi-lab5-lb-1 Healthy + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-dispatcher-1 Healthy + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-worker-1 Healthy + Container mtusi-lab5-bucket-bootstrap-1 Exited + Container mtusi-lab5-api-1 Healthy + Container mtusi-lab5-api-2 Healthy diff --git a/evidence/stopped-instances.jsonl b/evidence/stopped-instances.jsonl new file mode 100644 index 00000000..80b74b5b --- /dev/null +++ b/evidence/stopped-instances.jsonl @@ -0,0 +1 @@ +{"instance": "e4901b6a601a4dd094235d4dfcb0591c", "container": "e22e0a3894effd8ff3990a11ba75247f5fdb8eb5fe4272518d2b4c591e7b9efa", "running": false, "exit_code": 0, "seconds": 0.445} diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..2f51eee4 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18091" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +BASE_URL="http://127.0.0.1:18091" python3 -m unittest discover -s student-tests -p 'test_*.py' -v +test_concurrent_messages_and_pagination (test_api.APIContract) ... ok +test_cursor_scope_and_validation (test_api.APIContract) ... ok +test_foreign_chat (test_api.APIContract) ... ok +test_idempotency_concurrent_replicas (test_api.APIContract) ... ok +test_idempotency_conflicting_race (test_api.APIContract) ... ok +test_idempotency_invalid_keys (test_api.APIContract) ... ok +test_idempotency_scope (test_api.APIContract) ... ok +test_idempotency_semantic_body (test_api.APIContract) ... ok +test_invalid_chat_creation (test_api.APIContract) ... ok +test_malformed_json_and_extra_fields (test_api.APIContract) ... ok +test_protocol_error_headers (test_api.APIContract) ... ok +test_registration_race (test_api.APIContract) ... ok +test_unicode_limits (test_api.APIContract) ... ok +test_unicode_nul_round_trip (test_api.APIContract) ... ok +test_unicode_password_basic_and_logout (test_api.APIContract) ... ok +test_async_variants_and_replay (test_images.Images) ... ok +test_mime_bytes_pixels_and_filename (test_images.Images) ... ok +test_private_and_cross_chat (test_images.Images) ... ok + +---------------------------------------------------------------------- +Ran 18 tests in 5.184s + +OK diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..5495e1e8 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,46 @@ +global + maxconn 1024 + nbthread 1 + hard-stop-after 10s + +defaults + mode http + timeout connect 500ms + timeout check 500ms + timeout queue 1s + timeout client 30s + timeout server 30s + timeout http-request 3s + retries 2 + retry-on conn-failure + option http-server-close + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 2 + server-template api 1-4 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 + +listen metrics + bind :8404 + stats enable + stats uri /stats diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..eeac3c14 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,42 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY + , username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$') + , display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100) + , password_hash TEXT NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; +CREATE TABLE chats ( + id UUID PRIMARY KEY + , title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , last_message_at TIMESTAMPTZ +) +; +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) + , user_id UUID NOT NULL REFERENCES users(id) + , PRIMARY KEY (chat_id, user_id) +) +; +CREATE INDEX chat_members_user ON chat_members (user_id, chat_id) +; +CREATE TABLE messages ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , sender_id UUID NOT NULL REFERENCES users(id) + , text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000) + , created_at TIMESTAMPTZ NOT NULL + , FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX messages_history ON messages (chat_id, created_at, id) +; +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY + , user_id UUID NOT NULL REFERENCES users(id) + , expires_at TIMESTAMPTZ NOT NULL +) +; +CREATE INDEX sessions_expiry ON sessions (expires_at) +; diff --git a/migrations/002_unicode_nul.sql b/migrations/002_unicode_nul.sql new file mode 100644 index 00000000..224cda0d --- /dev/null +++ b/migrations/002_unicode_nul.sql @@ -0,0 +1,17 @@ +-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA. +-- API validation keeps the contract's code-point limits; these bounds cap bytes. +ALTER TABLE users + DROP CONSTRAINT users_display_name_check + , ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8') + , ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +; +ALTER TABLE chats + DROP CONSTRAINT chats_title_check + , ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8') + , ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +; +ALTER TABLE messages + DROP CONSTRAINT messages_text_check + , ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8') + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..3ca53a7e --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) + , chat_id UUID NOT NULL REFERENCES chats(id) + , key VARCHAR(128) NOT NULL + , request_hash CHAR(64) NOT NULL + , message_id UUID REFERENCES messages(id) + , expires_at TIMESTAMPTZ NOT NULL + , PRIMARY KEY (user_id, chat_id, key) +) +; +CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at) +; diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..bfd51898 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,30 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_bytes + , ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000) + , ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}' +; +CREATE TABLE attachments ( + id UUID PRIMARY KEY + , chat_id UUID NOT NULL REFERENCES chats(id) + , owner_id UUID NOT NULL REFERENCES users(id) + , filename TEXT NOT NULL + , object_key TEXT NOT NULL UNIQUE + , content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')) + , size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760) + , sha256 CHAR(64) NOT NULL + , state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing')) + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +) +; +CREATE INDEX attachments_chat ON attachments (chat_id, id) +; +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , PRIMARY KEY (message_id, attachment_id) +) +; +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id) +; diff --git a/migrations/005_pipeline.sql b/migrations/005_pipeline.sql new file mode 100644 index 00000000..0ef1e81d --- /dev/null +++ b/migrations/005_pipeline.sql @@ -0,0 +1,40 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_state_check + , ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing')) + , ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 + , ADD COLUMN staging_key TEXT + , ADD COLUMN variants JSONB NOT NULL DEFAULT '[]' + , ADD COLUMN error_code TEXT + , ADD COLUMN current_job_id UUID +; +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY + , attachment_id UUID NOT NULL REFERENCES attachments(id) + , pipeline_version INTEGER NOT NULL + , source_key TEXT NOT NULL + , state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')) + , attempt INTEGER NOT NULL DEFAULT 0 + , max_attempts INTEGER NOT NULL DEFAULT 5 + , lease_token UUID + , lease_until TIMESTAMPTZ + , next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , published_at TIMESTAMPTZ + , error_code TEXT + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() + , UNIQUE (attachment_id, pipeline_version) +) +; +ALTER TABLE attachments + ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id) +; +CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at) +; +CREATE TABLE job_events ( + id BIGSERIAL PRIMARY KEY + , job_id UUID NOT NULL REFERENCES attachment_jobs(id) + , event TEXT NOT NULL + , attempt INTEGER NOT NULL + , created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +; diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..d97ccc99 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Optional, only for plotting the saved evidence, Python 3.11+. +matplotlib==3.11.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..30a631ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +typing-extensions==4.16.0 +boto3==1.40.61 +Pillow==13.0.0 +botocore==1.40.76 +jmespath==1.1.0 +s3transfer==0.14.0 +python-dateutil==2.9.0.post0 +urllib3==2.8.0 +six==1.17.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..4b118cbd --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(mode=0o700, exist_ok=True) + +def save(name, value): + path = directory / name + if not path.exists(): + path.write_text(value) + path.chmod(0o644) # UID 10001 in Compose; parent is owner-only 0700. + return path.read_text().strip() + +password = save('postgres_password', secrets.token_hex(24)) +save('database_url', 'postgresql://messenger:' + password + '@postgres:5432/messenger') +save('cache_url', 'redis://redis:6379/0') +save('broker_url', 'redis://broker:6379/0') +identities = [] +for role, actions in { + 'admin': ['Admin', 'Read', 'Write', 'List'], + 'api': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], + 'worker': ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'], +}.items(): + key = save('s3_' + role + '_key', secrets.token_hex(12)) + secret = save('s3_' + role + '_secret', secrets.token_hex(24)) + identities.append({'name': role, 'credentials': [{'accessKey': key, 'secretKey': secret}], 'actions': actions}) +configuration = directory / 's3_config' +configuration.write_text(json.dumps({'identities': identities})) +configuration.chmod(0o600) # Only the root S3 gateway mounts the full identity configuration. +env = root / '.env' +if not env.exists(): + env.write_text((root / '.env.example').read_text()) + env.chmod(0o600) +print('Local configuration ready; no secrets printed.') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c8e00d70 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/snapshot.py > evidence/restart-before.json +docker compose stop api lb worker dispatcher +docker compose restart postgres redis s3-store s3 broker +docker compose up -d --wait postgres redis s3-store s3 broker +docker compose up -d --wait api lb worker dispatcher +python3 scripts/snapshot.py > evidence/restart-after.json diff --git a/scripts/snapshot.py b/scripts/snapshot.py new file mode 100644 index 00000000..3d0354f3 --- /dev/null +++ b/scripts/snapshot.py @@ -0,0 +1,11 @@ +"""Sanitized process and resource evidence; never dump container environments.""" +import json +import subprocess + +ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split() +records = [] +for container in ids: + raw = subprocess.check_output(['docker', 'inspect', container], text=True) + data = json.loads(raw)[0] + records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]}) +print(json.dumps(records, indent=2)) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..ce9279f2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/stop_one.py diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..bc495aa5 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,32 @@ +"""Resolve the requested process alias using each actual container, never a fake header.""" +import json +import os +from pathlib import Path +import subprocess +import time + + +def docker(*args): + return subprocess.check_output(['docker', *args], text=True).strip() + + +def instance(container): + code = "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=1).headers['X-Instance-Id'])" + return docker('exec', container, 'python', '-c', code) + + +if __name__ == '__main__': + target = os.environ['TARGET_INSTANCE_ID'] + started = time.monotonic() + for container in docker('compose', 'ps', '-q', 'api').splitlines(): + if instance(container) == target: + docker('stop', '--time', '8', container) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container)) + assert not state['Running'], 'target must remain stopped' + record = {'instance': target, 'container': container, 'running': state['Running'], 'exit_code': state['ExitCode'], 'seconds': round(time.monotonic() - started, 3)} + with Path('evidence/stopped-instances.jsonl').open('a') as out: + out.write(json.dumps(record) + '\n') + print(json.dumps(record)) + break + else: + raise SystemExit('Requested live API instance not found') diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..e635caef --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,16 @@ +import os +import time +import urllib.request + +base = os.environ.get('BASE_URL', 'http://127.0.0.1:18087') +for attempt in range(120): + try: + with urllib.request.urlopen(base + '/health/ready', timeout=2) as response: + if response.status == 200: + print('Ready through load balancer.') + break + except OSError: + pass + time.sleep(0.5) +else: + raise SystemExit('Readiness timeout') diff --git a/src/admin.py b/src/admin.py new file mode 100644 index 00000000..699bd3e9 --- /dev/null +++ b/src/admin.py @@ -0,0 +1,127 @@ +"""Explicit local operator tools; no administrative HTTP endpoint.""" +import argparse +import json +import os +import uuid + +from pipeline import connect, audit +from objects import Objects +from postgres import setting + + +def replay(job_id): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone() + if job is None or job['state'] != 'failed': + raise ValueError('Replay requires a failed job') + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone() + if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None: + raise ValueError('Current failed source is unavailable') + conn.execute(''' + UPDATE attachment_jobs + SET state = 'queued' + , max_attempts = attempt + %s + , published_at = NULL + , next_attempt_at = clock_timestamp() + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id)) + conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,)) + audit(conn, job, 'operator_replay') + return {'job_id': str(job_id), 'replayed': True} + + +def version(attachment_id, number): + if number not in (1, 2): + raise ValueError('Only implemented pipeline versions 1 and 2 are supported') + with connect() as conn, conn.transaction(): + row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() + if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'): + raise ValueError('Version must advance the current process exactly once') + old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone() + source = row['object_key'] if row['state'] == 'ready' else old['source_key'] + if Objects(setting).head(source) is None: + raise ValueError('Source unavailable') + conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],)) + audit(conn, old, 'operator_supersede') + job_id = uuid.uuid4() + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts) + VALUES (%s, %s, %s, %s, %s) + ''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))) + conn.execute(''' + UPDATE attachments + SET pipeline_version = %s + , current_job_id = %s + , state = 'queued' + , variants = '[]' + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (number, job_id, attachment_id)) + return {'job_id': str(job_id), 'version': number} + + +def cleanup(apply=False, retention=86400): + if retention < 86400: + raise ValueError('Retention must be >=24 hours') + objects = Objects(setting) + counts = {'ready_staging': 0, 'expired_staging': 0} + with connect() as conn: + # Intent rows cover every valid staging PUT. Never collect active jobs. + last = '00000000-0000-0000-0000-000000000000' + while True: + rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall() + if not rows: + break + for candidate in rows: + last = candidate['id'] + with conn.transaction(): + row = conn.execute(''' + SELECT * + FROM attachments + WHERE 1=1 + AND id = %s + AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second')) + FOR UPDATE SKIP LOCKED + ''', (last, retention)).fetchone() + if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'): + continue + counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1 + if apply: + objects.delete(row['staging_key']) + if row['state'] == 'uploading' and row['current_job_id'] is None: + conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,)) + else: + conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,)) + return dict(apply=apply, retention_seconds=retention, counts=counts) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup']) + p.add_argument('identifier', nargs='?') + p.add_argument('--version', type=int) + p.add_argument('--apply', action='store_true') + args = p.parse_args() + try: + if args.action == 'failed': + with connect() as conn: + result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall() + elif args.action == 'replay': + result = replay(uuid.UUID(args.identifier)) + elif args.action == 'version': + result = version(uuid.UUID(args.identifier), args.version) + else: + result = cleanup(args.apply) + print(json.dumps(result, default=str, indent=2)) + except Exception: + print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'})) + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/src/bucket_bootstrap.py b/src/bucket_bootstrap.py new file mode 100644 index 00000000..a0a2cbd3 --- /dev/null +++ b/src/bucket_bootstrap.py @@ -0,0 +1,13 @@ +"""Idempotent private bucket bootstrap, executed only with administrative S3 keys.""" +from botocore.exceptions import ClientError +from objects import Objects +from postgres import setting + +objects = Objects(setting) +try: + objects.client.create_bucket(Bucket=objects.bucket) +except ClientError as exc: + if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'): + raise RuntimeError('Bucket bootstrap failed') from None +objects.ready() +print('Private bucket ready; static authenticated identities loaded from local secret config.') diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..29c87c58 --- /dev/null +++ b/src/common.py @@ -0,0 +1,98 @@ +"""Shared HTTP contract; storage is supplied by each laboratory.""" +import base64 +import datetime as dt +import json +import re +import uuid + + +class APIError(Exception): + def __init__(self, status, code, message): + self.status, self.code, self.message = status, code, message + + +def invalid(): + raise APIError(400, 'invalid_request', 'Неверные параметры запроса') + + +def unauthorized(): + raise APIError(401, 'unauthorized', 'Требуется действительная идентификация') + + +def missing(): + raise APIError(404, 'not_found', 'Объект не найден') + + +def identifier(value): + try: + if not isinstance(value, str): + invalid() + return str(uuid.UUID(value)) + except ValueError: + invalid() + + +def text(value, maximum, minimum=1): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +def fields(value, required): + if not isinstance(value, dict) or set(value) != set(required): + invalid() + + +def username(value): + if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value): + invalid() + return value + + +def now(): + return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def page_args(query, scope): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + invalid() + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]+', limit): + invalid() + try: + limit = int(limit) + except ValueError: + invalid() + if not 1 <= limit <= 100: + invalid() + cursor = None + if 'cursor' in query: + raw = query['cursor'][0] + try: + if not raw or len(raw) > 1024: + invalid() + decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True)) + if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope: + invalid() + stamp = decoded[1] + if not isinstance(stamp, str) or not stamp.endswith('Z'): + invalid() + stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z') + cursor = (stamp, identifier(decoded[2])) + except (ValueError, TypeError, UnicodeError): + invalid() + return limit, cursor + + +def page(items, limit, scope): + more = len(items) > limit + items = items[:limit] + cursor = None + if more: + last = items[-1] + cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=') + return {'items': items, 'next_cursor': cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..ebd85f70 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,13 @@ +"""One-shot migration job; replicas never execute DDL.""" +from pathlib import Path +import psycopg +from postgres import setting + +with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2301003)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)') + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,)) + print('Applied', path.name) diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 00000000..f3a634cb --- /dev/null +++ b/src/objects.py @@ -0,0 +1,156 @@ +"""Bounded S3 operations. No root credentials or filename-derived object keys.""" +import contextlib +import hashlib +import os +import tempfile +import threading +import time +import unicodedata +import warnings + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, PngImagePlugin, UnidentifiedImageError + +from common import APIError, invalid + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024 +PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) +UPLOAD_SLOTS = threading.BoundedSemaphore(8) +DECODE_SLOTS = threading.BoundedSemaphore(1) + + +def unavailable(): + raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно') + + +def filename(value): + if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip(): + invalid() + if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + return value + + +@contextlib.contextmanager +def bounded_upload(): + if not UPLOAD_SLOTS.acquire(blocking=False): + raise APIError(503, 'unavailable', 'Все слоты загрузки заняты') + try: + yield + finally: + UPLOAD_SLOTS.release() + + +@contextlib.contextmanager +def receive(handler): + lengths = handler.headers.get_all('Content-Length', []) + if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Максимум 10 MiB') + if length <= 0: + invalid() + mime = handler.headers.get_content_type() + if mime not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG') + deadline = time.monotonic() + 15 + digest = hashlib.sha256() + # Ephemeral bounded spool is never a durable source of a confirmed attachment. + with tempfile.TemporaryFile(mode='w+b') as source: + remaining = length + while remaining: + handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic()))) + if time.monotonic() >= deadline: + invalid() + try: + chunk = handler.rfile.read1(min(64 * 1024, remaining)) + except (OSError, TimeoutError): + invalid() + if not chunk: + invalid() + digest.update(chunk) + source.write(chunk) + remaining -= len(chunk) + if not DECODE_SLOTS.acquire(timeout=10): + raise APIError(503, 'unavailable', 'Проверка изображений перегружена') + try: + source.flush() + for decode in (False,): + source.seek(0) + # Pillow.close releases pixel buffers and may close its input file. + # A duplicate descriptor preserves the original for the S3 PUT. + with os.fdopen(os.dup(source.fileno()), 'rb') as image_file: + image = Image.open(image_file, formats=['PNG', 'JPEG']) + try: + if image.width * image.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') + if Image.MIME[image.format] != mime: + raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению') + if decode: + image.load() + else: + image.verify() + finally: + image.close() # Free pixels BEFORE releasing the decoder semaphore. + except (Image.DecompressionBombWarning, Image.DecompressionBombError): + raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None + finally: + DECODE_SLOTS.release() + source.seek(0) + yield source, length, mime, digest.hexdigest() + + +class Objects: + def __init__(self, setting): + self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + self.client = boto3.client('s3', endpoint_url=os.environ.get('S3_ENDPOINT', 'http://s3:8333'), region_name='us-east-1', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required')) + + def head(self, key): + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'): + return None + unavailable() + except BotoCoreError: + unavailable() + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + except (ClientError, BotoCoreError): + unavailable() + + def put(self, key, source, length, mime, digest): + try: + self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest}) + except (ClientError, BotoCoreError): + unavailable() + + def get(self, key): + try: + return self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() + + def delete(self, key): + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError): + unavailable() diff --git a/src/pipeline.py b/src/pipeline.py new file mode 100644 index 00000000..9086ec9c --- /dev/null +++ b/src/pipeline.py @@ -0,0 +1,332 @@ +"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits.""" +import argparse +import concurrent.futures +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import random +import re +import signal +import subprocess +import sys +import tempfile +import threading +import time +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +import redis +from botocore.exceptions import ClientError, BotoCoreError +from common import APIError +from objects import Objects, MAX_BYTES +from postgres import setting + +STREAM, GROUP = 'attachment-jobs', 'image-workers' +LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60')) +STOP = threading.Event() + + +def connect(): + return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500') + + +def broker(): + client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3) + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): + raise + return client + + +def event(job): + return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z')} + + +def validate(message): + required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message): + return False + try: + uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id']) + stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')) + if stamp.tzinfo is None or not message['created_at'].endswith('Z'): + return False + if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])): + return False + if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512): + return False + return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt']) + except (ValueError, TypeError, AttributeError): + return False + + +def audit(conn, job, name): + conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt'])) + + +def retry(conn, job, code, permanent=False): + failed = permanent or job['attempt'] >= job['max_attempts'] + state = 'failed' if failed else 'queued' + code = code if permanent or not failed else 'retry_exhausted' + delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5) + conn.execute(''' + UPDATE attachment_jobs + SET state = %s + , error_code = %s + , lease_token = NULL + , lease_until = NULL + , published_at = NULL + , next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + ''', (state, code, delay, job['id'])) + conn.execute(''' + UPDATE attachments + SET state = %s + , error_code = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + ''', (state, code if failed else None, job['id'])) + audit(conn, job, 'failed:' + code if failed else 'retry:' + code) + + +def dispatch_once(client, fault=None): + with connect() as conn: + with conn.transaction(): + expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall() + for job in expired: + retry(conn, job, 'lease_expired') + with conn.transaction(): + jobs = conn.execute(''' + SELECT j.* + FROM attachment_jobs j + JOIN attachments a ON a.current_job_id = j.id + WHERE 1=1 + AND j.state = 'queued' + AND j.next_attempt_at <= clock_timestamp() + AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second') + ORDER BY j.created_at + LIMIT 10 + FOR UPDATE OF j SKIP LOCKED + ''', (LEASE,)).fetchall() + for job in jobs: + client.xadd(STREAM, {'event': json.dumps(event(job), separators=(',', ':'))}) + if fault: + fault('after_publish', job) + conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],)) + audit(conn, job, 'published') + return len(jobs) + + +def claim(message): + with connect() as conn, conn.transaction(): + job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone() + if job is None: + return None + attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone() + if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']: + return None + if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']: + audit(conn, job, 'stale_noop') + return None + if job['state'] in ('ready', 'failed', 'superseded'): + audit(conn, job, 'terminal_noop') + return None + if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1: + audit(conn, job, 'duplicate_or_early_noop') + return None + token = uuid.uuid4() + job = conn.execute(''' + UPDATE attachment_jobs + SET state = 'processing' + , attempt = attempt + 1 + , lease_token = %s + , lease_until = clock_timestamp() + %s * INTERVAL '1 second' + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (token, LEASE, job['id'])).fetchone() + conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],)) + audit(conn, job, 'claimed') + return job, attachment + + +class Permanent(Exception): + pass + + +def immutable(objects, key, path, mime, sha): + head = objects.head(key) + if head is not None: + if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha: + raise Permanent('result_conflict') + return + try: + with path.open('rb') as source: + objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*') + except ClientError as error: + if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412): + head = objects.head(key) + if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha: + return + raise + + +def process(message, fault=None): + claimed = claim(message) + if claimed is None: + return 'noop' + job, attachment = claimed + objects = Objects(setting) + try: + if fault: + fault('before_side_effect', job) + with tempfile.TemporaryDirectory(prefix='job-') as temporary: + directory = Path(temporary) + source = directory / 'original' + head = objects.head(job['source_key']) + if head is None: + raise Permanent('source_missing') + response = objects.get(job['source_key']) + digest, size = hashlib.sha256(), 0 + deadline = time.monotonic() + 15 + with response['Body'] as stream, source.open('wb') as out: + for chunk in iter(lambda: stream.read(64 * 1024), b''): + size += len(chunk) + if size > MAX_BYTES or time.monotonic() > deadline: + raise Permanent('invalid_image') + digest.update(chunk); out.write(chunk) + if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']: + raise Permanent('checksum_mismatch') + result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20'))) + if result.returncode == 2: + raise Permanent('invalid_image') + if result.returncode: + raise RuntimeError('render_failed') + variants = json.loads(result.stdout) + prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/' + original_key = prefix + 'original' + immutable(objects, original_key, source, attachment['content_type'], attachment['sha256']) + if fault: + fault('after_original', job) + for variant in variants: + variant['object_key'] = prefix + variant['name'] + immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256']) + if fault: + fault('after_objects', job) + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + changed = conn.execute(''' + UPDATE attachments + SET state = 'ready' + , object_key = %s + , variants = %s + , error_code = NULL + , updated_at = clock_timestamp() + WHERE 1=1 + AND current_job_id = %s + AND pipeline_version = %s + AND %s + RETURNING id + ''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone() + if changed: + conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],)) + audit(conn, job, 'ready') + else: + audit(conn, job, 'fenced_noop') + if fault: + fault('after_commit', job) + if changed and job['source_key'].startswith('staging/'): + objects.delete(job['source_key']) + return 'ready' if changed else 'fenced' + except Exception as error: + code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed' + with connect() as conn, conn.transaction(): + current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone() + if current['state'] == 'processing' and current['lease_token'] == job['lease_token']: + retry(conn, current, code, isinstance(error, Permanent)) + return 'failed_or_retry' + finally: + objects.client.close() + + +def ack(client, identifier): + # Atomic ack+delete; PostgreSQL is the durable completion/replay registry. + client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier) + + +def consume(once=False, fault=None): + client = broker() + consumer = uuid.uuid4().hex + while not STOP.is_set(): + messages = [] + try: + pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1] + if pending: + messages = pending + else: + batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500) + messages = batches[0][1] if batches else [] + for identifier, fields in messages: + raw = fields.get('event', '') + try: + message = json.loads(raw) if len(raw) <= 4096 else None + except ValueError: + message = None + if validate(message): + process(message, fault) + else: + client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False) + ack(client, identifier) + if once: + return + except (redis.RedisError, psycopg.Error): + STOP.wait(.5) + if once and not messages: + return + client.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['dispatch', 'work']) + parser.add_argument('--once', action='store_true') + parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish']) + args = parser.parse_args() + # Operator-only opt-in injection; no request header or public endpoint controls it. + def fault(point, job): + if point == args.fault_point: + os._exit(77) + signal.signal(signal.SIGTERM, lambda *_: STOP.set()) + signal.signal(signal.SIGINT, lambda *_: STOP.set()) + if args.mode == 'dispatch': + client = broker() + while not STOP.is_set(): + try: + dispatch_once(client, fault) + except (redis.RedisError, psycopg.Error): + pass + if args.once: + break + STOP.wait(.2) + else: + count = int(os.environ.get('WORKER_CONCURRENCY', '1')) + if not 1 <= count <= 2: + raise ValueError('WORKER_CONCURRENCY must be 1..2') + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + futures = [pool.submit(consume, args.once, fault) for _ in range(count)] + for result in futures: + result.result() + + +if __name__ == '__main__': + main() diff --git a/src/postgres.py b/src/postgres.py new file mode 100644 index 00000000..0ee57fa2 --- /dev/null +++ b/src/postgres.py @@ -0,0 +1,409 @@ +import base64 +import contextlib +import datetime as dt +import hashlib +import hmac +import json +import os +import re +from pathlib import Path +import secrets +import threading +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +import redis + +from objects import Objects, unavailable + +from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username + +KDF_SLOTS = threading.BoundedSemaphore(2) + + +def setting(name): + path = os.environ.get(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def public(row): + return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'} + + +def password_hash(password, salt=None): + salt = salt or secrets.token_bytes(16) + if not KDF_SLOTS.acquire(timeout=1.5): + raise APIError(503, 'unavailable', 'Аутентификация временно перегружена') + try: + digest = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024) + finally: + KDF_SLOTS.release() + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +class Store: + persistent = True + + def __init__(self): + self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600')) + if not 1 <= self.ttl <= 86400: + raise ValueError('SESSION_TTL_SECONDS must be 1..86400') + self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized') + if self.access_mode not in ('baseline', 'optimized'): + raise ValueError('DB_ACCESS_MODE must be baseline or optimized') + pool_size = int(os.environ.get('DB_POOL_MAX', '8')) + if not 1 <= pool_size <= 32: + raise ValueError('DB_POOL_MAX must be 1..32') + self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True) + self.pool.wait(timeout=10) + self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True) + self.objects = Objects(setting) + self.dummy_hash = password_hash('dummy-password-for-unknown-users') + with self.connection() as conn: + if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '005_pipeline.sql'").fetchone(): + raise RuntimeError('Run the separate migration service first') + + @contextlib.contextmanager + def connection(self): + try: + with self.pool.connection() as conn: + yield conn + except psycopg.errors.UniqueViolation: + raise APIError(409, 'conflict', 'Имя пользователя занято') from None + except (psycopg.Error, PoolTimeout): + raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None + + def close(self): + self.pool.close() + self.cache.close() + self.objects.client.close() + + def ready(self): + self.objects.ready() + with self.connection() as conn: + conn.execute('SELECT 1') + + def register(self, body): + fields(body, ('username', 'display_name', 'password')) + name, display = username(body['username']), text(body['display_name'], 100) + # Passwords may contain spaces and colons; only length/UTF-8 are restricted. + password = body['password'] + if not isinstance(password, str) or not 12 <= len(password) <= 128: + invalid() + try: + password.encode('utf-8') + except UnicodeError: + invalid() + hashed = password_hash(password) + with self.connection() as conn: + row = conn.execute(''' + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + ''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone() + return public(row) + + def login(self, headers): + authorization = headers.get('Authorization', '') + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic': + unauthorized() + name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError): + unauthorized() + if not re_username(name) or not 12 <= len(password) <= 128: + unauthorized() + with self.connection() as conn: + row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone() + hashed = row['password_hash'] if row else self.dummy_hash + verified = hmac.compare_digest(password_hash(password, bytes.fromhex(hashed.split('$')[4])), hashed) + if row is None or not verified: + unauthorized() + token = secrets.token_urlsafe(32) + with self.connection() as conn: + expiry = conn.execute(''' + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second') + RETURNING expires_at + ''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at'] + return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)} + + def token_hash(self, headers): + parts = headers.get('Authorization', '').split(' ') + if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii(): + unauthorized() + return hashlib.sha256(parts[1].encode()).hexdigest() + + def authenticate(self, headers): + token_hash = self.token_hash(headers) + with self.connection() as conn: + session = conn.execute(''' + SELECT user_id + FROM sessions + WHERE 1=1 + AND token_hash = %s + AND expires_at > clock_timestamp() + ''', (token_hash,)).fetchone() + if session is None: + unauthorized() + key = 'user:' + str(session['user_id']) + try: + cached = self.cache.get(key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError): + pass + row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone() + if row is None: + unauthorized() + user = public(row) + try: + self.cache.setex(key, 60, json.dumps(user)) + except redis.RedisError: + pass + return user + + def logout(self, headers): + with self.connection() as conn: + conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),)) + + def chat_in(self, conn, actor, chat_id): + row = conn.execute(''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND c.id = %s + AND cm.user_id = %s + ''', (chat_id, actor['id'])).fetchone() + if row is None: + missing() + members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall() + return dict(public(row), member_ids=[str(v['user_id']) for v in members]) + + def chat(self, actor, chat_id): + with self.connection() as conn: + return self.chat_in(conn, actor, chat_id) + + def create_chat(self, actor, body): + fields(body, ('title', 'member_ids')) + title = text(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + invalid() + members = [identifier(v) for v in members] + if len(set(members)) != len(members) or actor['id'] not in members: + invalid() + with self.connection() as conn: + chat_id = uuid.uuid4() + conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8'))) + # Checking after INSERT exercises rollback of the whole composite operation. + found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall() + if len(found) != len(members): + invalid() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members]) + return self.chat_in(conn, actor, chat_id) + + def list_chats(self, actor, limit, cursor, scope): + with self.connection() as conn: + where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [actor['id']] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT c.id + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE 1=1 + AND cm.user_id = %s + ''' + where + ''' + ORDER BY c.created_at, c.id + LIMIT %s + ''', params).fetchall() + return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope) + + def require_member(self, conn, actor, chat_id): + if self.access_mode == 'baseline': + return self.chat_in(conn, actor, chat_id) + if not conn.execute(''' + SELECT 1 + FROM chat_members + WHERE 1=1 + AND chat_id = %s + AND user_id = %s + ''', (chat_id, actor['id'])).fetchone(): + missing() + + def send(self, actor, chat_id, body, key): + if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body: + invalid() + attachments = body.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [identifier(v) for v in attachments] + if len(set(attachments)) != len(attachments): + invalid() + content = body['text'] + if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()): + invalid() + try: + content.encode('utf-8') + except UnicodeError: + invalid() + canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + # The unique key arbitrates concurrent processes; everything commits together. + claimed = conn.execute(''' + INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at) + VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours') + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash + , message_id = NULL + , expires_at = EXCLUDED.expires_at + WHERE 1=1 + AND idempotency_keys.expires_at <= clock_timestamp() + RETURNING request_hash + ''', (actor['id'], chat_id, key, digest)).fetchone() + if claimed is None: + previous = conn.execute(''' + SELECT ik.request_hash + , m.id + , m.chat_id + , m.sender_id + , m.text + , m.created_at + , m.attachment_ids + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE 1=1 + AND ik.user_id = %s + AND ik.chat_id = %s + AND ik.key = %s + ''', (actor['id'], chat_id, key)).fetchone() + if previous['request_hash'] != digest: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + previous.pop('request_hash') + return public(previous) + if attachments: + rows = conn.execute(''' + SELECT id, state + FROM attachments + WHERE 1=1 + AND id = ANY(%s::uuid[]) + AND chat_id = %s + ORDER BY id + FOR SHARE + ''', (attachments, chat_id)).fetchall() + if len(rows) != len(attachments): + missing() + if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows): + raise APIError(409, 'conflict', 'Вложение ещё не готово') + # Per-chat row lock preserves committed history order across replicas/clocks. + stamp = conn.execute(''' + UPDATE chats + SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond') + WHERE 1=1 + AND id = %s + RETURNING last_message_at + ''', (chat_id,)).fetchone()['last_message_at'] + row = conn.execute(''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (%s, %s, %s, %s, %s, %s::uuid[]) + RETURNING id, chat_id, sender_id, text, created_at, attachment_ids + ''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone() + if attachments: + with conn.cursor() as cur: + cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments]) + conn.execute(''' + UPDATE idempotency_keys + SET message_id = %s + WHERE 1=1 + AND user_id = %s + AND chat_id = %s + AND key = %s + ''', (row['id'], actor['id'], chat_id, key)) + return public(row) + + def history(self, actor, chat_id, limit, cursor, scope): + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else '' + params = [chat_id] + list(cursor or ()) + [limit + 1] + rows = conn.execute(''' + SELECT id, chat_id, sender_id, text, created_at, attachment_ids + FROM messages + WHERE 1=1 + AND chat_id = %s + ''' + where + ''' + ORDER BY created_at, id + LIMIT %s + ''', params).fetchall() + return page([public(v) for v in rows], limit, scope) + + def upload(self, actor, chat_id, name, source, length, mime, digest): + attachment_id, job_id = uuid.uuid4(), uuid.uuid4() + key = 'staging/' + str(attachment_id) + with self.connection() as conn: + self.require_member(conn, actor, chat_id) + conn.execute('SELECT pg_advisory_xact_lock(2301005)') + pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n'] + if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')): + raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена') + conn.execute(''' + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading') + ''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest)) + with self.connection() as conn: + if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None: + unavailable() + self.objects.put(key, source, length, mime, digest) + conn.execute(''' + INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts) + VALUES (%s, %s, 1, %s, %s) + ''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))) + row = conn.execute(''' + UPDATE attachments + SET state = 'queued' + , current_job_id = %s + , updated_at = clock_timestamp() + WHERE 1=1 + AND id = %s + RETURNING * + ''', (job_id, attachment_id)).fetchone() + # Capture the queued snapshot, even if the worker finishes before HTTP reply. + return self.attachment_public(row) + + def attachment_public(self, row): + keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at') + result = public({k: row[k] for k in keys}) + variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']] + return dict(result, status=row['state'], variants=variants, error_code=row['error_code']) + + def attachment(self, actor, attachment_id): + with self.connection() as conn: + row = conn.execute(''' + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id + WHERE 1=1 + AND a.id = %s + AND cm.user_id = %s + ''', (attachment_id, actor['id'])).fetchone() + if row is None: + missing() + if row['state'] == 'storage_missing': + unavailable() + if row['state'] == 'uploading': + raise APIError(409, 'conflict', 'Приём не завершён') + return row + + +def re_username(value): + import re + return re.fullmatch(r'[a-z0-9_]{3,32}', value) diff --git a/src/render.py b/src/render.py new file mode 100644 index 00000000..a9093347 --- /dev/null +++ b/src/render.py @@ -0,0 +1,64 @@ +"""Isolated image decoder/resizer; no network, database or credentials.""" +import hashlib +import json +import math +from pathlib import Path +import resource +import sys +import warnings + +resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) +resource.setrlimit(resource.RLIMIT_CPU, (15, 16)) +from PIL import Image, ImageOps, PngImagePlugin +Image.MAX_IMAGE_PIXELS = 40_000_000 +PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024 +warnings.filterwarnings('error', category=Image.DecompressionBombWarning) + + +def render(source, directory, version): + output = [] + with Image.open(source, formats=['PNG', 'JPEG']) as image: + if image.width * image.height > 40_000_000: + raise ValueError('pixels') + image.load() + ImageOps.exif_transpose(image, in_place=True) + rgba = None + if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info): + rgba = image if image.mode == 'RGBA' else image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + alpha = rgba.getchannel('A') + rgb.paste(rgba, mask=alpha) + alpha.close() + else: + rgb = image if image.mode == 'RGB' else image.convert('RGB') + try: + for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]: + if name == 'crop': + result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5)) + else: + scale = min(1, bound / max(rgb.size)) + size = tuple(max(1, math.floor(v * scale)) for v in rgb.size) + result = rgb.resize(size, Image.Resampling.LANCZOS) + try: + path = directory / name + options = {'quality': quality, 'exif': b'', 'icc_profile': None} + if fmt == 'WEBP': + options['method'] = 4 + result.save(path, format=fmt, **options) + raw = path.read_bytes() + output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()}) + finally: + result.close() + finally: + if rgb is not image: + rgb.close() + if rgba is not None and rgba is not image: + rgba.close() + return output + + +if __name__ == '__main__': + try: + print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3])))) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning): + raise SystemExit(2) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..0b43d181 --- /dev/null +++ b/src/server.py @@ -0,0 +1,296 @@ +"""Bounded threaded API; persistent correctness state is shared across replicas.""" +import concurrent.futures +import http.server +import json +import logging +import os +import time +import signal +import socket +import threading +import uuid +from urllib.parse import parse_qs, urlsplit + +from objects import bounded_upload, receive, filename, unavailable + +from common import APIError, identifier, invalid, missing, page_args + +LOG = logging.getLogger('messenger') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + invalid() + result[key] = value + return result + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def setup(self): + super().setup() + self.connection.settimeout(3) + + def send_error(self, code, message=None, explain=None): + # BaseHTTPRequestHandler otherwise emits HTML without contract headers. + self.request_id = uuid.uuid4().hex + self.route = '' + self.close_connection = True + status = 413 if code == 413 else 400 + self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}}) + + def log_message(self, *args): + pass + + def reply(self, status, body): + if status == 200 and isinstance(body, tuple): + return self.download(*body) + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + if status >= 400: + self.close_connection = True + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + if self.close_connection: + self.send_header('Connection', 'close') + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + if status == 401 and self.server.store.persistent: + self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(raw) + + def download(self, row, response, chunk): + stream = response['Body'] + try: + self.send_response(200) + self.send_header('Content-Type', row['content_type']) + self.send_header('Content-Length', str(row['size_bytes'])) + self.send_header('X-Request-Id', self.request_id) + self.send_header('X-Instance-Id', self.server.instance_id) + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cache-Control', 'private, no-store') + self.end_headers() + deadline = time.monotonic() + 15 + while chunk: + self.wfile.write(chunk) + if time.monotonic() > deadline: + raise TimeoutError() + chunk = stream.read(64 * 1024) + finally: + stream.close() + + def body(self): + lengths = self.headers.get_all('Content-Length', []) + if len(lengths) != 1 or self.headers.get('Transfer-Encoding'): + invalid() + try: + length = int(lengths[0]) + except ValueError: + invalid() + if length > 32768: + raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт') + if length <= 0: + invalid() + if self.headers.get_content_type() != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Требуется application/json') + try: + raw = self.rfile.read(length) + if len(raw) != length: + invalid() + return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid()) + except (ValueError, UnicodeError, RecursionError): + invalid() + + def dispatch(self): + self.request_id = uuid.uuid4().hex + self.route = urlsplit(self.path).path + status = 500 + try: + status, result = self.route_request() + except APIError as exc: + status = exc.status + result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}} + except Exception: + # No bodies, credentials, tokens, SQL or exception details in logs/responses. + LOG.error('request_failed request_id=%s', self.request_id) + result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}} + try: + self.reply(status, result) + except Exception: + self.close_connection = True + LOG.info('method=%s status=%s request_id=%s', self.command, status, self.request_id) + + def route_request(self): + method, path, store = self.command, self.route, self.server.store + if method == 'GET' and path in ('/health/live', '/health/ready'): + if path.endswith('ready'): + store.ready() + return 200, {'status': 'ok'} + if method == 'POST' and path == '/api/v1/users': + return 201, store.register(self.body()) + if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + invalid() + return 201, store.login(self.headers) + actor = store.authenticate(self.headers) + if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.logout(self.headers) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, actor + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + if path == '/api/v1/chats': + if method == 'POST': + return 201, store.create_chat(actor, self.body()) + if method == 'GET': + scope = 'chats:' + actor['id'] + limit, cursor = page_args(query, scope) + return 200, store.list_chats(actor, limit, cursor, scope) + parts = path.split('/') + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET': + row = store.attachment(actor, identifier(parts[4])) + if len(parts) == 5: + if query: + invalid() + return 200, store.attachment_public(row) + if parts[5] == 'content': + if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()): + invalid() + if row['state'] != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + variant = query.get('variant', ['original'])[0] + if variant != 'original': + item = next((v for v in row['variants'] if v['name'] == variant), None) + if item is None: + missing() + row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes']) + response = store.objects.get(row['object_key']) + try: + chunk = response['Body'].read(64 * 1024) + if response['ContentLength'] != row['size_bytes'] or not chunk: + unavailable() + except Exception: + response['Body'].close() + unavailable() + return 200, (row, response, chunk) + if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']: + chat_id = identifier(parts[4]) + if store.access_mode == 'baseline': + store.chat(actor, chat_id) + if len(parts) == 5 and method == 'GET': + return 200, store.chat(actor, chat_id) + if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST': + store.chat(actor, chat_id) + if set(query) != {'filename'} or len(query['filename']) != 1: + invalid() + name = filename(query['filename'][0]) + with bounded_upload(): + with receive(self) as (source, length, mime, digest): + return 202, store.upload(actor, chat_id, name, source, length, mime, digest) + if len(parts) == 6 and parts[5] == 'messages': + if method == 'POST': + keys = self.headers.get_all('Idempotency-Key', []) + return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None) + if method == 'GET': + scope = 'messages:' + chat_id + limit, cursor = page_args(query, scope) + return 200, store.history(actor, chat_id, limit, cursor, scope) + missing() + + do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch + + +class Server(http.server.HTTPServer): + request_queue_size = 64 + allow_reuse_address = True + + def __init__(self, address, store): + self.store = store + self.instance_id = uuid.uuid4().hex + workers = int(os.environ.get('HTTP_WORKERS', '16')) + if not 1 <= workers <= 128: + raise ValueError('HTTP_WORKERS must be 1..128') + self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) + self.slots = threading.BoundedSemaphore(workers * 2) + self.connections = set() + self.connections_lock = threading.Lock() + super().__init__(address, Handler) + + def process_request(self, request, address): + if not self.slots.acquire(blocking=False): + rid = uuid.uuid4().hex + raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode() + try: + request.settimeout(0.2) + request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw) + except OSError: + pass + self.shutdown_request(request) + return + with self.connections_lock: + self.connections.add(request) + self.executor.submit(self.worker, request, address) + + def worker(self, request, address): + try: + self.finish_request(request, address) + except OSError: + pass + finally: + self.shutdown_request(request) + with self.connections_lock: + self.connections.discard(request) + self.slots.release() + + def close_connections(self): + with self.connections_lock: + connections = list(self.connections) + for connection in connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def server_close(self): + super().server_close() + # Bound draining even when a slow client keeps an incomplete request alive. + deadline = threading.Timer(5, self.close_connections) + deadline.start() + try: + self.executor.shutdown(wait=True) + finally: + deadline.cancel() + + +def main(): + logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s') + from postgres import Store + store = Store() + server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store) + stopping = threading.Event() + + def stop(signum, frame): + if not stopping.is_set(): + stopping.set() + LOG.info('shutdown_started signal=%s', signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats()))) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + LOG.info('started port=%s', server.server_address[1]) + try: + server.serve_forever(poll_interval=0.1) + finally: + server.server_close() + store.close() + LOG.info('shutdown_complete') + + +if __name__ == '__main__': + main() diff --git a/student-tests/durability.py b/student-tests/durability.py new file mode 100644 index 00000000..3470de0d --- /dev/null +++ b/student-tests/durability.py @@ -0,0 +1,52 @@ +"""Stop worker, accept queued, restart API/broker, then complete the same attachment.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client,Failure + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap();record={} + before=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + try: + docker('compose','stop','worker','dispatcher') + queued,raw=c.attachment(alice,chat) + assert queued['status']=='queued' + message=c.message(alice,chat,'accepted with worker stopped',attachments=[queued['id']])['json'] + c.call('GET','/api/v1/attachments/'+queued['id']+'/content',409,actor=bob) + killed=docker('compose','ps','-q','api').splitlines() + for container in killed:docker('kill','--signal=KILL',container) + record['killed_api_containers']=killed + docker('compose','up','-d','--wait','--no-deps','api') + time.sleep(2) + assert c.call('GET','/api/v1/attachments/'+queued['id'],actor=bob,schema='Attachment')['json']['status']=='queued' + docker('compose','up','-d','--no-deps','dispatcher') + deadline=time.monotonic()+10 + while int(docker('compose','exec','-T','broker','redis-cli','XLEN','attachment-jobs'))<1: + assert time.monotonic()=length_before>=1 + record['stream_entries_before_after_broker_restart']=[length_before,length_after] + docker('compose','up','-d','--no-deps','worker','dispatcher') + ready=c.wait_ready(bob,queued) + assert c.call('GET','/api/v1/attachments/'+queued['id']+'/content',actor=bob)['bytes']==raw + record.update(attachment_id=queued['id'],original_sha256=hashlib.sha256(raw).hexdigest(),status=ready['status'],variants=[v['name'] for v in ready['variants']],old_bearer_survived=True,message_attachment_retained=message['attachment_ids']==[queued['id']]) + finally:docker('compose','up','-d','--no-deps','worker','dispatcher') + after=json.loads(subprocess.check_output([sys.executable,'scripts/snapshot.py'],text=True)) + record['before']=before;record['after']=after + print(json.dumps(record,indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 00000000..906356fa --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,24 @@ +"""Small deterministic PNG helpers; generation happens outside measurement.""" +import random +import struct +import zlib + + +def chunk(name, data): + return struct.pack('!I', len(data)) + name + data + struct.pack('!I', zlib.crc32(name + data) & 0xffffffff) + + +def png(width, height, noise=False, rgb=True): + channels = 3 if rgb else 1 + rand = random.Random(2301) + encoded = zlib.compressobj(1) + parts = [] + for _ in range(height): + line = bytes(rand.getrandbits(8) for _ in range(width * channels)) if noise else b'\x7f' * (width * channels) + parts.append(encoded.compress(b'\0' + line)) + parts.append(encoded.flush()) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2 if rgb else 0, 0, 0, 0)) + chunk(b'IDAT', b''.join(parts)) + chunk(b'IEND', b'') + + +def bomb(): + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'x')) + chunk(b'IEND', b'') diff --git a/student-tests/pipeline_checks.py b/student-tests/pipeline_checks.py new file mode 100644 index 00000000..c4d36ba1 --- /dev/null +++ b/student-tests/pipeline_checks.py @@ -0,0 +1,188 @@ +"""Real DB/S3/Redis fault injection. Stop production worker/dispatcher first.""" +import concurrent.futures +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from PIL import Image +from client import Client +from pipeline import STREAM,GROUP,broker,connect,event,dispatch_once,process,ack +from objects import Objects +from postgres import setting +from admin import replay,version,cleanup + + +def main(): + c=Client();alice,bob,eve,chat=c.bootstrap() + bus=broker(); objects=Objects(setting) + report={'invariants':{},'faults':[]} + def fetch(aid): + with connect() as conn: + a=conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s',(aid,)).fetchone() + j=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(a['current_job_id'],)).fetchone() + return a,j + def upload(raw=None,mime='image/png'): + if raw is None: a,_=c.attachment(alice,chat) + else:a=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=research.png',202,actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment')['json'] + return a,fetch(a['id'])[1] + def run_once(fault=None): + args=[sys.executable,'/app/src/pipeline.py','work','--once'] + if fault:args+=['--fault-point',fault] + return subprocess.run(args,capture_output=True,timeout=30).returncode + def drain(): + for _ in range(200): + batch=bus.xreadgroup(GROUP,'research-drain',{STREAM:'>'},count=1) + if not batch:break + for mid,data in batch[0][1]:process(json.loads(data['event']));ack(bus,mid) + def expire(job): + with connect() as conn: + conn.execute("UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE 1=1 AND id = %s",(job['id'],)) + def recover(aid): + until=time.monotonic()+20 + while time.monotonic() no publisher yet: one queued row, no Redis message, then automatic outbox pass. + a,j=upload() + assert fetch(a['id'])[0]['state']=='queued' + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',409,actor=bob) + c.message(bob,chat,'queued is linkable',attachments=[a['id']]) + dispatch_once(bus);drain();assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['commit_before_publish_recovered']=True + # Broker accepted XADD but publisher died before marking outbox: true exit(77). + a,j=upload() + code=subprocess.run([sys.executable,'/app/src/pipeline.py','dispatch','--once','--fault-point','after_publish'],capture_output=True,timeout=10).returncode + assert code==77 + dispatch_once(bus);drain() + assert fetch(a['id'])[0]['state']=='ready' + report['invariants']['publish_before_marker_duplicate_safe']=True + for point in ['before_side_effect','after_original','after_objects','after_commit']: + a,j=upload();dispatch_once(bus) + code=run_once(point);assert code==77,(point,code) + before,job=fetch(a['id']) + if point not in ('after_commit','before_side_effect'):expire(job) + result,job=recover(a['id']) + assert c.call('GET','/api/v1/attachments/'+a['id']+'/content',actor=bob)['bytes']==(Path.cwd()/'tests/fixtures/sample.png').read_bytes() + reclaimed=bus.xautoclaim(STREAM,GROUP,'post-crash-redelivery',min_idle_time=0,start_id='0-0',count=100)[1] + for mid,data in reclaimed: + process(json.loads(data['event']));ack(bus,mid) + assert bus.xpending(STREAM,GROUP)['pending']==0 + keys=objects.client.list_objects_v2(Bucket=objects.bucket,Prefix='results/'+a['id']+'/v1/').get('Contents',[]) + assert len(keys)==4,(point,len(keys)) + report['faults'].append({'point':point,'exit_code':code,'state_at_crash':before['state'],'recovered':result['state'],'attempts':job['attempt'],'result_objects':len(keys)}) + report['invariants']['crashes_before_after_side_effect_and_commit']=True + bus.xadd(STREAM,{'event':json.dumps({'not':'a job'})}) + assert run_once()==0 + assert bus.xlen('rejected-jobs')>=1 + report['invariants']['invalid_envelope_observable_dlq']=True + # Duplicate/reorder through broker on a completed job: no extra attempt/effect. + a,j=upload();dispatch_once(bus);drain();before,done=fetch(a['id']) + old=event(j) + for attempt in [2,1,3,1,1]:bus.xadd(STREAM,{'event':json.dumps(dict(old,attempt=attempt))}) + drain();after,done2=fetch(a['id']) + assert before['variants']==after['variants'] and done2['attempt']==done['attempt'] + report['invariants']['duplicate_and_reorder_no_second_effect']=True + # Race two consumers with the SAME logical job, not just one serial duplicate. + a,j=upload();e=event(j) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: results=list(pool.map(lambda _:process(e),range(2))) + assert fetch(a['id'])[0]['state']=='ready' and sorted(results)==['noop','ready'] + report['invariants']['concurrent_duplicate_single_claim']=True + # Pause actual old process after all immutable side effects, advance version and finish v2. + a,j=upload();e=event(j) + directory=Path('/tmp')/('fence-'+uuid.uuid4().hex);directory.mkdir();marker=directory/'paused';release=directory/'release' + code="import sys,json,time;from pathlib import Path;sys.path.insert(0,'/app/src');from pipeline import process\ndef hook(point,job):\n if point=='after_objects':\n Path(sys.argv[2]).write_text('paused')\n while not Path(sys.argv[3]).exists():time.sleep(.05)\nprint(process(json.loads(sys.argv[1]),hook))" + child=subprocess.Popen([sys.executable,'-c',code,json.dumps(e),str(marker),str(release)],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True) + try: + deadline=time.monotonic()+20 + while not marker.exists(): + assert child.poll() is None + assert time.monotonic()=250 + transforms.append({'kind':kind,'size':thumb.size,'exif_removed':not bool(thumb.getexif())}) + report['transforms']=transforms + pending,pending_job=upload() + dry=cleanup(False);fixed=cleanup(True);repeat=cleanup(True) + assert fetch(pending['id'])[0]['state']=='queued' and objects.head(pending_job['source_key']) is not None + report['invariants']['cleanup_preserves_current_and_fresh_failed']=True + assert repeat['counts']['ready_staging']==0 + report['invariants']['ready_staging_cleanup_idempotent']=True + report['cleanup']=[dry,fixed,repeat] + print(json.dumps(report,indent=2)) + + +if __name__=='__main__': main() diff --git a/student-tests/plot.py b/student-tests/plot.py new file mode 100644 index 00000000..157828de --- /dev/null +++ b/student-tests/plot.py @@ -0,0 +1,28 @@ +"""Figures from real accepted/ready and cgroup samples, no synthetic results.""" +import json +from pathlib import Path +import statistics +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +r=Path(__file__).resolve().parents[1]/'evidence' +x=json.loads((r/'resources.json').read_text())['experiments'] +fig,axes=plt.subplots(1,2,figsize=(11,4.3),layout='constrained') +labels=['320×200\n1,3 KiB','1800×1400\n7,21 MiB','10000×4000\n0,51 MiB'] +a=[max(v['accepted_seconds'] for v in e['replies']) for e in x] +b=[max(v['ready_seconds'] for v in e['replies']) for e in x] +axes[0].bar([i-.18 for i in range(3)],a,.36,label='p95 до 202') +axes[0].bar([i+.18 for i in range(3)],b,.36,label='p95 до ready') +axes[0].set(xticks=range(3),xticklabels=labels,ylabel='Секунды',title='10 одновременных загрузок · worker=1') +axes[0].legend() +peak=[int(e['after']['memory.peak'])/1048576 for e in x] +anon=[max(int(dict(line.split() for line in s['worker']['memory.stat'].splitlines())['anon']) for s in e['samples'])/1048576 for e in x] +axes[1].bar([i-.18 for i in range(3)],peak,.36,label='cgroup memory.peak') +axes[1].bar([i+.18 for i in range(3)],anon,.36,label='Пик resident anon') +axes[1].axhline(768,color='#d62728',linestyle='--',label='Лимит worker') +axes[1].set(xticks=range(3),xticklabels=labels,ylabel='MiB',title='Worker + дочерний декодер') +axes[1].legend(fontsize=8) +for ax in axes:ax.grid(axis='y',alpha=.2);ax.set_axisbelow(True) +fig.suptitle('ЛР 5 · реальные измерения · concurrency=1, prefetch=1') +fig.savefig(r/'pipeline.png',dpi=150) +fig.savefig(r/'pipeline.svg') diff --git a/student-tests/resources.py b/student-tests/resources.py new file mode 100644 index 00000000..a9c409d3 --- /dev/null +++ b/student-tests/resources.py @@ -0,0 +1,52 @@ +"""Finite simultaneous jobs including 40 MP; record actual worker cgroup peak/RSS.""" +import concurrent.futures +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from client import Client +from images import png + + +def docker(*args):return subprocess.check_output(['docker',*args],text=True).strip() + + +def memory(container): + code="import json;from pathlib import Path;p=Path('/sys/fs/cgroup');print(json.dumps({k:(p/k).read_text().strip() for k in ['memory.current','memory.peak','memory.events','memory.stat','cpu.stat']}))" + return json.loads(docker('exec',container,'python','-c',code)) + + +def main(): + c=Client(timeout=30);a,b,e,chat=c.bootstrap() + profiles=[(320,200,False),(1800,1400,True),(10000,4000,False)] + result=[] + for w,h,noise in profiles: + content=png(w,h,noise=noise) + docker('compose','restart','worker');container=docker('compose','ps','-q','worker') + before=memory(container);samples=[];done=threading.Event();start=time.monotonic() + def sample(): + while not done.is_set(): + samples.append({'seconds':round(time.monotonic()-start,4),'worker':memory(container)}) + done.wait(.05) + sampler=threading.Thread(target=sample);sampler.start() + barrier=threading.Barrier(10) + def upload(i): + barrier.wait();t=time.monotonic() + r=Client(timeout=30).call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=load.png',202,actor=a,raw=content,headers={'Content-Type':'image/png'},schema='Attachment') + accepted=time.monotonic()-t + ready=Client(timeout=30).wait_ready(b,r['json']) + assert ready['sha256']==hashlib.sha256(content).hexdigest() + return {'index':i,'accepted_seconds':accepted,'ready_seconds':time.monotonic()-t,'status':ready['status']} + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: replies=list(pool.map(upload,range(10))) + finally:done.set();sampler.join() + after=memory(container);assert 'oom_kill 0' in after['memory.events'] + result.append({'width':w,'height':h,'bytes':len(content),'clients':10,'before':before,'after':after,'samples':samples,'replies':replies,'seconds':time.monotonic()-start}) + print(json.dumps({'method':'10 simultaneous uploads, worker concurrency=1; cgroup v2 samples include child renderer; anon is resident anonymous memory, memory.peak includes tmpfs/kernel/cache','experiments':result},indent=2)) + + +if __name__=='__main__':main() diff --git a/student-tests/security.py b/student-tests/security.py new file mode 100644 index 00000000..9538f7ea --- /dev/null +++ b/student-tests/security.py @@ -0,0 +1,27 @@ +"""Private S3 and job envelopes do not expose source bytes or credentials.""" +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request +sys.path[:0]=['/app/src',str(Path.cwd()/'tests')] +from client import Client +from pipeline import connect,validate,event +from objects import Objects +from postgres import setting + +c=Client();a,b,e,chat=c.bootstrap();att,raw=c.attachment(a,chat);ready=c.wait_ready(b,att) +with connect() as conn: + row=conn.execute('SELECT object_key, current_job_id FROM attachments WHERE 1=1 AND id = %s',(att['id'],)).fetchone() + job=conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s',(row['current_job_id'],)).fetchone() +message=event(job);message['attempt']=max(1,job['attempt']) +assert validate(message) +assert set(message)=={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} +assert len(json.dumps(message))<1024 +for path in ('/'+Objects(setting).bucket+'/'+row['object_key'],'/'+Objects(setting).bucket+'?list-type=2'): + try:urllib.request.urlopen('http://s3:8333'+path,timeout=2);raise AssertionError('anonymous access') + except urllib.error.HTTPError as error:assert error.code==403 +for key in ['schema_version','pipeline_version','attempt']: + bad=dict(message);bad[key]=True;assert not validate(bad) +for body in [dict(message,extra='x'),dict(message,created_at='bad'),dict(message,traceparent='bad')]:assert not validate(body) +print(json.dumps({'anonymous_object_and_list_403':True,'job_fields':sorted(message),'job_bytes':len(json.dumps(message)),'no_source_bytes_or_credentials_in_envelope':True,'strict_invalid_envelope_checks':True},indent=2)) diff --git a/student-tests/slow_upload.py b/student-tests/slow_upload.py new file mode 100644 index 00000000..c17d2b01 --- /dev/null +++ b/student-tests/slow_upload.py @@ -0,0 +1,32 @@ +"""A continuously trickling client still reaches the absolute upload deadline.""" +import json +from pathlib import Path +import select +import socket +import sys +import time +import uuid +from urllib.parse import urlsplit +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +c = Client() +alice, bob, eve, chat = c.bootstrap() +target = urlsplit(c.base) +connection = socket.create_connection((target.hostname, target.port), timeout=3) +header = 'POST /api/v1/chats/' + chat['id'] + '/attachments?filename=slow_' + uuid.uuid4().hex + '.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ' + alice['token'] + '\r\nContent-Type: image/png\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' +started = time.monotonic() +connection.sendall(header.encode()) +response = b'' +while time.monotonic() - started < 20: + if select.select([connection], [], [], .15)[0]: + response += connection.recv(4096) + if b'\r\n\r\n' in response: + break + connection.sendall(b'x') +connection.close() +elapsed = time.monotonic() - started +assert response.startswith(b'HTTP/1.1 400') or response.startswith(b'HTTP/1.0 400'), response[:100] +assert 14 <= elapsed < 19 +assert b'x-instance-id:' in response.lower() and b'x-request-id:' in response.lower() +print(json.dumps({'trickle_seconds': round(elapsed, 3), 'http_status': 400, 'absolute_deadline_enforced': True})) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..45f0d800 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,177 @@ +"""Extra black-box tests, without changing the course's public tests.""" +import concurrent.futures +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client as PublicClient, LAB + + +class Client(PublicClient): + def call(self, method, path, *args, headers=None, **kwargs): + headers = dict(headers or {}) + if method == 'POST' and path.endswith('/messages'): + headers.setdefault('Idempotency-Key', uuid.uuid4().hex) + return super().call(method, path, *args, headers=headers, **kwargs) + + +class APIContract(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = '/api/v1/chats/' + self.chat['id'] + + def test_malformed_json_and_extra_fields(self): + for raw in (b'{', b'[]', b'{"text":"x","text":"y"}', b'{"text":NaN}', b'{"text":"\\ud800"}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + for body in ({'text': 'ok', 'sender_id': self.eve['id']}, {'text': True}, {'text': ' '}, {'text': ''}, {'text': None}): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body=body) + deep = b'{"text":' + b'[' * 1200 + b'0' + b']' * 1200 + b'}' + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=deep, headers={'Content-Type': 'application/json'}) + + def test_unicode_limits(self): + result = self.client.message(self.alice, self.chat, '👩' * 4000)['json'] + self.assertEqual(result['text'], '👩' * 4000) + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': '👩' * 4001}) + + def test_unicode_nul_round_trip(self): + import base64 + value = 'A\x00B\\u0000👩' + registration = dict(self.alice['registration'], username='nul_' + uuid.uuid4().hex[:20], display_name=value) + user = self.client.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json'] + self.assertEqual(user['display_name'], value) + credential = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(session['user'], user) + actor = {'id': user['id'], 'token': session['token']} + # Exercise the database path and then the Redis profile cache. + for _ in range(2): + self.assertEqual(self.client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json'], user) + chat = self.client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': value, 'member_ids': [actor['id']]}, schema='Chat')['json'] + self.assertEqual(chat['title'], value) + self.assertEqual(self.client.call('GET', '/api/v1/chats/' + chat['id'], actor=actor, schema='Chat')['json'], chat) + contents = [value, '\x00' * 4000, '👩' * 3999 + '\x00'] + for content in contents: + self.assertEqual(self.client.message(actor, chat, content)['json']['text'], content) + history = self.client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=actor, schema='MessagePage')['json'] + self.assertEqual([message['text'] for message in history['items']], contents) + + def test_foreign_chat(self): + for suffix in ('', '/messages'): + self.client.call('GET', self.path + suffix, 404, actor=self.eve) + self.client.call('POST', self.path + '/messages', 404, actor=self.eve, body={'text': 'hidden'}) + self.client.call('GET', '/api/v1/chats/' + str(uuid.uuid4()), 404, actor=self.alice) + + def test_registration_race(self): + body = dict(self.alice['registration'], username='race_' + uuid.uuid4().hex[:20]) + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + statuses = list(executor.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + def test_invalid_chat_creation(self): + members = ([self.alice['id'], self.alice['id']], [self.bob['id']], [self.alice['id'], str(uuid.uuid4())], [], [True]) + for member_ids in members: + self.client.call('POST', '/api/v1/chats', 400, actor=self.alice, body={'title': 'rollback', 'member_ids': member_ids}) + + def test_cursor_scope_and_validation(self): + for i in range(3): + self.client.message(self.alice, self.chat, str(i)) + cursor = self.client.call('GET', self.path + '/messages?limit=1', actor=self.alice, schema='MessagePage')['json']['next_cursor'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'other', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + self.client.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(cursor), 400, actor=self.alice) + for query in ('limit=0', 'limit=101', 'limit=' + '9' * 5000, 'limit=true', 'limit=1&limit=2', 'cursor=', 'cursor=bad', 'cursor=e30=', 'offset=1'): + self.client.call('GET', self.path + '/messages?' + query, 400, actor=self.alice) + + def test_concurrent_messages_and_pagination(self): + def send(i): + return Client().message(self.alice if i % 2 else self.bob, self.chat, 'parallel-' + str(i))['json'] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + sent = list(executor.map(send, range(100))) + found = [] + cursor = None + while True: + url = self.path + '/messages?limit=7' + ('&cursor=' + quote(cursor) if cursor else '') + page = self.client.call('GET', url, actor=self.alice, schema='MessagePage')['json'] + found.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(len(found), 100) + self.assertEqual(len({v['id'] for v in found}), 100) + self.assertEqual({v['id'] for v in found}, {v['id'] for v in sent}) + self.assertEqual(found, sorted(found, key=lambda v: (v['created_at'], v['id']))) + + def test_idempotency_concurrent_replicas(self): + key = uuid.uuid4().hex + def send(_): + return Client().message(self.alice, self.chat, 'Один раз\x00👩', key=key) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertGreaterEqual(len({v['headers']['X-Instance-Id'] for v in replies}), 2) + self.assertTrue(all(v['json'] == replies[0]['json'] for v in replies)) + page = self.client.call('GET', self.path + '/messages', actor=self.bob, schema='MessagePage')['json'] + self.assertEqual(page['items'], [replies[0]['json']]) + self.client.call('POST', self.path + '/messages', 409, actor=self.alice, body={'text': 'different'}, headers={'Idempotency-Key': key}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=self.alice) + self.client.call('POST', self.path + '/messages', 401, actor=self.alice, body={'text': 'Один раз\x00👩'}, headers={'Idempotency-Key': key}) + + def test_protocol_error_headers(self): + self.client.call('TRACE', '/api/v1/users/me', 400, actor=self.alice) + + def test_idempotency_scope(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'same', key=key)['json'] + two = self.client.message(self.bob, self.chat, 'same', key=key)['json'] + other = self.client.call('POST', '/api/v1/chats', 201, actor=self.alice, body={'title': 'scope', 'member_ids': [self.alice['id']]}, schema='Chat')['json'] + three = self.client.message(self.alice, other, 'same', key=key)['json'] + self.assertEqual(len({one['id'], two['id'], three['id']}), 3) + + def test_idempotency_semantic_body(self): + key = uuid.uuid4().hex + one = self.client.message(self.alice, self.chat, 'semantic', key=key)['json'] + two = self.client.call('POST', self.path + '/messages', 201, actor=self.alice, body={'attachment_ids': [], 'text': 'semantic'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + self.assertEqual(one, two) + + def test_idempotency_invalid_keys(self): + for key in (None, '', 'x' * 129, 'space key', 'bad,key', 'slash/key'): + headers = {} if key is None else {'Idempotency-Key': key} + PublicClient().call('POST', self.path + '/messages', 400, actor=self.alice, body={'text': 'valid'}, headers=headers) + for raw in (b'{"text":true}', b'{"text":"ok","extra":1}', b'{"text":"ok","attachment_ids":false}'): + self.client.call('POST', self.path + '/messages', 400, actor=self.alice, raw=raw, headers={'Content-Type': 'application/json'}) + + def test_idempotency_conflicting_race(self): + key = uuid.uuid4().hex + def send(i): + return Client().call('POST', self.path + '/messages', [201, 409], actor=self.alice, body={'text': str(i % 2)}, headers={'Idempotency-Key': key}, schema=None) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + replies = list(executor.map(send, range(32))) + self.assertEqual(sum(v['status'] == 201 for v in replies), 16) + self.assertEqual(sum(v['status'] == 409 for v in replies), 16) + page = self.client.call('GET', self.path + '/messages', actor=self.alice, schema='MessagePage')['json'] + self.assertEqual(len(page['items']), 1) + + @unittest.skipUnless(LAB >= 2, 'sessions start in lab2') + def test_unicode_password_basic_and_logout(self): + import base64 + registration = {'username': 'unicode_' + uuid.uuid4().hex[:16], 'display_name': 'Софья', 'password': '🔐' * 127 + ':'} + self.client.call('POST', '/api/v1/users', 201, body=registration) + encoded = base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode() + headers = {'Authorization': 'Basic ' + encoded} + session = self.client.call('POST', '/api/v1/auth/sessions', 201, headers=headers, schema='Session')['json'] + actor = {'id': session['user']['id'], 'token': session['token']} + self.client.call('GET', '/api/v1/users/me', actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, headers=headers) + for authorization in ('Basic !!!', 'Bearer wrong', '', 'Basic Zm9v'): + self.client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': authorization}) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 00000000..f27c0393 --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,52 @@ +"""Independent tests for async image metadata, transformations and access.""" +import io +from pathlib import Path +import sys +import unittest +import uuid +from urllib.parse import quote +sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'tests')] +from client import Client +from images import bomb + + +class Images(unittest.TestCase): + def setUp(self): + self.c=Client(); self.alice,self.bob,self.eve,self.chat=self.c.bootstrap() + self.path='/api/v1/chats/'+self.chat['id'] + self.raw=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + + def test_async_variants_and_replay(self): + queued,_=self.c.attachment(self.alice,self.chat) + self.assertEqual(queued['status'],'queued') + key=uuid.uuid4().hex + one=self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'] + self.assertEqual(self.c.message(self.bob,self.chat,'',key=key,attachments=[queued['id']])['json'],one) + ready=self.c.wait_ready(self.alice,queued) + self.assertEqual({v['name'] for v in ready['variants']},{'thumbnail','crop','optimized'}) + dimensions={v['name']:(v['width'],v['height']) for v in ready['variants']} + self.assertEqual(dimensions,{'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}) + for v in ready['variants']: + import hashlib + r=self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant='+v['name'],actor=self.bob) + self.assertEqual(hashlib.sha256(r['bytes']).hexdigest(),v['sha256']) + self.assertEqual(len(r['bytes']),v['size_bytes']) + self.c.call('GET','/api/v1/attachments/'+ready['id']+'/content?variant=absent',404,actor=self.bob) + + def test_mime_bytes_pixels_and_filename(self): + path=self.path+'/attachments?filename=test.png' + for raw,mime,status in [(b'not image','image/png',415),(self.raw[:20],'image/png',415),(self.raw,'image/jpeg',415),(bomb(),'image/png',413),(b'x'*(10485760+1),'image/png',413)]: + self.c.call('POST',path,status,actor=self.alice,raw=raw,headers={'Content-Type':mime}) + for name in ('../x.png','a\\x.png','a\0.png','a\n.png','a'*256,''): + self.c.call('POST',self.path+'/attachments?filename='+quote(name,safe=''),400,actor=self.alice,raw=self.raw,headers={'Content-Type':'image/png'}) + + def test_private_and_cross_chat(self): + att,_=self.c.attachment(self.alice,self.chat) + for suffix in ('','/content'): + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=self.eve) + self.c.call('GET','/api/v1/attachments/'+att['id']+suffix,401) + other=self.c.call('POST','/api/v1/chats',201,actor=self.alice,body={'title':'other','member_ids':[self.alice['id']]},schema='Chat')['json'] + self.c.call('POST','/api/v1/chats/'+other['id']+'/messages',404,actor=self.alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + + +if __name__=='__main__': unittest.main(verbosity=2)