diff --git a/.env.example b/.env.example index d99d26a2..47c4f7ff 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,8 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# scripts/bootstrap.sh generates LOCAL random passwords, never overwrites .env. +POSTGRES_PASSWORD=choose_local_random_password +S3_ROOT_PASSWORD=choose_separate_admin_password +S3_API_PASSWORD=choose_separate_application_password +APP_PORT=8084 API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +API_CPUS=2 +API_IMAGE=mtusi-lab4-api:local diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..35a2feec 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,17 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор: Савин Александр, БВТ2303. Целевая оценка: **5, трек 2 — согласованность двух хранилищ**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Добавлены приватный MinIO с отдельными API credentials, синхронный потоковый PNG/JPEG upload, метаданные PostgreSQL, ссылки вложений в сообщениях и авторизованный streaming-download. Минимум lab3 сохранён: две API за HAProxy, общие сессии, идемпотентность и отдельные миграции. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Разрыв PUT/COMMIT обрабатывается durable pending-намерением и повторяемым reconciler с dry-run, grace и общими object locks. Исследование внедряет реальные и моделируемые сбои, проверяет защиту fresh/used/locked объектов, повтор apply, восстановление missing и стоимость обхода. Дополнительно измерен пик памяти при 10 одновременных uploads двух размеров. + +Запуск: scripts/bootstrap.sh → scripts/compose.sh up -d --build → scripts/wait_ready.py. Полные команды и результаты: REPORT.md; методика, raw data и графики: RESEARCH.md и evidence/. + +Ограничения: один host, нет репликации MinIO/БД/LB, backup и полного SHA-аудита всех bytes; готовые неиспользуемые вложения автоматически не удаляются. Использовался Codex, первичные материалы указаны в отчёте. + +- [x] Обязательный контракт, README и публичные тесты сохранены. +- [x] Секреты вне Git; root credentials не передаются API. +- [x] Реальные S3/DB outages и разрыв PUT/COMMIT проверены. +- [x] Публичный smoke, файлы, ACL и сохранность проверяются отдельными сценариями. +- [x] Исследование содержит гипотезу, raw data, повторения, результаты и ограничения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..88d98f3e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + DATABASE_URL=postgresql://messenger:messenger@db:5432/messenger \ + SESSION_TTL_SECONDS=3600 + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt +ARG APP_REVISION=lab4-v1 +ENV APP_REVISION=$APP_REVISION +COPY app.py attachments.py reconcile.py migrate.py /app/ +COPY migrations /app/migrations +COPY scripts /app/scripts +COPY tests /app/tests +COPY contracts /app/contracts +COPY lab.json /app/lab.json +EXPOSE 8080 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 5d86d171..affd024a 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,47 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: up down test-idempotency test-cache test-uploads test-private test-attachment-failover test-dependencies research-memory research-consistency reconcile reconcile-apply +up: + ./scripts/bootstrap.sh + ./scripts/compose.sh up -d --build + +# Preserve volumes; data deletion is an explicit operator action. +down: + ./scripts/compose.sh down + +test-idempotency: + $(PYTHON) tests/idempotency_concurrent.py --base-url "$(BASE_URL)" + +test-cache: + ./scripts/compose.sh exec -T -e BASE_URL=http://lb:8080 -e CACHE_URL=redis://valkey:6379/0 api python - < tests/cache_consistency.py + +test-uploads: + ./scripts/compose.sh run --rm -T --no-deps ops python tests/uploads_negative.py + +test-private: + ./scripts/compose.sh run --rm -T --no-deps ops python tests/private_storage.py + +test-attachment-failover: + $(PYTHON) tests/attachment_failover.py --base-url "$(BASE_URL)" + +test-dependencies: + $(PYTHON) scripts/test_dependency_outages.py + +research-memory: + $(PYTHON) scripts/research_upload_memory.py + +research-consistency: + $(PYTHON) scripts/research_consistency_host.py + +reconcile: + ./scripts/compose.sh run --rm -T --no-deps ops python reconcile.py + +reconcile-apply: + ./scripts/compose.sh run --rm -T --no-deps ops python reconcile.py --apply + +.PHONY: plot-research +plot-research: + $(PYTHON) scripts/plot_research.py + $(PYTHON) scripts/summarize_research.py diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..8d949f82 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,228 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4 — S3-хранилище и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: Савин Александр, БВТ2303. Целевая оценка: 5, трек 2 — согласованность двух хранилищ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine/Compose v2 и Python 3.9+ для внешних тестов; для перестроения графиков — Python 3.10+ (локально 3.13). Сборка использует Python 3.11.11. Скрипт compose.sh поддерживает `docker compose` и `docker-compose`. -## Архитектура и решения +```sh +cd mtusi2026lab4 +./scripts/bootstrap.sh +./scripts/compose.sh config --quiet +./scripts/compose.sh up -d --build +python3 scripts/wait_ready.py --base-url http://localhost:8084 +export BASE_URL=http://localhost:8084 +make check +make test +make test-idempotency +make test-cache +make test-uploads +make test-private +make test-attachment-failover +./scripts/compose.sh start api +python3 scripts/wait_ready.py --base-url "$BASE_URL" +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +./scripts/compose.sh start api +python3 scripts/wait_ready.py --base-url "$BASE_URL" +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-dependencies +make research-consistency +make research-memory +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Все зависимости запускаются этим проектом. .env создаётся один раз с тремя разными случайными паролями и правами 600; существующий файл не перезаписывается. APP_PORT по умолчанию 8084. Хранилища/volumes предыдущих лабораторных не подключаются. API_REPLICAS, API_CPUS и API_IMAGE управляют числом, CPU и образом API; остальные обязательные значения описаны в compose.yaml/.env.example. Секреты исключены из Git и Docker build context. Credentials в environment доступны оператору Docker — учебное упрощение. -## Доказательства критериев +Остановка без потери данных: `./scripts/compose.sh down`; возврат: `./scripts/compose.sh up -d`. Намеренное удаление **всех учебных данных lab4**: `./scripts/compose.sh down -v`; для проверок команда не нужна. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-research.txt +make validate PYTHON=.venv/bin/python +.venv/bin/python scripts/plot_research.py +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Компоненты и версии -## Эксперименты и отказы +Клиент → HAProxy → API ×2 → PostgreSQL (метаданные) и приватный MinIO (оригиналы). Valkey остаётся общим кэшем сессий. Мигратор и S3 bootstrap запускаются отдельно; reconciler автоматически сверяет хранилища. ops — профиль инструментов для тестов/ручной сверки, обычный запуск его не поднимает. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Компонент | Проверенная версия/ограничение | +| --- | --- | +| API | Python 3.11.11-slim, FastAPI 0.115.0, Uvicorn 0.30.6; 2 CPU/512 MiB на реплику | +| PostgreSQL | 16.6-alpine; 1 CPU/512 MiB | +| Valkey | 8.0.2-alpine, AOF; 0.25 CPU/128 MiB | +| HAProxy | 3.0.5-alpine; 0.25 CPU/64 MiB | +| MinIO | RELEASE.2025-09-07T16-13-09Z, digest закреплён в Compose; 0.5 CPU/768 MiB | +| MinIO mc | RELEASE.2025-08-13T08-35-41Z, digest закреплён; bootstrap 1 CPU/512 MiB, затем завершается | +| Reconciler | Тот же образ приложения; 0.25 CPU/256 MiB | +| Инструменты ops | 1 CPU/512 MiB, запускаются явно | +| SDK/декодер | boto3 1.42.0, Pillow 13.0.0 | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Проверено на macOS/Colima, Linux arm64, Docker 27.4.0, VM 4 vCPU и около 8 GB RAM. Квоты — пределы, не резервирование: сумма возможных CPU-квот больше 4 vCPU. Все образы закреплены тегом или digest; проверена arm64-среда. Фактические процессы, image IDs, лимиты, порты и volumes — evidence/final-containers.json. -## Ограничения и следующие шаги +## Что перенесено из lab3 + +Две взаимозаменяемые API за HAProxy, Docker DNS discovery, round robin, общие сессии PostgreSQL/Valkey, Argon2id, проверка прав, keyset-пагинация, X-Request-Id/X-Instance-Id, транзакционная идемпотентность сообщений и отдельный согласованный мигратор. SQL-пул каждой API ограничен пятью соединениями. + +Новые текстовые сообщения остаются валидны и получают attachment_ids=[]. Миграция 003 переводит сохранённые lab3 request bodies в JSONB и добавляет [] к сохранённым ответам, чтобы сохранить replay при таком обновлении. В этом стенде используется отдельная новая БД lab4. + +Идемпотентность теперь сравнивает текст и упорядоченный список attachment_ids. Порядок JSON-полей несущественен, отсутствующий список равен []. Те же ключ/тело дают прежнее сообщение и 201; другой body — 409. Ключ действует в области пользователя/чата/POST, срок результата — 24 часа. Вложения проверяются в транзакции, их принадлежность чату обязательна. Один и тот же файл чата может прикреплять другой участник, не только загрузивший пользователь. + +## Метаданные, загрузка и download + +attachments хранит UUID, чат, владельца, отображаемое filename, внутренний object_key, тип, размер, SHA-256 и состояние. message_attachments хранит связь и порядок вложений. Бинарного поля в PostgreSQL нет. Имя объекта назначается сервером: originals/. Filename не участвует в пути. + +Порядок upload: + +1. Bearer-аутентификация и проверка членства в чате. +2. Проверка filename (1–255 символов, без разделителей и управляющих символов) и допустимого Content-Type. +3. Отклонение слишком большого Content-Length до чтения тела. Затем потоковое чтение с повторной проверкой суммарных bytes, в том числе при chunked transfer. +4. SHA-256 рассчитывается по поступающим байтам. SpooledTemporaryFile держит до 256 KiB в RAM, дальше использует временный файл. Исходник не становится обязательным локальным хранилищем API: spool закрывается/удаляется после операции, ошибки, disconnect или завершения процесса. +5. Pillow открывает только PNG/JPEG, проверяет фактический формат, размеры до полного decode, затем verify и полный load. Лимиты — 10 MiB и 40 млн пикселей; обрезанные JPEG тоже отклоняются. +6. Создание устойчивого pending-намерения в PostgreSQL, S3 PUT и перевод записи в ready. 201 возвращается после успешной фиксации ready, когда обе части сохранены. + +Размер и пиксели ограничены константами обязательного контракта. Multipart/form-data не используется. Превышение limits — 413; испорченное/чужое содержимое или подмена MIME — 415; неверные поля — 400; чужой/неизвестный объект — 404. Ошибки SDK не раскрывают credentials и превращаются в 503. + +GET /attachments/{id} и /content сначала проверяют сессию и членство в чате. Download идёт через API без redirect и отдаёт исходные байты порциями 64 KiB с настоящим Content-Type. В ответах отсутствуют object key, адрес MinIO и секреты. Только ready выдаётся как Attachment обязательной схемы с variants=[] и error_code=null. Для внутреннего pending/failed доступ участнику возвращает 409; чужой пользователь по-прежнему получает 404. При исчезновении объекта download сначала возвращает 503, после обнаружения reconciler — 409. Ссылка в подтверждённом сообщении сохраняется. + +## Ограничение ресурсов и timeouts + +На процесс допускаются две активные upload-операции; остальные ждут свободный слот до 30 секунд с ASGI-backpressure. При чтении тела — idle timeout 5 секунд и общий бюджет 30 секунд. Декодирование ограничено одним изображением на процесс. Блокирующие file/Pillow/S3-операции вынесены в thread pool. Uvicorn limit-concurrency=64 ограничивает общее число соединений/задач; при перегрузке возможна 503 самого сервера. + +S3 SDK: connect timeout=1s, read timeout=3s, одна попытка без скрытого повторного PUT. Применяется single PUT, multipart-upload-сессии не создаются. Ресурс декодирования ограничен bytes/pixels и семафором; отдельного принудительного wall-clock timeout нативного Pillow decode нет. + +API readiness требует PostgreSQL и MinIO; Valkey остаётся необязательным. HAProxy проверяет readiness каждую секунду, fall=2/rise=1/check timeout=1s; connect=1s, client/server inactivity timeout=45s. Docker healthcheck имеет процессный бюджет 4s. Uvicorn graceful shutdown=35s, Compose stop_grace_period=40s. Явный stop-one hook намеренно использует лишь 3s, проверяя и возможность обрыва запросов. SIGTERM reconciler завершает ожидание и закрывает соединения; при слишком долгом проходе возможен внешний SIGKILL, повтор остаётся безопасным. + +## Приватность MinIO и bootstrap + +MinIO и его консоль не публикуют порты на хост. Консоль привязана к loopback внутри S3-контейнера, browser UI выключен. API знает отдельного пользователя lab4api; root credentials есть только у MinIO и bootstrap, в API их нет. + +S3-init создаёт bucket messenger-attachments, убирает anonymous policy и создаёт/обновляет пользователя и политику. Повтор bootstrap не удаляет объекты. Разрешены ListBucket только этого bucket, Get/Put/DeleteObject только originals/*. У API нет прав создавать bucket, читать его политику или писать за пределами префикса. MinIO разрешает filtered ListBuckets с именем разрешённого bucket; это наблюдение отдельно записано, отказ в этой конкретной операции не заявляется. + +Реальная внешняя для API проверка выполняется из ops-контейнера непосредственно к MinIO: анонимные list/get возвращают 403; административные операции API-пользователя запрещены; console port недоступен. Политика — infra/api-policy.json, результаты — evidence/private-storage.json. + +## Согласованность PostgreSQL и S3 + +Распределённой транзакции между PostgreSQL и S3 нет. Вместо немедленного удаления объекта при любой ошибке COMMIT используется устойчивое намерение и повторяемая сверка. Это важно: при потере ответа COMMIT мог реально пройти, поэтому слепая компенсация могла бы удалить уже готовый объект. + +Перед PUT сохраняется pending со случайным UUID, ключом, размером и checksum. На время PUT/ready COMMIT берётся PostgreSQL advisory lock области object_key. Reconciler использует тот же lock и повторно читает обе стороны под ним. После успешного COMMIT запись ready больше автоматически не удаляется. + +| Расхождение | Безопасное действие | +| --- | --- | +| Объект есть, записи нет (orphan) | После grace и повторной проверки удалить объект, если нет использования | +| Старый pending, PUT не было или процесс умер после PUT | Удалить оставшийся объект, если он есть, отметить incomplete_upload; сохранить запись | +| Ready-метаданные есть, объект отсутствует | Отметить missing_object; сохранить UUID, метаданные и связи сообщений | +| Не совпадают размер/sha256 metadata в HEAD | Отметить corrupt_object, сохранить данные для диагностики | +| Missing/corrupt объект восстановлен и HEAD снова соответствует записи | Вернуть ready; не генерировать новый UUID и не менять сообщения | +| Свежий объект/запись, используемый готовый объект, занятый lock | Не удалять | +| Ошибка БД, S3 403/5xx/timeout | Прервать проход; не трактовать как отсутствие | + +Штатный grace=300s, проверяется возраст и объекта, и pending-записи. Обычный CLI запрещает grace меньше 60s. Автопроцесс выполняет apply, затем ждёт 60s до следующего прохода. Одновременные запуски сериализуются отдельным lock. Инвентарь S3 и метаданные читаются страницами по 200; JSON содержит максимум 1000 подробных действий и счётчики остальных, чтобы отчёт тоже не рос без ограничения RAM. + +```sh +make reconcile # dry-run по умолчанию +make reconcile-apply # те же правила с реальным исправлением +``` + +В исследовании только на изолированных синтетических данных grace=5s разрешён явным флагом; это не настройка штатного worker. Само восстановление утраченных bytes требует сохранившегося оригинала/backup. Reconciler обнаруживает и корректно обозначает потерю, но не создаёт исходное изображение из хэша. Проверка HEAD не является полным повторным SHA-аудитом всех байтов: размер и серверные sha256 metadata сравниваются с БД. + +## Доказательства уровней 3 и 4 + +| Критерий | Проверка | Файл | +| --- | --- | --- | +| Контракт и схема | make check; make validate | evidence/check.txt, validate.txt | +| Обязательный lab4 API | make test | evidence/smoke.txt | +| Сессии/идемпотентность lab3 | make test-idempotency; make test-cache | evidence/idempotency.txt, cache-consistency.txt | +| Форматы, limits, filename, ACL, чужой чат/attachment, прерывание | make test-uploads, 21 случая | evidence/uploads-negative.json | +| Приватный bucket и ограниченные права | make test-private | evidence/private-storage.json | +| Установленная загрузившая API действительно остановлена | make test-attachment-failover | evidence/attachment-failover.json | +| Публичный failover и исходный файл | make test-failover ACTION_SCRIPT=scripts/stop-one.sh | evidence/failover.txt, after-public-failover.json | +| Полный restart с прежним токеном и файлом | make test-persistence ACTION_SCRIPT=scripts/restart.sh | evidence/persistence.txt, before-persistence.json, after-persistence.json | +| Реальная недоступность MinIO и PostgreSQL | make test-dependencies | evidence/dependency-outages.json, dependency-outages-compose.txt | +| ≥10 одновременных uploads и peak RAM | make research-memory | evidence/upload-memory.json, upload-memory.png | +| Компенсация и безопасный grace | make research-consistency | evidence/reconciliation.json | +| Повторяемый bootstrap и отсутствие публичной консоли | s3-init + test-private | infra/init-s3.sh, evidence/private-storage.json | + +В outage-тесте диагностические запросы идут внутри Compose к api:8080: это позволяет проверить структурированные ошибки приложения, когда HAProxy уже исключил обе API из-за недоступного S3/БД. Обычный маршрут всех успешных пользовательских сценариев — LB. + +Все таблицы результатов и оба графика включены ниже в этот отчёт; полная методика уровня 5 — в [RESEARCH.md](RESEARCH.md). Проверки изменяют только синтетические данные отдельного проекта lab4. Токены/подписанные URL/пароли в evidence не сохраняются. + +## Фактические результаты + +- Публичный smoke, 16 конкурентных повторов сообщений и регрессии идентичности кэша — PASS. +- Негативный suite: 21 проверка — PASS. Приватный S3 GET/List: 403, опасные операции API principal запрещены, console недоступна. +- Остановка загрузившей API: uploader `e2b4291a5ae6`, скачавшая реплика `f73f3d296d3e`; восстановление download+write 2.159s, неудачных попыток 3. Bytes и SHA-256 совпали, настоящий uploader оставался остановленным до отдельного восстановления. +- Публичный failover: 111 успешных записей, 3 неудачные попытки, восстановление 1.846s. Подтверждённые сообщения, прежняя сессия и оригинал сохранены. +- Persistence — PASS. Реально перезапущены семь процессов API×2/DB/Valkey/MinIO/LB/reconciler; ID и volumes сохранились, StartedAt изменился. +- Десять одновременных uploads, 40 запросов двух размеров: 0 ошибок. Измеренные пики API: около 91 MiB и 125–146 MiB при лимите 512 MiB. +- Трек 2: real crash после PUT завершился с кодом 86; dry-run не изменил данные; apply исправил расхождения; повтор — 0 новых действий; fresh/used/locked объекты сохранены. + + +## Таблицы и графики экспериментов + +Числа и рисунки ниже получены из сохранённых JSON. Полная методика и исходные условия приведены в [RESEARCH.md](RESEARCH.md). + +### Обнаружение и исправление + +| Действие | Количество в apply | +| --- | --- | +| abort_pending | 2 | +| corrupt_object | 1 | +| delete_orphan | 1 | +| missing_object | 1 | + +Реальный fault-процесс завершился с кодом **86** после PUT. Последующий apply нашёл его pending и удалил объект после grace. Для исчезнувшего/повреждённого используемого вложения сохраняются метаданные и связи сообщений; меняется доступность, а не история. После восстановления исходного объекта действие restore_ready снова разрешило скачивание с тем же UUID. + +| Проверка безопасности | Результат | +| --- | --- | +| fresh_orphan_preserved | True | +| fresh_ready_preserved | True | +| used_ready_preserved | True | +| active_lock_preserved | True | +| missing_metadata_and_links_retained | True | +| dry_run_no_mutation | True | +| second_apply_actions | 0 | + +Повторный apply не предложил новых действий. Это относится к уже исправленному тестовому состоянию; при появлении новых объектов/ошибок следующий проход закономерно может выполнить новые действия. Ошибки зависимостей в отдельной проверке не изменили статусы. Реальные остановки MinIO/PostgreSQL дополнительно проверены в evidence/dependency-outages.json. + +### Стоимость обхода + +К существующему фиксированному набору метаданных добавлялись 0, 50, 200 S3-объектов без метаданных. Это отдельные известные test-only ключи. Перед нулевой группой базовые timestamps выдерживаются за пределами тестового grace; после создания каждой следующей группы также выдерживается grace. Для каждого размера выполнено три dry-run, затем известные дополнительные ключи удалены. Порядок размеров возрастающий, кэш не сбрасывался. + +Время измеряется time.monotonic внутри инструмента; запуск контейнера, подготовка fixtures и ожидание grace не входят в elapsed. Исходные отдельные результаты: reconciliation.json → scan_cost_runs. В таблице среднее/выборочное стандартное отклонение, не доверительный интервал. + +| Дополнительных orphan | Объектов S3 | Метаданных | Время, мс: mean ± SD | min–max, мс | List / HEAD вызовов | +| --- | --- | --- | --- | --- | --- | +| 0 | 65 | 65 | 160.59 ± 10.962 | 151.176–172.625 | 1 / 130 | +| 50 | 115 | 65 | 259.99 ± 62.361 | 219.672–331.819 | 1 / 180 | +| 200 | 265 | 65 | 579.413 ± 113.494 | 460.906–687.121 | 2 / 330 | + +![Стоимость обхода](evidence/scan-cost.png) + +При росте набора увеличиваются число point-checks и время. Список S3 и DB-метаданные читаются страницами по 200, подробный журнал ограничен 1000 действиями; остальные учитываются счётчиками. Алгоритм делает O(N+M) обход при N объектах и M подходящих метаданных, с индексированными DB point lookups. Полный SHA-аудит bytes сюда не входит: HEAD сравнивает размер и записанный uploader-ом sha256 metadata. Результат не доказывает предельную производительность хранилища или точную линейность на миллионах объектов. + +### Эксперимент уровня 4: память при параллельной загрузке + +Десять клиентов одновременно стартуют по barrier, два раунда для каждого из двух размеров. Генератор находится внутри Compose-сети и идёт через LB. Каждая загрузка проверяет 201/ready, размер и SHA-256; один оригинал каждой фазы скачивается вторым участником и сравнивается побайтно. + +Изображения — реальные синтетические RGB PNG, однотонные, сохранены без сжатия (compress_level=0), чтобы получить воспроизводимый большой файл. Это профиль передачи/памяти, а не worst-case для времени сложного JPEG/PNG-декодирования. Payload создаётся один раз на фазу и разделяется клиентскими потоками. + +В каждом API cgroup memory.current читается раз в 50ms. Измерение включает page cache, kernel charges и небольшой Python sampler; это не RSS/heap-профиль. Пики — максимумы samples внутри фазы, не математическая гарантия отсутствия краткого пропущенного пика. В raw traces отдельно есть lifetime memory.peak, который включает более ранние операции. Лимит каждой API — 512 MiB, активных upload slots=2, decoder=1. + +| Файл, байт | Размер, MiB | Успешных загрузок | Ошибок | p50 / p95 / p99, мс | Пики API 1 / API 2, MiB | +| --- | --- | --- | --- | --- | --- | +| 787271 | 0.751 | 20 | 0 | 102.767 / 176.198 / 183.16 | 91.094 / 91.312 | +| 7081758 | 6.754 | 20 | 0 | 617.87 / 832.535 / 896.355 | 124.629 / 146.086 | + +![Параллельные загрузки и память](evidence/upload-memory.png) + +Все 40 запросов прошли, в каждой фазе наблюдались две настоящие API. Измеренные пики ниже лимита. Это подтверждает выбранный профиль ≥10 конкурентных загрузок, но не доказывает пик для всех изображений вплоть до 40 млн пикселей или для одновременных массовых логинов Argon2id. + + + +## Ограничения + +Один Docker-host; MinIO, PostgreSQL и LB не реплицированы. Поддержан отказ процесса API и обычный restart с volumes, а не потеря диска/узла/backup. HTTP используется для локального синтетического стенда. Краткие ошибки при обнаружении отказа учитываются. Обрыв S3 в середине уже начатого streaming-download может оборвать HTTP-тело после отправки headers; клиент сверяет длину/хэш и повторяет безопасный GET. + +Готовые, но ещё не прикреплённые вложения автоматически не удаляются: клиент может сослаться на них позже. Failed-записи и связи сохраняются для диагностики; tombstone retention отдельно не реализован. Изменять приватные объекты напрямую допускается только оператору; ключи приложения неизменяемы и не переиспользуются. Защита от произвольных злонамеренных изменений root/SQL-администратора и от несогласованного восстановления всей БД вне модели этой работы. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. diff --git a/RESEARCH.md b/RESEARCH.md new file mode 100644 index 00000000..e4ce2f0b --- /dev/null +++ b/RESEARCH.md @@ -0,0 +1,110 @@ +# Исследование lab4: согласованность PostgreSQL и S3 + +Автор: Савин Александр, БВТ2303. Уровень 5, трек 2. Выполнено 6 ноября 2026 года на отдельном стенде lab4. Исходные данные находятся в evidence/, обязательные API-тесты не менялись. + +## Гипотеза + +Если устойчивое намерение создаётся до S3 PUT, а uploader и reconciler используют одну область транзакционной блокировки, можно обнаруживать незавершённые операции и расхождения без удаления свежих, используемых или ещё записываемых объектов. Повторное применение к уже исправленному состоянию не должно создавать новых изменений. + +Дополнительная гипотеза: стоимость полной сверки растёт приблизительно с числом объектов и метаданных, а не с размером каждого оригинала, поскольку сверка использует List/HEAD и не скачивает все байты. + +## Среда и методика + +macOS/Colima, Linux arm64, Docker Engine 27.4.0, VM 4 vCPU/около 8 GB RAM. Версии и caps всех компонентов закреплены в compose.yaml, SDK — boto3 1.42.0. Исследовательский ops-контейнер: 1 CPU/512 MiB; обычный reconciler: 0.25 CPU/256 MiB, поэтому их абсолютное время не обязано совпадать. MinIO: 0.5 CPU/768 MiB, PostgreSQL: 1 CPU/512 MiB. + +scripts/research_consistency_host.py временно останавливает штатный worker, запускает один изолированный эксперимент и восстанавливает worker в finally. Пользователи, чаты, файлы и fault fixtures синтетические. Прямые повреждения выполняются только исследовательским инструментом с доступом к двум хранилищам; HTTP-клиент не может включить failpoint. + +Фазы опыта: + +1. Загрузить valid PNG и связать часть вложений с сообщениями. +2. Удалить метаданные одного неиспользуемого вложения, оставив S3-объект: orphan. +3. Запустить отдельный Python-процесс, который делает настоящий PUT и завершает себя os._exit(86) до ready COMMIT. Проверить pending и существующий объект. +4. Создать pending без объекта, моделируя неуспешный PUT. +5. Удалить объект используемого вложения; заменить другой объект неверным содержимым. +6. Состарить DB timestamps, дождаться реального старения S3 timestamps. Использовать grace=5s только с явным разрешением для теста; штатный grace остаётся 300s. +7. Создать свежие ready/orphan fixtures и удерживать общий advisory lock для старого объекта, моделируя активный uploader. +8. Выполнить dry-run и сравнить состояния/наличие: мутаций быть не должно. Затем apply с повторной проверкой обеих сторон под lock. +9. Повторить apply; восстановить потерянный оригинал с его SHA metadata и вернуть ready без изменения UUID/связей. +10. Имитировать S3 503 и ошибку соединения PostgreSQL: проход должен завершиться ошибкой без вывода «объект отсутствует» и без изменения статусов. + +В тесте перед отдельными sweep обновляется timestamp test-only fresh orphan, чтобы на начало каждого прохода он действительно был моложе grace. Заблокированный тестовый orphan удаляется оператором только после проверки его защиты. Это отличает проверку текущей свежести от обещания хранить неподтверждённый orphan бесконечно. + +## Обнаружение и исправление + +| Действие | Количество в apply | +| --- | --- | +| abort_pending | 2 | +| corrupt_object | 1 | +| delete_orphan | 1 | +| missing_object | 1 | + +Реальный fault-процесс завершился с кодом **86** после PUT. Последующий apply нашёл его pending и удалил объект после grace. Для исчезнувшего/повреждённого используемого вложения сохраняются метаданные и связи сообщений; меняется доступность, а не история. После восстановления исходного объекта действие restore_ready снова разрешило скачивание с тем же UUID. + +| Проверка безопасности | Результат | +| --- | --- | +| fresh_orphan_preserved | True | +| fresh_ready_preserved | True | +| used_ready_preserved | True | +| active_lock_preserved | True | +| missing_metadata_and_links_retained | True | +| dry_run_no_mutation | True | +| second_apply_actions | 0 | + +Повторный apply не предложил новых действий. Это относится к уже исправленному тестовому состоянию; при появлении новых объектов/ошибок следующий проход закономерно может выполнить новые действия. Ошибки зависимостей в отдельной проверке не изменили статусы. Реальные остановки MinIO/PostgreSQL дополнительно проверены в evidence/dependency-outages.json. + +## Стоимость обхода + +К существующему фиксированному набору метаданных добавлялись 0, 50, 200 S3-объектов без метаданных. Это отдельные известные test-only ключи. Перед нулевой группой базовые timestamps выдерживаются за пределами тестового grace; после создания каждой следующей группы также выдерживается grace. Для каждого размера выполнено три dry-run, затем известные дополнительные ключи удалены. Порядок размеров возрастающий, кэш не сбрасывался. + +Время измеряется time.monotonic внутри инструмента; запуск контейнера, подготовка fixtures и ожидание grace не входят в elapsed. Исходные отдельные результаты: reconciliation.json → scan_cost_runs. В таблице среднее/выборочное стандартное отклонение, не доверительный интервал. + +| Дополнительных orphan | Объектов S3 | Метаданных | Время, мс: mean ± SD | min–max, мс | List / HEAD вызовов | +| --- | --- | --- | --- | --- | --- | +| 0 | 65 | 65 | 160.59 ± 10.962 | 151.176–172.625 | 1 / 130 | +| 50 | 115 | 65 | 259.99 ± 62.361 | 219.672–331.819 | 1 / 180 | +| 200 | 265 | 65 | 579.413 ± 113.494 | 460.906–687.121 | 2 / 330 | + +![Стоимость обхода](evidence/scan-cost.png) + +При росте набора увеличиваются число point-checks и время. Список S3 и DB-метаданные читаются страницами по 200, подробный журнал ограничен 1000 действиями; остальные учитываются счётчиками. Алгоритм делает O(N+M) обход при N объектах и M подходящих метаданных, с индексированными DB point lookups. Полный SHA-аудит bytes сюда не входит: HEAD сравнивает размер и записанный uploader-ом sha256 metadata. Результат не доказывает предельную производительность хранилища или точную линейность на миллионах объектов. + +## Эксперимент уровня 4: память при параллельной загрузке + +Десять клиентов одновременно стартуют по barrier, два раунда для каждого из двух размеров. Генератор находится внутри Compose-сети и идёт через LB. Каждая загрузка проверяет 201/ready, размер и SHA-256; один оригинал каждой фазы скачивается вторым участником и сравнивается побайтно. + +Изображения — реальные синтетические RGB PNG, однотонные, сохранены без сжатия (compress_level=0), чтобы получить воспроизводимый большой файл. Это профиль передачи/памяти, а не worst-case для времени сложного JPEG/PNG-декодирования. Payload создаётся один раз на фазу и разделяется клиентскими потоками. + +В каждом API cgroup memory.current читается раз в 50ms. Измерение включает page cache, kernel charges и небольшой Python sampler; это не RSS/heap-профиль. Пики — максимумы samples внутри фазы, не математическая гарантия отсутствия краткого пропущенного пика. В raw traces отдельно есть lifetime memory.peak, который включает более ранние операции. Лимит каждой API — 512 MiB, активных upload slots=2, decoder=1. + +| Файл, байт | Размер, MiB | Успешных загрузок | Ошибок | p50 / p95 / p99, мс | Пики API 1 / API 2, MiB | +| --- | --- | --- | --- | --- | --- | +| 787271 | 0.751 | 20 | 0 | 102.767 / 176.198 / 183.16 | 91.094 / 91.312 | +| 7081758 | 6.754 | 20 | 0 | 617.87 / 832.535 / 896.355 | 124.629 / 146.086 | + +![Параллельные загрузки и память](evidence/upload-memory.png) + +Все 40 запросов прошли, в каждой фазе наблюдались две настоящие API. Измеренные пики ниже лимита. Это подтверждает выбранный профиль ≥10 конкурентных загрузок, но не доказывает пик для всех изображений вплоть до 40 млн пикселей или для одновременных массовых логинов Argon2id. + +## Воспроизведение + +```sh +./scripts/bootstrap.sh +./scripts/compose.sh up -d --build +python3 scripts/wait_ready.py --base-url http://localhost:8084 +export BASE_URL=http://localhost:8084 +make research-consistency +make research-memory +make test-dependencies +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-research.txt +.venv/bin/python scripts/plot_research.py +python3 scripts/summarize_research.py +``` + +Методика добавляет синтетические данные; не удаляет volumes. Известные дополнительные cost/orphan fixtures очищаются точечно. Готовые вложения и сообщения остаются, поскольку обязательный API не включает delete. Повтор может иметь другой размер baseline, поэтому реальные N/M сохраняются в каждой строке. + +## Вывод и границы + +Гипотеза подтверждена в заявленной модели: durable intent, grace и общие object locks позволяют корректно обработать проверенные разрывы между двумя хранилищами; dry-run наблюдаем, apply повторяем, свежие/используемые/активные объекты защищены. Штатный worker запускает ту же логику с grace=300s. + +Восстановление утраченных bytes требует внешней копии; reconciler не заменяет backup. При потере всей БД, произвольных действиях root или нарушении инварианта неизменяемых ключей нужны отдельные процедуры восстановления. Метаданные являются авторитетом для классификации orphan. Локальные три коротких повтора на небольшом наборе и тёплом кэше не дают универсального SLA; стоимость штатного worker с меньшей CPU-квотой может отличаться от ops. Нет заявления о высокой доступности S3/БД, полной побайтовой проверке всех объектов или производственном zero-downtime. diff --git a/THEORY_SHORT.md b/THEORY_SHORT.md new file mode 100644 index 00000000..40e987a2 --- /dev/null +++ b/THEORY_SHORT.md @@ -0,0 +1,98 @@ +# Краткая теория и защита: лабораторная 4 + +## Задача + +Добавить к мессенджеру изображения и разделить два вида данных: + +- **PostgreSQL:** кто загрузил файл, в каком чате он доступен, тип, размер, SHA-256, состояние и связи с сообщениями. +- **MinIO:** исходные байты в приватном S3 bucket. + +Схема: `клиент → HAProxy → API ×2 → PostgreSQL + MinIO`. Valkey остаётся общим кэшем сессий из lab3. + +## Docker в этой работе + +Compose описывает общий стенд и приватную сеть. Две API запускаются из одного образа; только балансировщик публикует порт 8084. MinIO, БД и Valkey имеют собственные persistent volumes. API не хранит оригинал как обязательный локальный файл. + +Во время передачи API использует временный spool: первые 256 KiB в RAM, затем временный диск. После операции spool закрывается. Устойчивый оригинал находится в S3, поэтому другая реплика может его скачать. + +Volume переживает обычный restart, но не заменяет backup. Два API на одном Docker-host не обеспечивают выживание при потере всего host, единственного MinIO или PostgreSQL. + +## Как загружается файл + +1. Проверяется Bearer и членство в чате. +2. Проверяются filename и допустимый Content-Type. +3. Тело читается порциями с ограничением **10 MiB**, одновременно считается SHA-256. +4. Pillow проверяет фактический PNG/JPEG, целостность и лимит **40 млн пикселей** до полного декодирования. +5. В PostgreSQL сохраняется pending-намерение с серверным UUID и ключом. +6. Выполняется S3 PUT, запись переводится в ready и фиксируется в БД. +7. Только после этого API возвращает **201 ready**. + +Filename — только отображаемое имя. Ключ `originals/` назначает сервер. Готовый файл читается через авторизованный API, без redirect; публичный bucket не нужен. + +Сообщение содержит attachment_ids. Все ID должны принадлежать этому чату; другой участник может прикрепить доступный файл. Для сообщений сохраняется Idempotency-Key: повтор не создаёт дубль. + +## Почему приватного bucket недостаточно + +Приватность запрещает анонимный прямой S3 GET. Но при обращении через API всё равно нужно проверить права участника чата: случайный UUID не является авторизацией. Посторонний получает 404 для метаданных и bytes. + +API использует отдельные ограниченные S3 credentials. Root credentials есть только у MinIO/bootstrap. Консоль не опубликована; запись за пределами originals/* запрещена. + +## Главная проблема: два независимых хранилища + +Единой транзакции PostgreSQL + S3 нет. PUT может пройти, а COMMIT — не пройти. Немедленно удалять объект при любой ошибке тоже опасно: COMMIT мог реально выполниться, а ответ потеряться. + +Поэтому намерение записывается **до PUT**, а затем работает **reconciler** — повторяемая сверка двух хранилищ. + +| Ситуация | Действие reconciler | +| --- | --- | +| Старый объект без метаданных, orphan | Удалить после повторной проверки и grace | +| Старый pending после неуспешной операции | Удалить оставшийся объект, отметить incomplete_upload | +| Метаданные есть, объекта нет, missing | Сохранить метаданные/связи, отметить недоступность | +| Объект восстановлен и соответствует метаданным | Вернуть ready с прежним UUID | +| Свежий объект или активный uploader держит lock | Пропустить | +| Готовый используемый объект | Не удалять | +| Ошибка S3/БД | Остановить проход, не считать её отсутствием | + +**Grace period** — время ожидания перед очисткой. Обычно 300 секунд; в изолированном тесте — 5 секунд с явным разрешением. API и reconciler используют один PostgreSQL advisory lock для ключа объекта. Проверки повторяются под lock, чтобы не удалять объект на основании старого снимка. + +Dry-run показывает действия без изменения данных. Apply выполняет их. Повтор на исправленном состоянии не должен давать новые действия. Автопроцесс ждёт 60 секунд после каждого прохода. + +## Что выполнено на уровнях 3 и 4 + +Проверены оригинальные bytes и checksum, чужой чат/attachment, повреждение изображения, подмена MIME, опасные имена, превышение bytes/pixels и прерывание HTTP. Пользовательские ошибки дают 400/404/413/415, а не 500. + +Остановлена именно загрузившая файл API: другая скачала те же bytes через LB. Полный restart проверяет прежний токен, метаданные и оригинал. При реальном отказе MinIO/БД получены контролируемые 503 и проверено восстановление. + +При десяти одновременных uploads двух размеров прошли 40 запросов. Пики памяти API были примерно 91 MiB для файлов 0.75 MiB и 125–146 MiB для файлов 6.75 MiB при лимите 512 MiB. Это sampled cgroup memory, включая cache и sampler, а не точный heap-профиль. + +## Уровень 5: трек 2 + +**Гипотеза:** устойчивое намерение, общий lock и grace позволят исправлять расхождения без удаления свежих/используемых данных. + +В исследовании: + +- Реальный процесс завершался после успешного PUT до ready COMMIT. +- Создавались orphan, pending без объекта, missing и повреждённый объект. +- Проверялись dry-run, apply, повтор, защита fresh/used/locked объектов и восстановление прежнего UUID. +- Ошибки S3/БД не превращались в массовое ложное обнаружение missing. +- Стоимость обхода измерялась на базовом наборе с добавлением 0/50/200 orphan, по три dry-run. + +Обход использует List/HEAD и индексированные DB-запросы, без скачивания всех изображений. Поэтому оценивается стоимость инвентаризации, а не полного SHA-аудита bytes. Точные таблицы и графики находятся в [RESEARCH.md](RESEARCH.md). + +## Что сказать на защите + +«Я перенёс две API и балансировщик из третьей работы и добавил приватный MinIO. Метаданные хранятся в PostgreSQL, оригиналы — в S3. Загрузка потоковая, с проверкой реального PNG/JPEG, размера, пикселей и прав. 201 возвращается только после сохранения обеих частей. + +Главная инженерная проблема — отсутствие общей транзакции S3 и БД. Я сохраняю pending до PUT и использую reconciler с dry-run, grace и общими блокировками. В исследовании проверены реальные разрывы записи, orphan/missing, безопасное исправление и повтор. Свежие, используемые и активные объекты не удаляются. Дополнительно проверены отказ загрузившей API, restart, приватность bucket и память при десяти одновременных загрузках». + +## Четыре главных вопроса + +**Почему не файл на диске одной API?** После переключения другая реплика не обязана видеть её диск. S3 — общее устойчивое хранилище. + +**Когда честно вернуть 201?** После успешного PUT и COMMIT ready-метаданных. + +**Что если PUT прошёл, а COMMIT нет?** Остаётся известный pending; после grace reconciler проверит обе стороны. Уже готовая запись не удаляется слепой компенсацией. + +**Почему UUID и private bucket не заменяют ACL?** Пользователь обращается к API, поэтому она должна проверить его доступ к конкретному чату независимо от сложности угадывания ID. + +Ограничения: один host, S3/БД/LB не реплицированы; lost bytes требуют backup. Готовые неиспользуемые файлы не удаляются автоматически. Исследование — локальный опыт, не production SLA. diff --git a/app.py b/app.py new file mode 100644 index 00000000..efa5b062 --- /dev/null +++ b/app.py @@ -0,0 +1,896 @@ +from __future__ import annotations + +import base64 +import json +import os +import secrets +import socket +import re +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import psycopg +from psycopg_pool import ConnectionPool +import redis +from argon2 import PasswordHasher +from fastapi import FastAPI, Header, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.concurrency import run_in_threadpool +from pydantic import BaseModel, ConfigDict, StringConstraints, ValidationError +from typing_extensions import Annotated + + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname()) + +USERNAME_RE = r"^[a-z0-9_]{3,32}$" + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, *, headers: Optional[Dict[str, str]] = None): + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +class UserOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + username: str + display_name: str + created_at: str + + +class SessionOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str + token_type: str + expires_at: str + user: UserOut + + +class ChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + title: str + member_ids: List[str] + created_at: str + + +class MessageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: List[str] + + +class CreateUserIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: Annotated[str, StringConstraints(pattern=USERNAME_RE, min_length=3, max_length=32)] + display_name: Annotated[str, StringConstraints(min_length=1, max_length=100)] + password: Annotated[str, StringConstraints(min_length=12, max_length=128)] + + +class CreateChatIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: Annotated[str, StringConstraints(min_length=1, max_length=100)] + member_ids: List[str] + + +class CreateMessageIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + attachment_ids: List[str] = [] + text: Annotated[str, StringConstraints(max_length=4000)] + + +class Store: + def __init__(self) -> None: + self.connection_url = self._resolve_database_url() + self.cache_url = self._resolve_cache_url() + self.shared_cache = redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2) + self.session_store = os.getenv("SESSION_STORE", "valkey") + self.password_hasher = PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16) + self.pool = ConnectionPool( + self.connection_url, min_size=1, + max_size=int(os.getenv("DB_POOL_MAX_SIZE", "10")), + timeout=5, max_waiting=20, open=False, + kwargs={"connect_timeout": 5, "autocommit": False, + "options": "-c statement_timeout=5000 -c lock_timeout=500"}, + ) + + def _resolve_database_url(self) -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + secret_path = os.getenv("DATABASE_URL_FILE") + if secret_path and os.path.exists(secret_path): + with open(secret_path, "r", encoding="utf-8") as handle: + return handle.read().strip() + return os.getenv("DEFAULT_DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") + + def _resolve_cache_url(self) -> str: + value = os.getenv("CACHE_URL") + if value: + return value + secret_path = os.getenv("CACHE_URL_FILE") + if secret_path and os.path.exists(secret_path): + with open(secret_path, "r", encoding="utf-8") as handle: + return handle.read().strip() + return os.getenv("DEFAULT_CACHE_URL", "redis://localhost:6379/0") + + def connect(self): + return self.pool.connection() + + def cache_client(self): + if os.getenv("CACHE_CLIENT_MODE", "shared") == "shared": + return self.shared_cache + return redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2) + + def cache_session(self, session: Dict[str, Any]) -> None: + if self.session_store != "valkey": + return + try: + client = self.cache_client() + ttl_seconds = max(1, int((self.parse_session_expiry(session["expires_at"]) - self.utc_now()).total_seconds())) + payload = json.dumps({ + "token": session["token"], + "user_id": session["user_id"], + "expires_at": session["expires_at"], + "revoked_at": session.get("revoked_at"), + "user": session["user"], + }, separators=(",", ":"), ensure_ascii=False) + client.setex(f"session:{session['token']}", ttl_seconds, payload) + except redis.RedisError: + return + + def get_cached_session(self, token: str, *, db_session) -> Optional[Dict[str, Any]]: + if self.session_store != "valkey": + return None + try: + client = self.cache_client() + payload = client.get(f"session:{token}") + if not payload: + return None + data = json.loads(payload) + session = { + "token": data["token"], + "user_id": data["user_id"], + "expires_at": data["expires_at"], + "revoked_at": data.get("revoked_at"), + "user": data["user"], + } + if not self.is_session_valid(session): + client.delete(f"session:{token}") + return None + if session != db_session: + client.delete(f"session:{token}") + return None + return db_session + except (redis.RedisError, KeyError, TypeError, ValueError): + return None + + def delete_cached_session(self, token: str) -> None: + if self.session_store != "valkey": + return + try: + self.cache_client().delete(f"session:{token}") + except redis.RedisError: + return + + @staticmethod + def parse_session_expiry(value: Any) -> datetime: + if isinstance(value, datetime): + return value.astimezone(timezone.utc) + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + raise ValueError("invalid session expiry") + + def init(self) -> None: + migration_dir = Path(__file__).resolve().parent / "migrations" + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(202603)") + cur.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + """ + ) + + applied = set() + cur.execute("SELECT version FROM schema_migrations") + for (version,) in cur.fetchall(): + applied.add(version) + + for migration_path in sorted(migration_dir.glob("*.sql")): + version = migration_path.stem + if version in applied: + continue + sql = migration_path.read_text(encoding="utf-8") + cur.execute(sql) + cur.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (%s, %s)", + (version, self.utc_now()), + ) + conn.commit() + + def utc_now(self) -> datetime: + return datetime.now(timezone.utc) + + def isoformat_utc(self, value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + def hash_password(self, password: str) -> str: + return self.password_hasher.hash(password) + + def verify_password(self, password_hash: str, password: str) -> bool: + try: + self.password_hasher.verify(password_hash, password) + return True + except Exception: + return False + + def require_user_by_username(self, username: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", (username,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": self.isoformat_utc(row[3]), + "password_hash": row[4], + } + + def fetch_user(self, user_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": self.isoformat_utc(row[3]), + } + + def fetch_session_by_token(self, token: str, conn): + with conn.cursor() as cur: + cur.execute( + "SELECT s.token, s.user_id, s.expires_at, s.revoked_at, u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token = %s", + (token,), + ) + row = cur.fetchone() + if row is None: + return None + return { + "token": row[0], + "user_id": str(row[1]), + "expires_at": self.isoformat_utc(row[2]), + "revoked_at": row[3], + "user": { + "id": str(row[4]), + "username": row[5], + "display_name": row[6], + "created_at": self.isoformat_utc(row[7]), + }, + } + + def list_member_ids(self, chat_id: str, conn) -> List[str]: + with conn.cursor() as cur: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat_id,)) + rows = cur.fetchall() + return [str(row[0]) for row in rows] + + def fetch_chat(self, chat_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, title, created_at FROM chats WHERE id = %s", (chat_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "title": row[1], + "created_at": self.isoformat_utc(row[2]), + "member_ids": self.list_member_ids(str(row[0]), conn), + } + + def fetch_message(self, message_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", (message_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": self.isoformat_utc(row[4]), + "attachment_ids": self.message_attachment_ids(str(row[0]), conn), + } + + def fetch_chat_membership(self, chat_id: str, user_id: str, conn) -> bool: + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)) + return cur.fetchone() is not None + + def insert_user(self, username: str, display_name: str, password: str): + now = self.utc_now() + user_id = uuid.uuid4() + password_hash = self.hash_password(password) + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (%s,%s,%s,%s,%s)", + (user_id, username, display_name, password_hash, now), + ) + conn.commit() + return { + "id": str(user_id), + "username": username, + "display_name": display_name, + "created_at": self.isoformat_utc(now), + } + + def insert_chat(self, title: str, member_ids: List[str], actor_id: str): + now = self.utc_now() + chat_id = uuid.uuid4() + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("INSERT INTO chats(id,title,created_at) VALUES (%s,%s,%s)", (chat_id, title, now)) + for member_id in sorted(set(member_ids)): + cur.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (chat_id, member_id)) + conn.commit() + return { + "id": str(chat_id), + "title": title, + "member_ids": sorted(set(member_ids)), + "created_at": self.isoformat_utc(now), + } + + def insert_message(self, chat_id: str, sender_id: str, text: str, key: str, attachment_ids: List[str]): + # Transaction-level lock serializes the same scope across all replicas. + scope = json.dumps([sender_id, "POST", chat_id, key]) + request_body = {"text": text, "attachment_ids": attachment_ids} + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,)) + cur.execute("DELETE FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s AND expires_at <= NOW()", (sender_id, chat_id, key)) + cur.execute("SELECT body, response FROM message_requests WHERE sender_id=%s AND chat_id=%s AND key=%s", (sender_id, chat_id, key)) + previous = cur.fetchone() + if previous: + if previous[0] != request_body: + raise ApiError(409, "conflict", "idempotency key reused with different body") + return previous[1] + for aid in attachment_ids: + cur.execute("SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE", (aid, chat_id)) + att = cur.fetchone() + if att is None: + raise ApiError(404, "not_found", "attachment not found") + if att[0] != "ready": + raise ApiError(409, "conflict", "attachment unavailable") + now = self.utc_now() + message_id = str(uuid.uuid4()) + message = {"id": message_id, "chat_id": chat_id, "sender_id": sender_id, + "text": text, "created_at": self.isoformat_utc(now), "attachment_ids": attachment_ids} + cur.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES (%s,%s,%s,%s,%s)", + (message_id, chat_id, sender_id, text, now)) + for position, aid in enumerate(attachment_ids): + cur.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, aid, position)) + cur.execute("INSERT INTO message_requests(sender_id,chat_id,key,body,response,expires_at) VALUES (%s,%s,%s,%s::jsonb,%s::jsonb,%s)", + (sender_id, chat_id, key, json.dumps(request_body), json.dumps(message), now + timedelta(hours=24))) + conn.commit() + return message + + def message_attachment_ids(self, message_id, conn): + return [str(row[0]) for row in conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (message_id,)).fetchall()] + + @staticmethod + def cursor_uuid_key(value: str) -> int: + try: + return uuid.UUID(value).int + except (ValueError, TypeError): + return 0 + + def list_chats_for_user(self, user_id: str, limit: int, cursor: Optional[str]): + with self.connect() as conn: + with conn.cursor() as cur: + query = """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = %s + """ + params: List[Any] = [user_id] + if cursor: + prefix = f"user:{user_id}|" + if not cursor.startswith(prefix): + raise ApiError(400, "invalid_request", "cursor does not match user chat list") + remainder = cursor[len(prefix):] + if not remainder: + raise ApiError(400, "invalid_request", "invalid cursor") + try: + cursor_ts_raw, cursor_id = remainder.split("|", 1) + cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + query += " AND (c.created_at > %s OR (c.created_at = %s AND c.id > %s::uuid))" + params.extend([cursor_dt, cursor_dt, cursor_id]) + query += " ORDER BY c.created_at ASC, c.id ASC LIMIT %s" + params.append(limit + 1) + cur.execute(query, tuple(params)) + rows = cur.fetchall() + items = [ + { + "id": str(row[0]), + "title": row[1], + "created_at": self.isoformat_utc(row[2]), + "member_ids": self.list_member_ids(str(row[0]), conn), + } + for row in rows + ] + + next_cursor = None + if len(items) > limit: + last_item = items[limit - 1] + next_cursor = f"user:{user_id}|{last_item['created_at']}|{last_item['id']}" + items = items[:limit] + + return {"items": items, "next_cursor": next_cursor} + + def list_messages_for_chat(self, chat_id: str, limit: int, cursor: Optional[str]): + with self.connect() as conn: + with conn.cursor() as cur: + query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + prefix = f"chat:{chat_id}|" + if not cursor.startswith(prefix): + raise ApiError(400, "invalid_request", "cursor does not match chat") + remainder = cursor[len(prefix):] + if not remainder: + raise ApiError(400, "invalid_request", "invalid cursor") + try: + cursor_ts_raw, cursor_id = remainder.split("|", 1) + cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + query += " AND (created_at, id) > (%s, %s::uuid)" + params.extend([cursor_dt, cursor_id]) + query += " ORDER BY created_at ASC, id ASC LIMIT %s" + params.append(limit + 1) + cur.execute(query, tuple(params)) + rows = cur.fetchall() + links = {} + if rows: + cur.execute("SELECT message_id, attachment_id FROM message_attachments WHERE message_id=ANY(%s::uuid[]) ORDER BY message_id, position", ([str(row[0]) for row in rows],)) + for mid, aid in cur.fetchall(): + links.setdefault(str(mid), []).append(str(aid)) + + items = [{ + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": self.isoformat_utc(row[4]), + "attachment_ids": links.get(str(row[0]), []), + } for row in rows] + + next_cursor = None + if len(items) > limit: + last_item = items[limit - 1] + next_cursor = f"chat:{chat_id}|{last_item['created_at']}|{last_item['id']}" + items = items[:limit] + + return {"items": items, "next_cursor": next_cursor} + + def create_session(self, user_id: str, username: str): + token = secrets.token_urlsafe(32) + expires_at = self.utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO sessions(id, token, user_id, expires_at, created_at) VALUES (%s,%s,%s,%s,%s)", + (uuid.uuid4(), token, user_id, expires_at, self.utc_now()), + ) + conn.commit() + with self.connect() as conn: + user = self.fetch_user(user_id, conn) + if user is None: + raise ApiError(500, "internal_error", "session user missing") + session = { + "token": token, + "user_id": user_id, + "expires_at": self.isoformat_utc(expires_at), + "revoked_at": None, + "user": user, + } + self.cache_session(session) + return { + "token": token, + "token_type": "Bearer", + "expires_at": self.isoformat_utc(expires_at), + "user": user, + } + + def get_session_by_token(self, token: str, conn=None): + if conn is None: + with self.connect() as db_conn: + return self.get_session_by_token(token, db_conn) + db_session = self.fetch_session_by_token(token, conn) + if db_session is None or not self.is_session_valid(db_session): + self.delete_cached_session(token) + return None + cached = self.get_cached_session(token, db_session=db_session) + if cached is None: + self.cache_session(db_session) + return db_session + + def revoke_current_session(self, token: str): + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("UPDATE sessions SET revoked_at = %s WHERE token = %s AND revoked_at IS NULL", (self.utc_now(), token)) + conn.commit() + self.delete_cached_session(token) + + def is_session_valid(self, session) -> bool: + if session is None: + return False + if session.get("revoked_at") is not None: + return False + try: + expiry = self.parse_session_expiry(session["expires_at"]) + except ValueError: + return False + if expiry <= self.utc_now(): + return False + return True + + +store = Store() +from attachments import ObjectStore +object_store = ObjectStore(store, ApiError) +app = FastAPI(title="MTUSI Messenger") + + +@app.on_event("startup") +def startup_event() -> None: + store.pool.open(wait=True, timeout=10) + + + +@app.on_event("shutdown") +def shutdown_event() -> None: + store.pool.close() + store.shared_cache.close() + object_store.close() + + +@app.exception_handler(psycopg.OperationalError) +async def database_error_handler(request: Request, exc): + return await api_error_handler(request, ApiError(503, "unavailable", "database unavailable")) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = uuid.uuid4().hex + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + response.headers["X-App-Revision"] = os.getenv("APP_REVISION", "lab4-v1") + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + body = {"error": {"code": exc.code, "message": exc.message, "request_id": getattr(request.state, "request_id", "")}} + return JSONResponse(status_code=exc.status_code, headers={**exc.headers, "X-Request-Id": getattr(request.state, "request_id", ""), "X-Instance-Id": INSTANCE_ID}, content=body) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=400, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={"error": {"code": "invalid_request", "message": "invalid request", "request_id": request_id}}, + ) + + +@app.exception_handler(Exception) +async def generic_error_handler(request: Request, exc: Exception): + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=500, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={"error": {"code": "internal_error", "message": "internal error", "request_id": request_id}}, + ) + + +def get_auth_header(request: Request): + auth = request.headers.get("Authorization") + if not auth: + raise ApiError(401, "unauthorized", "missing authorization", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + scheme, _, token = auth.partition(" ") + if scheme.lower() != "bearer" or not token: + raise ApiError(401, "unauthorized", "invalid bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + return token.strip() + + +def require_bearer_session(request: Request): + token = get_auth_header(request) + with store.connect() as conn: + session = store.get_session_by_token(token, conn) + if session is None or not store.is_session_valid(session): + raise ApiError(401, "unauthorized", "invalid or expired session", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + return session + + +async def parse_json_body(request: Request) -> Dict[str, Any]: + try: + raw = await request.body() + if not raw: + return {} + return __import__("json").loads(raw) + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + try: + with store.connect() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + object_store.ready() + return {"status": "ok"} + except Exception as exc: + raise ApiError(503, "unavailable", "required storage unavailable") from exc + + +@app.post("/api/v1/users", status_code=201) +async def create_user_endpoint(request: Request): + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid user payload") + try: + payload = CreateUserIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid user payload") from exc + if not payload.username or not payload.display_name.strip(): + raise ApiError(400, "invalid_request", "invalid request") + payload.display_name = payload.display_name.strip() + try: + user = store.insert_user(payload.username, payload.display_name, payload.password) + return UserOut.model_validate(user) + except Exception as exc: + if "duplicate key" in str(exc).lower() or "unique" in str(exc).lower(): + raise ApiError(409, "conflict", "username already exists") from exc + raise ApiError(503, "unavailable", "required storage unavailable") from exc + + +@app.get("/api/v1/users/me") +def current_user(request: Request): + session = require_bearer_session(request) + user = session["user"] + return UserOut.model_validate(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request): + auth = request.headers.get("Authorization", "") + if not auth.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + encoded = auth.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded.encode("ascii"), validate=True) + raw = decoded.decode("utf-8") + except Exception: + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + if ":" not in raw: + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + username, password = raw.split(":", 1) + if not re_match(username): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + if not (12 <= len(password) <= 128): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + try: + with store.connect() as conn: + user = store.require_user_by_username(username, conn) + if user is None or not store.verify_password(user["password_hash"], password): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + session = store.create_session(user["id"], username) + return SessionOut.model_validate(session) + except ApiError: + raise + except Exception as exc: + raise ApiError(503, "unavailable", "required storage unavailable") from exc + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + session = require_bearer_session(request) + token = session["token"] + store.revoke_current_session(token) + return Response(status_code=204) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat_endpoint(request: Request): + session = require_bearer_session(request) + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid chat payload") + try: + payload = CreateChatIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid chat payload") from exc + if len(payload.member_ids) < 1 or len(payload.member_ids) > 100: + raise ApiError(400, "invalid_request", "invalid member_ids") + title = payload.title.strip() + if not title: + raise ApiError(400, "invalid_request", "invalid title") + normalized = [] + seen = set() + for member_id in payload.member_ids: + try: + member_id = str(uuid.UUID(member_id)) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid member_ids") from exc + if member_id in seen: + raise ApiError(400, "invalid_request", "duplicate member_ids") + seen.add(member_id) + normalized.append(member_id) + if session["user_id"] not in normalized: + raise ApiError(400, "invalid_request", "current user must be a member") + try: + with store.connect() as conn: + for member_id in normalized: + with conn.cursor() as cur: + cur.execute("SELECT id FROM users WHERE id = %s", (member_id,)) + if cur.fetchone() is None: + raise ApiError(400, "invalid_request", "unknown member") + chat = store.insert_chat(title, normalized, session["user_id"]) + return ChatOut.model_validate(chat) + except ApiError: + raise + except Exception as exc: + raise ApiError(503, "unavailable", "required storage unavailable") from exc + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + session = require_bearer_session(request) + limit = request.query_params.get("limit", "50") + cursor = request.query_params.get("cursor") + try: + limit_int = int(limit) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid limit") from exc + if not (1 <= limit_int <= 100): + raise ApiError(400, "invalid_request", "invalid limit") + result = store.list_chats_for_user(session["user_id"], limit_int, cursor) + return {"items": [ChatOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + with store.connect() as conn: + chat = store.fetch_chat(chat_id, conn) + if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + return ChatOut.model_validate(chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid message payload") + try: + payload = CreateMessageIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid message payload") from exc + if len(payload.attachment_ids) > 10: + raise ApiError(400, "invalid_request", "too many attachments") + try: + payload.attachment_ids = [str(uuid.UUID(aid)) for aid in payload.attachment_ids] + except (ValueError, TypeError) as exc: + raise ApiError(400, "invalid_request", "invalid attachment id") from exc + if len(set(payload.attachment_ids)) != len(payload.attachment_ids): + raise ApiError(400, "invalid_request", "duplicate attachment ids") + if not payload.text.strip() and not payload.attachment_ids: + raise ApiError(400, "invalid_request", "invalid text") + key = request.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise ApiError(400, "invalid_request", "invalid Idempotency-Key") + with store.connect() as conn: + chat = store.fetch_chat(chat_id, conn) + if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + message = store.insert_message(str(uuid.UUID(chat_id)), session["user_id"], payload.text, key, payload.attachment_ids) + return MessageOut.model_validate(message) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + limit = request.query_params.get("limit", "50") + cursor = request.query_params.get("cursor") + try: + limit_int = int(limit) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid limit") from exc + if not (1 <= limit_int <= 100): + raise ApiError(400, "invalid_request", "invalid limit") + with store.connect() as conn: + if not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + result = store.list_messages_for_chat(chat_id, limit_int, cursor) + return {"items": [MessageOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]} + + +def re_match(value: str) -> bool: + import re + + return bool(re.fullmatch(USERNAME_RE, value)) + + +from attachments import register_routes +register_routes(app, object_store, require_bearer_session) + +if __name__ == "__main__": + import uvicorn + + uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False) diff --git a/attachments.py b/attachments.py new file mode 100644 index 00000000..8a05ce5e --- /dev/null +++ b/attachments.py @@ -0,0 +1,234 @@ +"""Bounded synchronous uploads into private S3, with durable PostgreSQL intents.""" +import asyncio +import hashlib +import os +from pathlib import Path +import tempfile +import threading +import time +import unicodedata +import uuid +import warnings + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from fastapi import Request +from fastapi.responses import StreamingResponse +from PIL import Image, UnidentifiedImageError +from psycopg.rows import dict_row +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +MAX_BYTES = 10485760 +MAX_PIXELS = 40000000 +CHUNK = 65536 +# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is +# retained as another bound, but its warning threshold is not the contract cap. +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at' + + +def canonical_id(value, error): + try: + return str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError) as exc: + raise error(400, 'invalid_request', 'invalid id') from exc + + +def object_lock(conn, key, *, wait=True): + function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' + return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0] + + +class ObjectStore: + def __init__(self, store, error): + self.store, self.error = store, error + self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments') + self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'), + aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'), + config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3, + retries={'total_max_attempts': 1, 'mode': 'standard'}, + max_pool_connections=8, s3={'addressing_style': 'path'})) + self.decode_lock = threading.BoundedSemaphore(1) + # At most two payload spools per process; queued requests receive ASGI + # backpressure instead of being read into an unbounded request.body(). + self.upload_slots = asyncio.Semaphore(2) + + def close(self): + self.client.close() + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + except (BotoCoreError, ClientError) as exc: + raise self.error(503, 'unavailable', 'object storage unavailable') from exc + + def authorize_chat(self, request, chat_id, authenticate): + actor = authenticate(request) + chat_id = canonical_id(chat_id, self.error) + with self.store.connect() as conn: + if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn): + raise self.error(404, 'not_found', 'chat not found') + return actor, chat_id + + def validate_filename(self, name): + if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name): + raise self.error(400, 'invalid_request', 'invalid filename') + + def inspect_image(self, stream, claimed_type): + try: + with self.decode_lock, warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + stream.seek(0) + with Image.open(stream, formats=['PNG', 'JPEG']) as image: + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) + if image.width * image.height > MAX_PIXELS: + raise self.error(413, 'payload_too_large', 'too many image pixels') + if actual != claimed_type: + raise self.error(415, 'unsupported_media_type', 'content type mismatch') + image.verify() + stream.seek(0) + with Image.open(stream, formats=['PNG', 'JPEG']) as image: + image.load() # JPEG.verify alone does not ensure a full decode. + stream.seek(0) + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + raise self.error(413, 'payload_too_large', 'too many image pixels') from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc + + def fetch(self, aid, conn): + with conn.cursor(row_factory=dict_row) as cur: + cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,)) + return cur.fetchone() + + def public(self, row): + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), + 'owner_id': str(row['owner_id']), 'filename': row['filename'], + 'content_type': row['content_type'], 'size_bytes': row['size_bytes'], + 'sha256': row['sha256'], 'status': 'ready', 'variants': [], 'error_code': None, + 'created_at': self.store.isoformat_utc(row['created_at']), + 'updated_at': self.store.isoformat_utc(row['updated_at'])} + + def authorized(self, request, aid, authenticate): + actor = authenticate(request) + aid = canonical_id(aid, self.error) + with self.store.connect() as conn: + row = self.fetch(aid, conn) + if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn): + raise self.error(404, 'not_found', 'attachment not found') + if row['state'] != 'ready': + raise self.error(409, 'conflict', 'attachment unavailable') + return row + + def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum, *, after_put=None): + aid = str(uuid.uuid4()) + key = 'originals/' + aid + now = self.store.utc_now() + # Intent is durable BEFORE any S3 write. A crash leaves a known pending + # row; periodic reconciliation cleans it only after a safe grace period. + with self.store.connect() as conn: + conn.execute('INSERT INTO attachments('+COLUMNS+') VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,NULL,%s,%s)', + (aid, chat_id, actor_id, key, filename, content_type, size, checksum, 'pending', now, now)) + with self.store.connect() as conn: + object_lock(conn, key) + try: + stream.seek(0) + self.client.put_object(Bucket=self.bucket, Key=key, Body=stream, + ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum}) + # Optional in-process hook for isolated CLI fault injection only; + # HTTP clients cannot select failpoints. + if after_put: + after_put(aid, key) + conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (aid,)) + row = self.fetch(aid, conn) + except (BotoCoreError, ClientError) as exc: + raise self.error(503, 'unavailable', 'object storage unavailable') from exc + return self.public(row) + + def download(self, row): + try: + response = self.client.get_object(Bucket=self.bucket, Key=row['object_key']) + except (BotoCoreError, ClientError) as exc: + raise self.error(503, 'unavailable', 'object storage unavailable') from exc + body = response['Body'] + if response['ContentLength'] != row['size_bytes']: + body.close() + raise self.error(503, 'unavailable', 'object unavailable') + def chunks(): + try: + while True: + part = body.read(CHUNK) + if not part: + break + yield part + finally: + body.close() + return StreamingResponse(chunks(), media_type=row['content_type'], headers={ + 'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff', + 'Content-Disposition': 'attachment'}) + + +def register_routes(app, objects, authenticate): + @app.post('/api/v1/chats/{chat_id}/attachments', status_code=201) + async def upload(request: Request, chat_id: str): + actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate) + filename = request.query_params.get('filename', '') + objects.validate_filename(filename) + claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower() + if claimed not in ('image/png', 'image/jpeg'): + raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required') + length = request.headers.get('content-length') + if length: + try: + expected = int(length) + except ValueError as exc: + raise objects.error(400, 'invalid_request', 'invalid content length') from exc + if expected > MAX_BYTES: + raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB') + if expected < 1: + raise objects.error(415, 'unsupported_media_type', 'empty image') + try: + await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30) + except asyncio.TimeoutError as exc: + raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc + try: + # Temporary payloads are never persistent API storage. Every path, + # including disconnect/cancellation/error, closes and unlinks the spool. + with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream: + checksum, size = hashlib.sha256(), 0 + try: + iterator = request.stream().__aiter__() + async with asyncio.timeout(30): + while True: + try: + chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5) + except StopAsyncIteration: + break + size += len(chunk) + if size > MAX_BYTES: + raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB') + checksum.update(chunk) + await run_in_threadpool(stream.write, chunk) + except (asyncio.TimeoutError, ClientDisconnect) as exc: + raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc + if size < 1: + raise objects.error(415, 'unsupported_media_type', 'empty image') + await run_in_threadpool(objects.inspect_image, stream, claimed) + return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id, + filename, claimed, size, checksum.hexdigest()) + finally: + objects.upload_slots.release() + + @app.get('/api/v1/attachments/{aid}') + def metadata(request: Request, aid: str): + return objects.public(objects.authorized(request, aid, authenticate)) + + @app.get('/api/v1/attachments/{aid}/content') + def content(request: Request, aid: str): + row = objects.authorized(request, aid, authenticate) + if request.query_params.get('variant', 'original') != 'original': + raise objects.error(404, 'not_found', 'variant not found') + return objects.download(row) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..4bb608a1 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,110 @@ +name: mtusi-lab4 +x-api: &api + build: . + image: ${API_IMAGE:-mtusi-lab4-api:local} + environment: &environment + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh}@db:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: valkey + SESSION_TTL_SECONDS: 3600 + DB_POOL_MAX_SIZE: 5 + CACHE_CLIENT_MODE: shared + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: lab4api + S3_SECRET_ACCESS_KEY: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh} + cpus: ${API_CPUS:-2} + mem_limit: 512m + stop_grace_period: 40s +services: + db: + image: postgres:16.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh} + volumes: [postgres_data:/var/lib/postgresql/data] + cpus: 1 + mem_limit: 512m + healthcheck: + test: [CMD-SHELL, pg_isready -U messenger -d messenger] + interval: 2s + timeout: 1s + retries: 30 + valkey: + image: valkey/valkey:8.0.2-alpine + command: [valkey-server, --appendonly, 'yes'] + volumes: [valkey_data:/data] + cpus: 0.25 + mem_limit: 128m + s3: + # RELEASE.2025-09-07T16-13-09Z, verified locally, linux/arm64. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER: lab4root + MINIO_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh} + MINIO_BROWSER: 'off' + volumes: [s3_data:/data] + cpus: 0.5 + mem_limit: 768m + s3-init: + # mc RELEASE.2025-08-13T08-35-41Z. + image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 + entrypoint: [/bin/sh, /bootstrap/init-s3.sh] + environment: + S3_ROOT_USER: lab4root + S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Run scripts/bootstrap.sh} + S3_API_USER: lab4api + S3_API_PASSWORD: ${S3_API_PASSWORD:?Run scripts/bootstrap.sh} + volumes: ['./infra:/bootstrap:ro'] + depends_on: [s3] + cpus: 1 + mem_limit: 512m + migrate: + <<: *api + command: [python, migrate.py] + depends_on: + db: {condition: service_healthy} + api: + <<: *api + command: [uvicorn, 'app:app', --host, 0.0.0.0, --port, '8080', --timeout-graceful-shutdown, '35', --limit-concurrency, '64'] + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + valkey: {condition: service_started} + healthcheck: + test: [CMD, python, -c, "import socket; s=socket.create_connection(('127.0.0.1',8080),1); s.sendall(b'GET /health/ready HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n'); assert s.makefile('rb').readline().split()[1]==b'200'"] + interval: 2s + timeout: 4s + retries: 2 + lb: + image: haproxy:3.0.5-alpine + ports: ['${APP_PORT:-8084}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: [api] + cpus: 0.25 + mem_limit: 64m + reconciler: + <<: *api + command: [python, scripts/reconcile_loop.py] + cpus: 0.25 + mem_limit: 256m + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + restart: unless-stopped + ops: + <<: *api + profiles: [tools] + volumes: ['./tests:/app/tests:ro', './scripts:/app/scripts:ro'] + command: [python, reconcile.py] + cpus: 1 + mem_limit: 512m +volumes: + postgres_data: + valkey_data: + s3_data: diff --git a/evidence/README.md b/evidence/README.md index 8df70015..17a910bb 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,22 @@ -# Доказательства +# Доказательства lab4 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Проверки от 6 ноября 2026 года. Отдельный проект mtusi-lab4, публичный LB localhost:8084. Все данные синтетические, пароли/токены не сохранены. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +- check.txt, validate.txt: структура и OpenAPI. +- smoke.txt: публичный сценарий lab4, checksum и ACL. +- idempotency.txt, cache-consistency.txt: сохранённые свойства lab3. +- uploads-negative.json: 21 негативная/граничная проверка, включая прерванную передачу. +- private-storage.json: anonymous 403, запрещённые S3 admin-операции, префикс и console. +- attachment-failover.json/.txt: реальная остановка именно uploader, download другой API, checksum, snapshot. +- failover.txt, after-public-failover.json: публичный failover с повтором ключа и проверкой оригинала. +- persistence.txt, before-persistence.json, after-persistence.json: прежняя сессия/оригинал и реальный restart API×2/DB/Valkey/MinIO/LB/reconciler. +- final-containers.json: здоровые процессы, image IDs, лимиты/порты/volumes без environment. +- dependency-outages.json/.txt, dependency-outages-compose.txt: реальные stop/start MinIO и DB, контролируемые 503 и восстановление. +- reconciliation.json: трек 2, real crash exit 86, dry/apply/repeat, fresh/used/lock protection, восстановление и стоимость обхода. +- research-consistency.txt, research-consistency-compose.txt: ход запуска исследования и восстановление штатного worker. +- scan-cost-summary.json/.csv/.png/.svg: агрегаты трёх повторов каждой группы и график стоимости. +- upload-memory.json/.txt/.png/.svg: два размера, concurrency 10, 40 uploads, индивидуальные задержки и cgroup traces с шагом 50ms. +- s3-bootstrap.txt, versions.txt, migrations.txt: реальные bootstrap, версии и однократные SQL-версии. +- build.txt: сборка образа. + +Графики восстанавливаются scripts/plot_research.py, описание — scripts/summarize_research.py. Методика и ограничения: ../RESEARCH.md; запуск и рубрика: ../REPORT.md. diff --git a/evidence/after-persistence.json b/evidence/after-persistence.json new file mode 100644 index 00000000..6d152bb3 --- /dev/null +++ b/evidence/after-persistence.json @@ -0,0 +1,150 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "f73f3d296d3ed0ac7395f526fec915a7b0a0833957d27ed1f2de5d28ea8f89a7", + "hostname": "f73f3d296d3e", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:15.559505564Z", + "health": "starting", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-api-2", + "id": "e2b4291a5ae676735910d3c89e3ec9f81a2c72c316fc1c18595e4a8099d0df45", + "hostname": "e2b4291a5ae6", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:15.560857437Z", + "health": "starting", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-db-1", + "id": "fd19a37a2f105c1fed35c9497a060fba08e0c7a149ba100fa1501928dee2e47d", + "hostname": "fd19a37a2f10", + "service": "db", + "image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad", + "status": "running", + "started_at": "2026-11-06T16:33:15.285275581Z", + "health": "starting", + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "c0e590462a151fe77dcb24e7916cf35144561a992dc2bba9fc245d254081c183", + "hostname": "c0e590462a15", + "service": "lb", + "image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a", + "status": "running", + "started_at": "2026-11-06T16:33:15.278929955Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 67108864, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "", + "HostPort": "8084" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi-lab4-migrate-1", + "id": "89f12c68b068d1aa7d398325407a0fc2bd9e8e3f3e63d8bb56f218617e4f440e", + "hostname": "89f12c68b068", + "service": "migrate", + "image_id": "sha256:04c4a7c063087f247f2b71bebca9af01532751b7972abca830f0ec6b810fb864", + "status": "exited", + "started_at": "2026-11-06T16:33:09.510610693Z", + "health": null, + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-reconciler-1", + "id": "10543b9482036875de4048b2df74b850a98d958890dcfd5f9684980857525315", + "hostname": "10543b948203", + "service": "reconciler", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:15.730144813Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 268435456, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "41fe288f948ed63d950da39c42d23942703cb8e713f3110f3a0bb18b60508aac", + "hostname": "41fe288f948e", + "service": "s3", + "image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "status": "running", + "started_at": "2026-11-06T16:33:15.276760526Z", + "health": null, + "nano_cpus": 500000000, + "memory_bytes": 805306368, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_s3_data", + "destination": "/data" + } + ] + }, + { + "name": "/mtusi-lab4-s3-init-1", + "id": "1bac3f8b21417eca4cec0a8cfd779e7d804d8ef66ad830d7d0c60ed5c0fed9c1", + "hostname": "1bac3f8b2141", + "service": "s3-init", + "image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1", + "status": "exited", + "started_at": "2026-11-06T16:33:09.037626611Z", + "health": null, + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-valkey-1", + "id": "541af4b35c71939abfb80f67d4404eb23458551317324a09a5237bbe470e9625", + "hostname": "541af4b35c71", + "service": "valkey", + "image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866", + "status": "running", + "started_at": "2026-11-06T16:33:15.237666389Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 134217728, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_valkey_data", + "destination": "/data" + } + ] + } +] diff --git a/evidence/after-public-failover.json b/evidence/after-public-failover.json new file mode 100644 index 00000000..c27ce89e --- /dev/null +++ b/evidence/after-public-failover.json @@ -0,0 +1,150 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "f73f3d296d3ed0ac7395f526fec915a7b0a0833957d27ed1f2de5d28ea8f89a7", + "hostname": "f73f3d296d3e", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:30:37.647258568Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-api-2", + "id": "e2b4291a5ae676735910d3c89e3ec9f81a2c72c316fc1c18595e4a8099d0df45", + "hostname": "e2b4291a5ae6", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "exited", + "started_at": "2026-11-06T16:32:49.678956524Z", + "health": "unhealthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-db-1", + "id": "fd19a37a2f105c1fed35c9497a060fba08e0c7a149ba100fa1501928dee2e47d", + "hostname": "fd19a37a2f10", + "service": "db", + "image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad", + "status": "running", + "started_at": "2026-11-06T16:27:54.939507016Z", + "health": "healthy", + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "c0e590462a151fe77dcb24e7916cf35144561a992dc2bba9fc245d254081c183", + "hostname": "c0e590462a15", + "service": "lb", + "image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a", + "status": "running", + "started_at": "2026-11-06T16:11:29.096381472Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 67108864, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "", + "HostPort": "8084" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi-lab4-migrate-1", + "id": "89f12c68b068d1aa7d398325407a0fc2bd9e8e3f3e63d8bb56f218617e4f440e", + "hostname": "89f12c68b068", + "service": "migrate", + "image_id": "sha256:04c4a7c063087f247f2b71bebca9af01532751b7972abca830f0ec6b810fb864", + "status": "exited", + "started_at": "2026-11-06T16:32:48.463254232Z", + "health": null, + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-reconciler-1", + "id": "10543b9482036875de4048b2df74b850a98d958890dcfd5f9684980857525315", + "hostname": "10543b948203", + "service": "reconciler", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:32:44.14731806Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 268435456, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "41fe288f948ed63d950da39c42d23942703cb8e713f3110f3a0bb18b60508aac", + "hostname": "41fe288f948e", + "service": "s3", + "image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "status": "running", + "started_at": "2026-11-06T16:27:51.143991324Z", + "health": null, + "nano_cpus": 500000000, + "memory_bytes": 805306368, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_s3_data", + "destination": "/data" + } + ] + }, + { + "name": "/mtusi-lab4-s3-init-1", + "id": "1bac3f8b21417eca4cec0a8cfd779e7d804d8ef66ad830d7d0c60ed5c0fed9c1", + "hostname": "1bac3f8b2141", + "service": "s3-init", + "image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1", + "status": "exited", + "started_at": "2026-11-06T16:32:47.713300078Z", + "health": null, + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-valkey-1", + "id": "541af4b35c71939abfb80f67d4404eb23458551317324a09a5237bbe470e9625", + "hostname": "541af4b35c71", + "service": "valkey", + "image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866", + "status": "running", + "started_at": "2026-11-06T16:10:03.402515174Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 134217728, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_valkey_data", + "destination": "/data" + } + ] + } +] diff --git a/evidence/attachment-failover.json b/evidence/attachment-failover.json new file mode 100644 index 00000000..b9df6c6e --- /dev/null +++ b/evidence/attachment-failover.json @@ -0,0 +1,163 @@ +{ + "status": "PASS", + "uploader_instance": "e2b4291a5ae6", + "download_instance": "f73f3d296d3e", + "observed_instances_before": [ + "e2b4291a5ae6", + "f73f3d296d3e" + ], + "failed_attempts": 3, + "first_download_write_recovery_seconds": 2.159, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "same_bytes": true, + "containers_after_stop": [ + { + "name": "/mtusi-lab4-api-1", + "id": "f73f3d296d3ed0ac7395f526fec915a7b0a0833957d27ed1f2de5d28ea8f89a7", + "hostname": "f73f3d296d3e", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:30:37.647258568Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-api-2", + "id": "e2b4291a5ae676735910d3c89e3ec9f81a2c72c316fc1c18595e4a8099d0df45", + "hostname": "e2b4291a5ae6", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "exited", + "started_at": "2026-11-06T16:30:37.478730767Z", + "health": "unhealthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-db-1", + "id": "fd19a37a2f105c1fed35c9497a060fba08e0c7a149ba100fa1501928dee2e47d", + "hostname": "fd19a37a2f10", + "service": "db", + "image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad", + "status": "running", + "started_at": "2026-11-06T16:27:54.939507016Z", + "health": "healthy", + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "c0e590462a151fe77dcb24e7916cf35144561a992dc2bba9fc245d254081c183", + "hostname": "c0e590462a15", + "service": "lb", + "image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a", + "status": "running", + "started_at": "2026-11-06T16:11:29.096381472Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 67108864, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "", + "HostPort": "8084" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi-lab4-migrate-1", + "id": "89f12c68b068d1aa7d398325407a0fc2bd9e8e3f3e63d8bb56f218617e4f440e", + "hostname": "89f12c68b068", + "service": "migrate", + "image_id": "sha256:04c4a7c063087f247f2b71bebca9af01532751b7972abca830f0ec6b810fb864", + "status": "exited", + "started_at": "2026-11-06T16:32:43.014505315Z", + "health": null, + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-reconciler-1", + "id": "10543b9482036875de4048b2df74b850a98d958890dcfd5f9684980857525315", + "hostname": "10543b948203", + "service": "reconciler", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:32:44.14731806Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 268435456, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "41fe288f948ed63d950da39c42d23942703cb8e713f3110f3a0bb18b60508aac", + "hostname": "41fe288f948e", + "service": "s3", + "image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "status": "running", + "started_at": "2026-11-06T16:27:51.143991324Z", + "health": null, + "nano_cpus": 500000000, + "memory_bytes": 805306368, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_s3_data", + "destination": "/data" + } + ] + }, + { + "name": "/mtusi-lab4-s3-init-1", + "id": "1bac3f8b21417eca4cec0a8cfd779e7d804d8ef66ad830d7d0c60ed5c0fed9c1", + "hostname": "1bac3f8b2141", + "service": "s3-init", + "image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1", + "status": "exited", + "started_at": "2026-11-06T16:32:42.519564804Z", + "health": null, + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-valkey-1", + "id": "541af4b35c71939abfb80f67d4404eb23458551317324a09a5237bbe470e9625", + "hostname": "541af4b35c71", + "service": "valkey", + "image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866", + "status": "running", + "started_at": "2026-11-06T16:10:03.402515174Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 134217728, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_valkey_data", + "destination": "/data" + } + ] + } + ] +} diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 00000000..e214fe0a --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1,13 @@ +{ + "status": "PASS", + "uploader_instance": "e2b4291a5ae6", + "download_instance": "f73f3d296d3e", + "observed_instances_before": [ + "e2b4291a5ae6", + "f73f3d296d3e" + ], + "failed_attempts": 3, + "first_download_write_recovery_seconds": 2.159, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "same_bytes": true +} diff --git a/evidence/before-persistence.json b/evidence/before-persistence.json new file mode 100644 index 00000000..d94809cf --- /dev/null +++ b/evidence/before-persistence.json @@ -0,0 +1,150 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "f73f3d296d3ed0ac7395f526fec915a7b0a0833957d27ed1f2de5d28ea8f89a7", + "hostname": "f73f3d296d3e", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:30:37.647258568Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-api-2", + "id": "e2b4291a5ae676735910d3c89e3ec9f81a2c72c316fc1c18595e4a8099d0df45", + "hostname": "e2b4291a5ae6", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:10.693734728Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-db-1", + "id": "fd19a37a2f105c1fed35c9497a060fba08e0c7a149ba100fa1501928dee2e47d", + "hostname": "fd19a37a2f10", + "service": "db", + "image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad", + "status": "running", + "started_at": "2026-11-06T16:27:54.939507016Z", + "health": "healthy", + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "c0e590462a151fe77dcb24e7916cf35144561a992dc2bba9fc245d254081c183", + "hostname": "c0e590462a15", + "service": "lb", + "image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a", + "status": "running", + "started_at": "2026-11-06T16:11:29.096381472Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 67108864, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "", + "HostPort": "8084" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi-lab4-migrate-1", + "id": "89f12c68b068d1aa7d398325407a0fc2bd9e8e3f3e63d8bb56f218617e4f440e", + "hostname": "89f12c68b068", + "service": "migrate", + "image_id": "sha256:04c4a7c063087f247f2b71bebca9af01532751b7972abca830f0ec6b810fb864", + "status": "exited", + "started_at": "2026-11-06T16:33:09.510610693Z", + "health": null, + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-reconciler-1", + "id": "10543b9482036875de4048b2df74b850a98d958890dcfd5f9684980857525315", + "hostname": "10543b948203", + "service": "reconciler", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:32:44.14731806Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 268435456, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "41fe288f948ed63d950da39c42d23942703cb8e713f3110f3a0bb18b60508aac", + "hostname": "41fe288f948e", + "service": "s3", + "image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "status": "running", + "started_at": "2026-11-06T16:27:51.143991324Z", + "health": null, + "nano_cpus": 500000000, + "memory_bytes": 805306368, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_s3_data", + "destination": "/data" + } + ] + }, + { + "name": "/mtusi-lab4-s3-init-1", + "id": "1bac3f8b21417eca4cec0a8cfd779e7d804d8ef66ad830d7d0c60ed5c0fed9c1", + "hostname": "1bac3f8b2141", + "service": "s3-init", + "image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1", + "status": "exited", + "started_at": "2026-11-06T16:33:09.037626611Z", + "health": null, + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-valkey-1", + "id": "541af4b35c71939abfb80f67d4404eb23458551317324a09a5237bbe470e9625", + "hostname": "541af4b35c71", + "service": "valkey", + "image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866", + "status": "running", + "started_at": "2026-11-06T16:10:03.402515174Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 134217728, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_valkey_data", + "destination": "/data" + } + ] + } +] diff --git a/evidence/build.txt b/evidence/build.txt new file mode 100644 index 00000000..c1eaff7d --- /dev/null +++ b/evidence/build.txt @@ -0,0 +1,61 @@ +Compose can now delegate builds to bake for better performance. + To do so, set COMPOSE_BAKE=true. +#0 building with "colima" instance using docker driver + +#1 [api internal] load build definition from Dockerfile +#1 transferring dockerfile: 744B done +#1 DONE 0.0s + +#2 [api auth] library/python:pull token for registry-1.docker.io +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.11.11-slim +#3 DONE 0.9s + +#4 [api internal] load .dockerignore +#4 transferring context: 233B done +#4 DONE 0.0s + +#5 [api 1/10] FROM docker.io/library/python:3.11.11-slim@sha256:081075da77b2b55c23c088251026fb69a7b2bf92471e491ff5fd75c192fd38e5 +#5 DONE 0.0s + +#6 [api internal] load build context +#6 transferring context: 55.27kB 0.0s done +#6 DONE 0.0s + +#7 [api 2/10] WORKDIR /app +#7 CACHED + +#8 [api 3/10] COPY requirements.txt /app/requirements.txt +#8 CACHED + +#9 [api 4/10] RUN pip install --no-cache-dir -r /app/requirements.txt +#9 CACHED + +#10 [api 5/10] COPY app.py attachments.py reconcile.py migrate.py /app/ +#10 DONE 0.0s + +#11 [api 6/10] COPY migrations /app/migrations +#11 DONE 0.0s + +#12 [api 7/10] COPY scripts /app/scripts +#12 DONE 0.0s + +#13 [api 8/10] COPY tests /app/tests +#13 DONE 0.0s + +#14 [api 9/10] COPY contracts /app/contracts +#14 DONE 0.0s + +#15 [api 10/10] COPY lab.json /app/lab.json +#15 DONE 0.0s + +#16 [api] exporting to image +#16 exporting layers 0.0s done +#16 writing image sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d done +#16 naming to docker.io/library/mtusi-lab4-api:local done +#16 DONE 0.0s + +#17 [api] resolving provenance for metadata file +#17 DONE 0.0s + api Built diff --git a/evidence/cache-consistency.txt b/evidence/cache-consistency.txt new file mode 100644 index 00000000..7ffb35f9 --- /dev/null +++ b/evidence/cache-consistency.txt @@ -0,0 +1,12 @@ +./scripts/compose.sh exec -T -e BASE_URL=http://lb:8080 -e CACHE_URL=redis://valkey:6379/0 api python - < tests/cache_consistency.py +cache_key_exists=True +ttl_seconds=3599 +avg_latency_ms=1.53 +p95_latency_ms=1.68 +stale_cache_rejected=True +forged_identity_rejected=True +concurrent_identity_checks=8/8 +malformed_cache_recovers=True +repopulated_revoked_cache_rejected=True +short_ttl_recovers_from_db=True +logout_revokes_cache=True diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/dependency-outages-compose.txt b/evidence/dependency-outages-compose.txt new file mode 100644 index 00000000..fb5b75ec --- /dev/null +++ b/evidence/dependency-outages-compose.txt @@ -0,0 +1,8 @@ + Container mtusi-lab4-s3-1 Stopping + Container mtusi-lab4-s3-1 Stopped + Container mtusi-lab4-s3-1 Starting + Container mtusi-lab4-s3-1 Started + Container mtusi-lab4-db-1 Stopping + Container mtusi-lab4-db-1 Stopped + Container mtusi-lab4-db-1 Starting + Container mtusi-lab4-db-1 Started diff --git a/evidence/dependency-outages.json b/evidence/dependency-outages.json new file mode 100644 index 00000000..f8020d96 --- /dev/null +++ b/evidence/dependency-outages.json @@ -0,0 +1,19 @@ +[ + { + "dependency": "s3", + "readiness": 503, + "metadata_available": true, + "download": 503, + "upload": 503, + "status": "PASS", + "confirmed_original_preserved": true + }, + { + "dependency": "db", + "readiness": 503, + "upload": 503, + "no_new_s3_objects": true, + "status": "PASS", + "confirmed_original_preserved": true + } +] diff --git a/evidence/dependency-outages.txt b/evidence/dependency-outages.txt new file mode 100644 index 00000000..f8020d96 --- /dev/null +++ b/evidence/dependency-outages.txt @@ -0,0 +1,19 @@ +[ + { + "dependency": "s3", + "readiness": 503, + "metadata_available": true, + "download": 503, + "upload": 503, + "status": "PASS", + "confirmed_original_preserved": true + }, + { + "dependency": "db", + "readiness": 503, + "upload": 503, + "no_new_s3_objects": true, + "status": "PASS", + "confirmed_original_preserved": true + } +] diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..d02c9653 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8084" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 3, "first_recovery_seconds": 1.846} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-containers.json b/evidence/final-containers.json new file mode 100644 index 00000000..9d4e2708 --- /dev/null +++ b/evidence/final-containers.json @@ -0,0 +1,150 @@ +[ + { + "name": "/mtusi-lab4-api-1", + "id": "f73f3d296d3ed0ac7395f526fec915a7b0a0833957d27ed1f2de5d28ea8f89a7", + "hostname": "f73f3d296d3e", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:15.559505564Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-api-2", + "id": "e2b4291a5ae676735910d3c89e3ec9f81a2c72c316fc1c18595e4a8099d0df45", + "hostname": "e2b4291a5ae6", + "service": "api", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:33:15.560857437Z", + "health": "healthy", + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-db-1", + "id": "fd19a37a2f105c1fed35c9497a060fba08e0c7a149ba100fa1501928dee2e47d", + "hostname": "fd19a37a2f10", + "service": "db", + "image_id": "sha256:b4fc54c57484e6077fad42c66fc44a6f5e7f5892b1a92fa48e910938bf5ce0ad", + "status": "running", + "started_at": "2026-11-06T16:33:15.285275581Z", + "health": "healthy", + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "name": "/mtusi-lab4-lb-1", + "id": "c0e590462a151fe77dcb24e7916cf35144561a992dc2bba9fc245d254081c183", + "hostname": "c0e590462a15", + "service": "lb", + "image_id": "sha256:6a7f903d19e76bd3fa7fb5225e33d6acd5eb7f52407eece4858b58831c3e256a", + "status": "running", + "started_at": "2026-11-06T16:33:15.278929955Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 67108864, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "", + "HostPort": "8084" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi-lab4-migrate-1", + "id": "89f12c68b068d1aa7d398325407a0fc2bd9e8e3f3e63d8bb56f218617e4f440e", + "hostname": "89f12c68b068", + "service": "migrate", + "image_id": "sha256:04c4a7c063087f247f2b71bebca9af01532751b7972abca830f0ec6b810fb864", + "status": "exited", + "started_at": "2026-11-06T16:40:26.717903684Z", + "health": null, + "nano_cpus": 2000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-reconciler-1", + "id": "10543b9482036875de4048b2df74b850a98d958890dcfd5f9684980857525315", + "hostname": "10543b948203", + "service": "reconciler", + "image_id": "sha256:eb36496d11ab7325efa30097c8964754a9f326cdb2e3a5a4b0ce73d3c3ca517d", + "status": "running", + "started_at": "2026-11-06T16:40:27.887585144Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 268435456, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-s3-1", + "id": "41fe288f948ed63d950da39c42d23942703cb8e713f3110f3a0bb18b60508aac", + "hostname": "41fe288f948e", + "service": "s3", + "image_id": "sha256:8f08aee614800a237906bd48114d733e5ac5bfac4ccdf731f141b0e880d7a253", + "status": "running", + "started_at": "2026-11-06T16:33:15.276760526Z", + "health": null, + "nano_cpus": 500000000, + "memory_bytes": 805306368, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_s3_data", + "destination": "/data" + } + ] + }, + { + "name": "/mtusi-lab4-s3-init-1", + "id": "1bac3f8b21417eca4cec0a8cfd779e7d804d8ef66ad830d7d0c60ed5c0fed9c1", + "hostname": "1bac3f8b2141", + "service": "s3-init", + "image_id": "sha256:5dee113ef037d349ac22ab6c20193ade5c4701e2a38e3777fa1c1bec1c063ad1", + "status": "exited", + "started_at": "2026-11-06T16:40:26.207356505Z", + "health": null, + "nano_cpus": 1000000000, + "memory_bytes": 536870912, + "published_ports": {}, + "volumes": [] + }, + { + "name": "/mtusi-lab4-valkey-1", + "id": "541af4b35c71939abfb80f67d4404eb23458551317324a09a5237bbe470e9625", + "hostname": "541af4b35c71", + "service": "valkey", + "image_id": "sha256:7b2072cf1b86d58fee110f4da8dc3b0e1444b22ab64f58d7f61e822e0761f866", + "status": "running", + "started_at": "2026-11-06T16:33:15.237666389Z", + "health": null, + "nano_cpus": 250000000, + "memory_bytes": 134217728, + "published_ports": {}, + "volumes": [ + { + "name": "mtusi-lab4_valkey_data", + "destination": "/data" + } + ] + } +] diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..9a754741 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,2 @@ +python3 tests/idempotency_concurrent.py --base-url "http://localhost:8084" +{"parallel_requests": 16, "observed_instances": ["e2b4291a5ae6", "f73f3d296d3e"], "unique_messages": 1, "conflict": 409, "semantic_replay": true, "user_scope": true} diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..33ad9f06 --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,7 @@ + version | applied_at +--------------------+------------------------------- + 001_initial_schema | 2026-11-06 16:10:07.12842+00 + 002_idempotency | 2026-11-06 16:10:07.154748+00 + 003_attachments | 2026-11-06 16:10:07.177593+00 +(3 rows) + diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..1ce9c3f0 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8084" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private-storage.json b/evidence/private-storage.json new file mode 100644 index 00000000..85217a8a --- /dev/null +++ b/evidence/private-storage.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "anonymous_bucket_and_object_status": [ + 403, + 403 + ], + "application_admin_operations_denied": [ + "create_bucket", + "get_bucket_policy", + "write_outside_prefix" + ], + "visible_bucket_names": [ + "messenger-attachments" + ], + "console_reachable": false +} diff --git a/evidence/reconciliation.json b/evidence/reconciliation.json new file mode 100644 index 00000000..fffc6567 --- /dev/null +++ b/evidence/reconciliation.json @@ -0,0 +1,3328 @@ +{ + "status": "PASS", + "grace_seconds_in_isolated_test": 5, + "production_grace_seconds": 300, + "real_process_crash_exit": 86, + "dry_run": { + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 68, + "metadata_scanned": 67, + "s3_list_calls": 1, + "s3_head_calls": 131, + "s3_delete_calls": 0, + "skipped_fresh": 3, + "skipped_busy": 1, + "skipped_in_use": 13, + "action_counts": { + "abort_pending": 3, + "delete_orphan": 1, + "missing_object": 1, + "corrupt_object": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "delete_orphan", + "key": "originals/ded2de51-dd1a-41e3-962e-8a007880f78b" + }, + { + "kind": "abort_pending", + "key": "originals/7d93495e-403b-44ba-bbd6-08e7ab4e5f1d" + }, + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "missing_object", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + }, + { + "kind": "corrupt_object", + "key": "originals/f032c633-e0ec-4f52-8298-4b6497c6f0ef" + } + ], + "elapsed_seconds": 0.236072 + }, + "applied": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 68, + "metadata_scanned": 66, + "s3_list_calls": 1, + "s3_head_calls": 133, + "s3_delete_calls": 2, + "skipped_fresh": 3, + "skipped_busy": 1, + "skipped_in_use": 13, + "action_counts": { + "abort_pending": 2, + "delete_orphan": 1, + "missing_object": 1, + "corrupt_object": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "delete_orphan", + "key": "originals/ded2de51-dd1a-41e3-962e-8a007880f78b" + }, + { + "kind": "abort_pending", + "key": "originals/7d93495e-403b-44ba-bbd6-08e7ab4e5f1d" + }, + { + "kind": "missing_object", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + }, + { + "kind": "corrupt_object", + "key": "originals/f032c633-e0ec-4f52-8298-4b6497c6f0ef" + } + ], + "elapsed_seconds": 0.16314 + }, + "repeated": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 125, + "s3_delete_calls": 0, + "skipped_fresh": 5, + "skipped_busy": 0, + "skipped_in_use": 13, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.185942 + }, + "recovered": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 66, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 126, + "s3_delete_calls": 0, + "skipped_fresh": 5, + "skipped_busy": 0, + "skipped_in_use": 13, + "action_counts": { + "restore_ready": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "restore_ready", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + } + ], + "elapsed_seconds": 0.193984 + }, + "protection": { + "fresh_orphan_preserved": true, + "fresh_ready_preserved": true, + "used_ready_preserved": true, + "active_lock_preserved": true, + "missing_metadata_and_links_retained": true, + "dry_run_no_mutation": true, + "second_apply_actions": 0 + }, + "dependency_failure_checks": [ + "s3_503_aborts_without_missing_inference", + "db_outage_aborts_before_s3_mutation" + ], + "scan_cost_runs": [ + { + "extra_orphans": 0, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.172625 + }, + { + "extra_orphans": 0, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.15797 + }, + { + "extra_orphans": 0, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.151176 + }, + { + "extra_orphans": 50, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.331819 + }, + { + "extra_orphans": 50, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.228479 + }, + { + "extra_orphans": 50, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.219672 + }, + { + "extra_orphans": 200, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.590213 + }, + { + "extra_orphans": 200, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.687121 + }, + { + "extra_orphans": 200, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.460906 + } + ] +} diff --git a/evidence/research-consistency-compose.txt b/evidence/research-consistency-compose.txt new file mode 100644 index 00000000..c89b25a0 --- /dev/null +++ b/evidence/research-consistency-compose.txt @@ -0,0 +1,14 @@ + Container mtusi-lab4-reconciler-1 Stopping + Container mtusi-lab4-reconciler-1 Stopped + Container mtusi-lab4-s3-init-1 Starting + Container mtusi-lab4-db-1 Waiting + Container mtusi-lab4-s3-init-1 Started + Container mtusi-lab4-db-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-s3-init-1 Waiting + Container mtusi-lab4-s3-init-1 Exited + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-reconciler-1 Starting + Container mtusi-lab4-reconciler-1 Started diff --git a/evidence/research-consistency.txt b/evidence/research-consistency.txt new file mode 100644 index 00000000..dde806e8 --- /dev/null +++ b/evidence/research-consistency.txt @@ -0,0 +1,3329 @@ +{ + "status": "PASS", + "grace_seconds_in_isolated_test": 5, + "production_grace_seconds": 300, + "real_process_crash_exit": 86, + "dry_run": { + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 68, + "metadata_scanned": 67, + "s3_list_calls": 1, + "s3_head_calls": 131, + "s3_delete_calls": 0, + "skipped_fresh": 3, + "skipped_busy": 1, + "skipped_in_use": 13, + "action_counts": { + "abort_pending": 3, + "delete_orphan": 1, + "missing_object": 1, + "corrupt_object": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "delete_orphan", + "key": "originals/ded2de51-dd1a-41e3-962e-8a007880f78b" + }, + { + "kind": "abort_pending", + "key": "originals/7d93495e-403b-44ba-bbd6-08e7ab4e5f1d" + }, + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "missing_object", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + }, + { + "kind": "corrupt_object", + "key": "originals/f032c633-e0ec-4f52-8298-4b6497c6f0ef" + } + ], + "elapsed_seconds": 0.236072 + }, + "applied": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 68, + "metadata_scanned": 66, + "s3_list_calls": 1, + "s3_head_calls": 133, + "s3_delete_calls": 2, + "skipped_fresh": 3, + "skipped_busy": 1, + "skipped_in_use": 13, + "action_counts": { + "abort_pending": 2, + "delete_orphan": 1, + "missing_object": 1, + "corrupt_object": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "abort_pending", + "key": "originals/8d437eec-12d0-41ff-b3e2-0c9db96bb973" + }, + { + "kind": "delete_orphan", + "key": "originals/ded2de51-dd1a-41e3-962e-8a007880f78b" + }, + { + "kind": "abort_pending", + "key": "originals/7d93495e-403b-44ba-bbd6-08e7ab4e5f1d" + }, + { + "kind": "missing_object", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + }, + { + "kind": "corrupt_object", + "key": "originals/f032c633-e0ec-4f52-8298-4b6497c6f0ef" + } + ], + "elapsed_seconds": 0.16314 + }, + "repeated": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 125, + "s3_delete_calls": 0, + "skipped_fresh": 5, + "skipped_busy": 0, + "skipped_in_use": 13, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.185942 + }, + "recovered": { + "mode": "apply", + "grace_seconds": 5, + "objects_scanned": 66, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 126, + "s3_delete_calls": 0, + "skipped_fresh": 5, + "skipped_busy": 0, + "skipped_in_use": 13, + "action_counts": { + "restore_ready": 1 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "restore_ready", + "key": "originals/b8326da2-612b-4bb8-b25f-1f29d4f2d81b" + } + ], + "elapsed_seconds": 0.193984 + }, + "protection": { + "fresh_orphan_preserved": true, + "fresh_ready_preserved": true, + "used_ready_preserved": true, + "active_lock_preserved": true, + "missing_metadata_and_links_retained": true, + "dry_run_no_mutation": true, + "second_apply_actions": 0 + }, + "dependency_failure_checks": [ + "s3_503_aborts_without_missing_inference", + "db_outage_aborts_before_s3_mutation" + ], + "scan_cost_runs": [ + { + "extra_orphans": 0, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.172625 + }, + { + "extra_orphans": 0, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.15797 + }, + { + "extra_orphans": 0, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 65, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 130, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": {}, + "actions_truncated": 0, + "actions": [], + "elapsed_seconds": 0.151176 + }, + { + "extra_orphans": 50, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.331819 + }, + { + "extra_orphans": 50, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.228479 + }, + { + "extra_orphans": 50, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 115, + "metadata_scanned": 65, + "s3_list_calls": 1, + "s3_head_calls": 180, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 50 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + } + ], + "elapsed_seconds": 0.219672 + }, + { + "extra_orphans": 200, + "repeat": 1, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.590213 + }, + { + "extra_orphans": 200, + "repeat": 2, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.687121 + }, + { + "extra_orphans": 200, + "repeat": 3, + "mode": "dry-run", + "grace_seconds": 5, + "objects_scanned": 265, + "metadata_scanned": 65, + "s3_list_calls": 2, + "s3_head_calls": 330, + "s3_delete_calls": 0, + "skipped_fresh": 0, + "skipped_busy": 0, + "skipped_in_use": 14, + "action_counts": { + "delete_orphan": 200 + }, + "actions_truncated": 0, + "actions": [ + { + "kind": "delete_orphan", + "key": "originals/00392095-1de7-4c80-9b4c-2af1b11c9ddf" + }, + { + "kind": "delete_orphan", + "key": "originals/00645996-cdb6-4952-9128-f61a01fe4ab4" + }, + { + "kind": "delete_orphan", + "key": "originals/006c5cbd-f7d7-4a86-9b9e-b026ba4efdc5" + }, + { + "kind": "delete_orphan", + "key": "originals/016f1367-b80b-46db-91d1-a675a4725020" + }, + { + "kind": "delete_orphan", + "key": "originals/0407e07d-0cc5-4e20-a4b8-350fe840c7cc" + }, + { + "kind": "delete_orphan", + "key": "originals/04b3eb4b-8ab5-4580-a4b0-e69b02376941" + }, + { + "kind": "delete_orphan", + "key": "originals/05eb4b8f-2beb-4d67-8515-05b0bea135fe" + }, + { + "kind": "delete_orphan", + "key": "originals/05f27d0f-5b58-4b1e-a5cf-98972bfa595e" + }, + { + "kind": "delete_orphan", + "key": "originals/098f57e3-1ff9-4b53-a463-4458965fac3c" + }, + { + "kind": "delete_orphan", + "key": "originals/0f8335a4-136e-423e-8ace-5994ec88b4ba" + }, + { + "kind": "delete_orphan", + "key": "originals/0fb2dcaf-b040-4221-8100-2bba3400b228" + }, + { + "kind": "delete_orphan", + "key": "originals/0fd3ab69-70c9-4b72-8b36-3d09ff992368" + }, + { + "kind": "delete_orphan", + "key": "originals/11d522d5-82e3-4ddc-bd4f-2042478de542" + }, + { + "kind": "delete_orphan", + "key": "originals/141db3b6-b00b-4e54-a91e-d7116ee0db36" + }, + { + "kind": "delete_orphan", + "key": "originals/149acc20-77e6-4a4f-a1d0-470c0c6eaa1d" + }, + { + "kind": "delete_orphan", + "key": "originals/15597216-4f09-4e14-bf64-8581bf73c74f" + }, + { + "kind": "delete_orphan", + "key": "originals/15febd38-a615-4aac-a79e-cea1018cf31d" + }, + { + "kind": "delete_orphan", + "key": "originals/169c2fbf-7f21-4cc4-a282-08a28ce1271a" + }, + { + "kind": "delete_orphan", + "key": "originals/19a751ba-a762-4a10-b3cd-5dc3a0de13e9" + }, + { + "kind": "delete_orphan", + "key": "originals/1b7db63e-caba-48b7-8ad6-9d98760d8de3" + }, + { + "kind": "delete_orphan", + "key": "originals/1cfe303a-33c0-4d11-bad1-b646eedafd5e" + }, + { + "kind": "delete_orphan", + "key": "originals/1d5a6dc2-3565-4c8e-8d38-6ea3766b5a73" + }, + { + "kind": "delete_orphan", + "key": "originals/20f31c2e-5046-4599-92c3-9449b9ffcabf" + }, + { + "kind": "delete_orphan", + "key": "originals/216e2b20-2897-43c2-8cfe-6c3103a706af" + }, + { + "kind": "delete_orphan", + "key": "originals/21e005f7-649c-42de-affe-d0b8521d1d51" + }, + { + "kind": "delete_orphan", + "key": "originals/226868fa-03ab-4e57-8aff-b2883655ce19" + }, + { + "kind": "delete_orphan", + "key": "originals/2353918c-8ea8-4674-a05b-25612ebb4652" + }, + { + "kind": "delete_orphan", + "key": "originals/23a87cb2-dc05-41e7-a717-31c73cc74203" + }, + { + "kind": "delete_orphan", + "key": "originals/2517056b-e94b-4bbf-83a2-ef4059fe3cea" + }, + { + "kind": "delete_orphan", + "key": "originals/28f3d3d9-8c60-4c09-a852-bbe496f42549" + }, + { + "kind": "delete_orphan", + "key": "originals/2a3b3127-9c5b-42b7-8a5b-7d5a17c6dd0a" + }, + { + "kind": "delete_orphan", + "key": "originals/2c38d5b2-c16e-414f-a2ae-f797f81c7685" + }, + { + "kind": "delete_orphan", + "key": "originals/2e532cb3-7c9c-42f3-beab-3115f1002a2c" + }, + { + "kind": "delete_orphan", + "key": "originals/2fd55059-0a19-4215-8e22-0fb80bf92343" + }, + { + "kind": "delete_orphan", + "key": "originals/30af6a4c-07de-4030-a610-79d9a2da3a5d" + }, + { + "kind": "delete_orphan", + "key": "originals/32588605-9a56-4e1b-aa01-d4711e4c9a22" + }, + { + "kind": "delete_orphan", + "key": "originals/34761eb0-0b12-467a-9fe8-3b1fa666d8ab" + }, + { + "kind": "delete_orphan", + "key": "originals/34a4abce-a105-46fb-8f93-1acdf5b287e9" + }, + { + "kind": "delete_orphan", + "key": "originals/36982ca5-69ec-474e-a87d-a84d0a5fd63c" + }, + { + "kind": "delete_orphan", + "key": "originals/36cd2b2c-24ae-4bcc-9cec-f7a62db135aa" + }, + { + "kind": "delete_orphan", + "key": "originals/396af89b-5d1d-44f5-8294-4e7ab454342e" + }, + { + "kind": "delete_orphan", + "key": "originals/39acd73b-d051-417a-9f1e-431aff59c20c" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d0664-2947-40d2-8502-ff7c7632ef62" + }, + { + "kind": "delete_orphan", + "key": "originals/3b5d274b-0f8c-4f4a-8149-234c23bc74b5" + }, + { + "kind": "delete_orphan", + "key": "originals/3cf09679-4059-4cd7-ad79-70ee28c9fb93" + }, + { + "kind": "delete_orphan", + "key": "originals/3e2968e1-845a-4316-8899-945c37ed671b" + }, + { + "kind": "delete_orphan", + "key": "originals/40bf3899-e130-4747-806b-6b354414fb12" + }, + { + "kind": "delete_orphan", + "key": "originals/42b34b54-b8c7-4ceb-a096-1640a09d73d3" + }, + { + "kind": "delete_orphan", + "key": "originals/43c20a2f-5ed1-4c51-bbea-f49eb6fdb260" + }, + { + "kind": "delete_orphan", + "key": "originals/44a61e67-7b98-4d5d-abd1-a218e9da9256" + }, + { + "kind": "delete_orphan", + "key": "originals/454f164d-81e0-41bc-9a0a-d65329b32517" + }, + { + "kind": "delete_orphan", + "key": "originals/46ab73f7-6854-4aee-8606-5eefe42bf9d9" + }, + { + "kind": "delete_orphan", + "key": "originals/4a486829-ce3a-4973-b044-5d61683029a2" + }, + { + "kind": "delete_orphan", + "key": "originals/4b2b3db6-3a95-4e8c-b34e-6614c448b717" + }, + { + "kind": "delete_orphan", + "key": "originals/4c507a22-16cc-46f6-afa9-c08018843cdf" + }, + { + "kind": "delete_orphan", + "key": "originals/4c75946b-aea7-4f26-889e-9cb17a687a6e" + }, + { + "kind": "delete_orphan", + "key": "originals/4d121eb0-b006-470f-90e6-8a79561a388f" + }, + { + "kind": "delete_orphan", + "key": "originals/4f2a0575-031d-4858-9df5-1df1e513a3f1" + }, + { + "kind": "delete_orphan", + "key": "originals/4fb1f6a4-0ee1-4b0a-bc80-4b4ea97909a6" + }, + { + "kind": "delete_orphan", + "key": "originals/505d6b68-d0fa-48e6-8233-d53cefab5de1" + }, + { + "kind": "delete_orphan", + "key": "originals/512311ac-ae82-476e-a4f9-3a88f33b18f5" + }, + { + "kind": "delete_orphan", + "key": "originals/514cbb1d-2b9e-45c8-bfd6-5fdf1e81dae1" + }, + { + "kind": "delete_orphan", + "key": "originals/53f2d81d-4971-49b7-ae41-f14df81ad7ba" + }, + { + "kind": "delete_orphan", + "key": "originals/541052d6-6167-49ca-8a8a-b8a3ee39e549" + }, + { + "kind": "delete_orphan", + "key": "originals/54dd5d4c-feb9-4012-94d7-55b4efb2e8df" + }, + { + "kind": "delete_orphan", + "key": "originals/57c6ed5e-7b05-4e0d-b2ec-e66df8109377" + }, + { + "kind": "delete_orphan", + "key": "originals/58c359b5-eda4-47c4-955c-9326b2c18500" + }, + { + "kind": "delete_orphan", + "key": "originals/5999f2f5-078f-43cf-9404-3e94287c4768" + }, + { + "kind": "delete_orphan", + "key": "originals/5b3dbeb2-41ed-44a6-a929-caeca95cc563" + }, + { + "kind": "delete_orphan", + "key": "originals/5c16fc68-52f2-4f8c-b0d9-e08c3c2117ff" + }, + { + "kind": "delete_orphan", + "key": "originals/6041ce72-011d-41bc-84e1-93dc43df283a" + }, + { + "kind": "delete_orphan", + "key": "originals/629bfd2e-a0ea-4151-8f33-01a838a25576" + }, + { + "kind": "delete_orphan", + "key": "originals/6a41a96e-5730-453b-b7d7-5ae9365f6edc" + }, + { + "kind": "delete_orphan", + "key": "originals/6b7fa7f7-066c-41d4-ae95-30e4d82b56bd" + }, + { + "kind": "delete_orphan", + "key": "originals/6ba55679-83d5-476a-85e9-568487c6f72c" + }, + { + "kind": "delete_orphan", + "key": "originals/6c36ece1-4f63-41dc-a472-401f7d3f26ca" + }, + { + "kind": "delete_orphan", + "key": "originals/6d3d453b-2bd4-4284-8a77-4471386c62c2" + }, + { + "kind": "delete_orphan", + "key": "originals/6e5a6f0d-744f-47cf-8e5d-23684f8b42a6" + }, + { + "kind": "delete_orphan", + "key": "originals/6fc82eeb-0db6-4e71-8576-103cb20da77f" + }, + { + "kind": "delete_orphan", + "key": "originals/70fbceb3-d0a1-401e-82ad-be39d288e84f" + }, + { + "kind": "delete_orphan", + "key": "originals/726fdde2-5a75-4093-864f-87987a823c0a" + }, + { + "kind": "delete_orphan", + "key": "originals/73f367e6-9598-46de-a779-093241be75d9" + }, + { + "kind": "delete_orphan", + "key": "originals/74e1ea7c-219c-4ecb-aa2c-84a300f48f9a" + }, + { + "kind": "delete_orphan", + "key": "originals/758a391c-9fe0-4706-839d-06568752ccdf" + }, + { + "kind": "delete_orphan", + "key": "originals/761bce8c-ece8-43fd-a331-f59bfe105b2d" + }, + { + "kind": "delete_orphan", + "key": "originals/7667a660-d9f1-4bcc-9eb7-050590ae8ada" + }, + { + "kind": "delete_orphan", + "key": "originals/76c86753-effc-4d22-95a5-77632c161457" + }, + { + "kind": "delete_orphan", + "key": "originals/780b11a1-0ca9-440b-8043-843a058c9401" + }, + { + "kind": "delete_orphan", + "key": "originals/78c84a5c-72d2-4280-adbd-e63047701423" + }, + { + "kind": "delete_orphan", + "key": "originals/796930b8-28dc-478d-9bc0-4736d8a4a0f1" + }, + { + "kind": "delete_orphan", + "key": "originals/7b019bd8-eccf-4112-b87d-278a816bfe34" + }, + { + "kind": "delete_orphan", + "key": "originals/7ebf7006-5501-47d1-a051-ab7c9cff217f" + }, + { + "kind": "delete_orphan", + "key": "originals/7ec22ff7-3573-4387-9606-0da4d4b84510" + }, + { + "kind": "delete_orphan", + "key": "originals/7fa42515-0d30-415b-bbf7-12f028a7fa6a" + }, + { + "kind": "delete_orphan", + "key": "originals/7fe94ee2-c47a-4ff3-8179-151f3be4c9ee" + }, + { + "kind": "delete_orphan", + "key": "originals/831efc84-15ee-4ab8-a82d-62ed6548b6e4" + }, + { + "kind": "delete_orphan", + "key": "originals/83d9a844-1800-48d0-a881-66525a1b717a" + }, + { + "kind": "delete_orphan", + "key": "originals/85329b71-1a37-41fb-8548-5e544cebfe83" + }, + { + "kind": "delete_orphan", + "key": "originals/85490cbf-42bf-448b-b164-35c8e814cd0d" + }, + { + "kind": "delete_orphan", + "key": "originals/8591d252-5251-4296-9173-6a2f8e4152d1" + }, + { + "kind": "delete_orphan", + "key": "originals/870fc8dc-599c-4f71-9a01-1b5aebf5f232" + }, + { + "kind": "delete_orphan", + "key": "originals/8727a458-a793-4e06-9a52-f0cf9dff21be" + }, + { + "kind": "delete_orphan", + "key": "originals/8769b36b-aa5d-45d8-88fb-c8c289722f54" + }, + { + "kind": "delete_orphan", + "key": "originals/880894f8-9fe3-40ef-83cd-9481527ed922" + }, + { + "kind": "delete_orphan", + "key": "originals/88d139cd-1686-4cd0-8220-4145a8bc4750" + }, + { + "kind": "delete_orphan", + "key": "originals/89e9742a-d55b-4f0b-ad67-39be0e5ab24d" + }, + { + "kind": "delete_orphan", + "key": "originals/8a374a35-2930-44b2-b569-e1ae8275af34" + }, + { + "kind": "delete_orphan", + "key": "originals/8e07be4b-a58a-4573-8e42-06f0bdda887a" + }, + { + "kind": "delete_orphan", + "key": "originals/8e8ec5aa-c37a-4f97-8d32-dd96a8ba3031" + }, + { + "kind": "delete_orphan", + "key": "originals/8f13b17a-0a4b-4379-9b90-a35decc81377" + }, + { + "kind": "delete_orphan", + "key": "originals/8f17afb6-5ed8-4daa-9ed0-e96423098b69" + }, + { + "kind": "delete_orphan", + "key": "originals/8f8f87af-1377-44ea-a9d1-f40464a360e3" + }, + { + "kind": "delete_orphan", + "key": "originals/8fa1cc1a-980b-42f9-b1cf-abfa69f0a034" + }, + { + "kind": "delete_orphan", + "key": "originals/9080fa56-117f-4d02-bc07-995a56f5aa14" + }, + { + "kind": "delete_orphan", + "key": "originals/921ca2c0-6ce3-4e34-b5a8-0ecd24443401" + }, + { + "kind": "delete_orphan", + "key": "originals/935a8def-cf73-4837-bd05-c2d351b312d2" + }, + { + "kind": "delete_orphan", + "key": "originals/94043bac-33d6-44a6-8628-07eefa79e57b" + }, + { + "kind": "delete_orphan", + "key": "originals/944906f1-724f-4175-9720-1d647fffb8e3" + }, + { + "kind": "delete_orphan", + "key": "originals/9455d15e-5bc4-4ee1-ab0c-6494119039f2" + }, + { + "kind": "delete_orphan", + "key": "originals/9672ac82-1541-4afd-83ee-35f1d36b706c" + }, + { + "kind": "delete_orphan", + "key": "originals/9d281625-6571-4a04-806e-18fd7df9d0d2" + }, + { + "kind": "delete_orphan", + "key": "originals/9fc32c23-34cb-4b80-9d3a-2febfbd56586" + }, + { + "kind": "delete_orphan", + "key": "originals/a0a00c50-a433-4a99-980c-66df1906f8fe" + }, + { + "kind": "delete_orphan", + "key": "originals/a169adbc-ab98-4f62-a499-dd3e11e8ac6a" + }, + { + "kind": "delete_orphan", + "key": "originals/a21c7a69-a0e9-4a82-83ca-22b1539d811a" + }, + { + "kind": "delete_orphan", + "key": "originals/a31eb88e-2d33-4fda-8684-a57a78e25bbc" + }, + { + "kind": "delete_orphan", + "key": "originals/a4507a75-5a68-4cad-93bc-df2134f9e284" + }, + { + "kind": "delete_orphan", + "key": "originals/a48a24a1-3fe4-476c-922b-3434a5c48b57" + }, + { + "kind": "delete_orphan", + "key": "originals/a65fd4a7-2921-4320-b8db-ab92b692b4d8" + }, + { + "kind": "delete_orphan", + "key": "originals/a860ac52-2c16-4970-b5c4-1c5d05a1095e" + }, + { + "kind": "delete_orphan", + "key": "originals/a89e05bc-8fed-4808-a54e-76832d6155d1" + }, + { + "kind": "delete_orphan", + "key": "originals/aabb3d39-39ab-4382-a42b-0ddfb96c8522" + }, + { + "kind": "delete_orphan", + "key": "originals/ac821e9e-16f5-4d2d-810a-bbe017959963" + }, + { + "kind": "delete_orphan", + "key": "originals/ae412ffa-ada0-4403-a055-a4f2d4914408" + }, + { + "kind": "delete_orphan", + "key": "originals/aedbb26c-a67f-49bd-8e7c-14effa83e0a4" + }, + { + "kind": "delete_orphan", + "key": "originals/af54fdf7-05fb-4ab6-b0f1-3e4164948ce4" + }, + { + "kind": "delete_orphan", + "key": "originals/afedaeb9-888f-4c21-ace2-925ec650c818" + }, + { + "kind": "delete_orphan", + "key": "originals/b0376016-9e05-42f7-b0e7-5652022fe73b" + }, + { + "kind": "delete_orphan", + "key": "originals/b11b7f00-806a-4dea-b7da-009d1063e5f8" + }, + { + "kind": "delete_orphan", + "key": "originals/b19c8d2c-150a-481b-8a0a-1d82bfa8ff12" + }, + { + "kind": "delete_orphan", + "key": "originals/b30d0c3e-cd05-4926-bf6f-a6c099d496ba" + }, + { + "kind": "delete_orphan", + "key": "originals/b314387c-6b3b-4dd0-944b-2275b5f58f79" + }, + { + "kind": "delete_orphan", + "key": "originals/b3ea8e45-610a-40e4-a755-b1a125c0c92c" + }, + { + "kind": "delete_orphan", + "key": "originals/b499ac38-23b2-4d4d-aefa-c35f3a638b40" + }, + { + "kind": "delete_orphan", + "key": "originals/b5a51d13-3ddd-44f4-bf7f-84aa1597c508" + }, + { + "kind": "delete_orphan", + "key": "originals/b5c4c2e1-cc82-4925-a48a-8379bc4e1b46" + }, + { + "kind": "delete_orphan", + "key": "originals/b9b7bad3-9a0f-4cbe-a309-daafe3c31e92" + }, + { + "kind": "delete_orphan", + "key": "originals/b9f29a04-e97c-4215-ba62-38f2af88316d" + }, + { + "kind": "delete_orphan", + "key": "originals/baa672ce-547f-40f3-9e3d-51e0a6886380" + }, + { + "kind": "delete_orphan", + "key": "originals/bb978e76-e3f4-4d26-aad0-7dd96b033db0" + }, + { + "kind": "delete_orphan", + "key": "originals/bbbf5782-eddd-4d66-a442-9eb56ec9a087" + }, + { + "kind": "delete_orphan", + "key": "originals/bcfa9867-5c65-420f-98ef-3c2aa8ff17cd" + }, + { + "kind": "delete_orphan", + "key": "originals/bdab80a1-0bf9-4ac4-8e7a-9814ab393a04" + }, + { + "kind": "delete_orphan", + "key": "originals/c2bc0235-08bb-4e35-97f3-0ee4e4a72a5a" + }, + { + "kind": "delete_orphan", + "key": "originals/c2dabe59-ccaa-4b98-84cc-754c41217f62" + }, + { + "kind": "delete_orphan", + "key": "originals/c38dfa28-fc88-4038-88ab-73b4d8af360c" + }, + { + "kind": "delete_orphan", + "key": "originals/c3b345cc-2787-4100-a377-7e5ef09a4573" + }, + { + "kind": "delete_orphan", + "key": "originals/c4c8794f-5daf-4fbc-9a58-9de827186ffb" + }, + { + "kind": "delete_orphan", + "key": "originals/c6290010-0e7b-4bcb-ac6e-68bfbcee1b24" + }, + { + "kind": "delete_orphan", + "key": "originals/c6ee825d-52db-47f9-8673-e55bffe45414" + }, + { + "kind": "delete_orphan", + "key": "originals/c7f625d9-a2da-4fdb-8719-5a200eb66961" + }, + { + "kind": "delete_orphan", + "key": "originals/c820696d-2ce1-4879-b31d-ab2409b7525b" + }, + { + "kind": "delete_orphan", + "key": "originals/c93b7cab-3a75-4bf4-abf0-0b0ca742f832" + }, + { + "kind": "delete_orphan", + "key": "originals/c9c4616a-1a5f-4c48-a7fb-6f8ced0dd853" + }, + { + "kind": "delete_orphan", + "key": "originals/cb0d15ac-e10d-43f7-a832-fe52a36cb512" + }, + { + "kind": "delete_orphan", + "key": "originals/cc005cb6-84ac-48f0-8149-0460f7b2349d" + }, + { + "kind": "delete_orphan", + "key": "originals/ce169a0b-a4c5-4b01-b366-765e192696ad" + }, + { + "kind": "delete_orphan", + "key": "originals/ce3386f1-1c53-4162-a1d9-9b831614e68f" + }, + { + "kind": "delete_orphan", + "key": "originals/ce69d2cc-c493-4e3f-8995-49e7bf71be88" + }, + { + "kind": "delete_orphan", + "key": "originals/cebb1867-efbb-4752-bd59-ac50da5af5ea" + }, + { + "kind": "delete_orphan", + "key": "originals/cfdfb585-5b3b-44b2-b190-4285e17af603" + }, + { + "kind": "delete_orphan", + "key": "originals/d438fe9c-90c9-4c6b-bc75-665fea594c3f" + }, + { + "kind": "delete_orphan", + "key": "originals/d7c35408-e4cb-49a4-bda0-d5b2cbca5055" + }, + { + "kind": "delete_orphan", + "key": "originals/d7f3b16f-5e77-44a3-a3d6-61e80a713ca5" + }, + { + "kind": "delete_orphan", + "key": "originals/dcd0e85b-36f5-4354-96ab-598d9311a36a" + }, + { + "kind": "delete_orphan", + "key": "originals/dd05bffe-024c-4e74-8a25-83fcc7472f44" + }, + { + "kind": "delete_orphan", + "key": "originals/de1f02e8-cf5b-45cd-8293-216c73b806fd" + }, + { + "kind": "delete_orphan", + "key": "originals/df14f47e-6e83-4f6d-b8b9-64dd01c19570" + }, + { + "kind": "delete_orphan", + "key": "originals/e0f4e17e-2eb3-45f3-b2fc-9d546c2c4b83" + }, + { + "kind": "delete_orphan", + "key": "originals/e11aadd6-2e86-466d-a4d2-c8211e2e743a" + }, + { + "kind": "delete_orphan", + "key": "originals/e143cb23-0165-4ba2-9cf6-ed291d6cbda0" + }, + { + "kind": "delete_orphan", + "key": "originals/e377a78f-d163-4c9a-baa9-2ece5a472302" + }, + { + "kind": "delete_orphan", + "key": "originals/e493f8e3-368b-4f24-8d9f-57f898ec1b29" + }, + { + "kind": "delete_orphan", + "key": "originals/e5c20ff2-2d66-46fb-899c-a55770b776da" + }, + { + "kind": "delete_orphan", + "key": "originals/e7479dcf-f564-44f9-9c93-29664e1e1731" + }, + { + "kind": "delete_orphan", + "key": "originals/e7f0263d-d6e2-4496-bde5-46f67129fc8d" + }, + { + "kind": "delete_orphan", + "key": "originals/e8b6c23d-cc4b-401b-935b-c14818dbb129" + }, + { + "kind": "delete_orphan", + "key": "originals/eb04e77e-58e7-463d-ad41-3ff383863e91" + }, + { + "kind": "delete_orphan", + "key": "originals/ee418a99-e458-41c4-a3a9-891c0820638f" + }, + { + "kind": "delete_orphan", + "key": "originals/ee54a9c8-ecf6-4420-b666-28bde3473504" + }, + { + "kind": "delete_orphan", + "key": "originals/f07096bd-06de-477d-85aa-698914c37be4" + }, + { + "kind": "delete_orphan", + "key": "originals/f121249c-a54f-4c29-9f7a-9f53088812d2" + }, + { + "kind": "delete_orphan", + "key": "originals/f5aa7688-3dc4-4e37-a70d-aefca962c826" + }, + { + "kind": "delete_orphan", + "key": "originals/f66332d3-7d01-4ee4-ac93-b1241ee97cf6" + }, + { + "kind": "delete_orphan", + "key": "originals/f8f6df8a-f64c-4714-a36f-3b4c5b19cd11" + }, + { + "kind": "delete_orphan", + "key": "originals/f91fc8de-85f5-4c18-a80b-2de866b858f1" + }, + { + "kind": "delete_orphan", + "key": "originals/f9d89e8d-293a-4ebe-9089-04e02d49d1ff" + }, + { + "kind": "delete_orphan", + "key": "originals/f9deb944-42ab-4f10-8ee8-be940502dc44" + }, + { + "kind": "delete_orphan", + "key": "originals/fa61939c-094c-48fc-af1f-f22519357c0f" + }, + { + "kind": "delete_orphan", + "key": "originals/fe61538b-bbbc-4f4a-8530-7390ae9ef634" + } + ], + "elapsed_seconds": 0.460906 + } + ] +} + diff --git a/evidence/research-summary.json b/evidence/research-summary.json new file mode 100644 index 00000000..8b1a2b22 --- /dev/null +++ b/evidence/research-summary.json @@ -0,0 +1,106 @@ +{ + "scan_cost": [ + { + "extra_orphans": 0, + "objects_scanned": 65, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 160.59, + "min_ms": 151.176, + "max_ms": 172.625, + "stdev_ms": 10.962, + "list_calls": 1, + "head_calls": 130 + }, + { + "extra_orphans": 50, + "objects_scanned": 115, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 259.99, + "min_ms": 219.672, + "max_ms": 331.819, + "stdev_ms": 62.361, + "list_calls": 1, + "head_calls": 180 + }, + { + "extra_orphans": 200, + "objects_scanned": 265, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 579.413, + "min_ms": 460.906, + "max_ms": 687.121, + "stdev_ms": 113.494, + "list_calls": 2, + "head_calls": 330 + } + ], + "upload_phases": [ + { + "width": 512, + "height": 512, + "size_bytes": 787271, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982275.5182576, + "finished_unix": 1793982276.575706, + "elapsed_seconds": 1.057448, + "successful_rps": 18.913, + "latency_ms": { + "p50": 102.767, + "p95": 176.198, + "p99": 183.16 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 91.094 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 91.312 + } + ] + }, + { + "width": 1536, + "height": 1536, + "size_bytes": 7081758, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982276.6054053, + "finished_unix": 1793982278.9907844, + "elapsed_seconds": 2.385379, + "successful_rps": 8.384, + "latency_ms": { + "p50": 617.87, + "p95": 832.535, + "p99": 896.355 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 124.629 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 146.086 + } + ] + } + ] +} diff --git a/evidence/s3-bootstrap.txt b/evidence/s3-bootstrap.txt new file mode 100644 index 00000000..cf546bcd --- /dev/null +++ b/evidence/s3-bootstrap.txt @@ -0,0 +1,6 @@ +s3-init-1 | PASS: private bucket and restricted application principal configured +s3-init-1 | PASS: private bucket and restricted application principal configured +s3-init-1 | PASS: private bucket and restricted application principal configured +s3-init-1 | PASS: private bucket and restricted application principal configured +s3-init-1 | PASS: private bucket and restricted application principal configured +s3-init-1 | PASS: private bucket and restricted application principal configured diff --git a/evidence/scan-cost-summary.json b/evidence/scan-cost-summary.json new file mode 100644 index 00000000..1966df8c --- /dev/null +++ b/evidence/scan-cost-summary.json @@ -0,0 +1,38 @@ +[ + { + "extra_orphans": 0, + "objects_scanned": 65, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 160.59, + "min_ms": 151.176, + "max_ms": 172.625, + "stdev_ms": 10.962, + "list_calls": 1, + "head_calls": 130 + }, + { + "extra_orphans": 50, + "objects_scanned": 115, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 259.99, + "min_ms": 219.672, + "max_ms": 331.819, + "stdev_ms": 62.361, + "list_calls": 1, + "head_calls": 180 + }, + { + "extra_orphans": 200, + "objects_scanned": 265, + "metadata_scanned": 65, + "repeats": 3, + "mean_ms": 579.413, + "min_ms": 460.906, + "max_ms": 687.121, + "stdev_ms": 113.494, + "list_calls": 2, + "head_calls": 330 + } +] diff --git a/evidence/scan-cost.csv b/evidence/scan-cost.csv new file mode 100644 index 00000000..77924b06 --- /dev/null +++ b/evidence/scan-cost.csv @@ -0,0 +1,4 @@ +extra_orphans,objects_scanned,metadata_scanned,repeats,mean_ms,min_ms,max_ms,stdev_ms,list_calls,head_calls +0,65,65,3,160.59,151.176,172.625,10.962,1,130 +50,115,65,3,259.99,219.672,331.819,62.361,1,180 +200,265,65,3,579.413,460.906,687.121,113.494,2,330 diff --git a/evidence/scan-cost.png b/evidence/scan-cost.png new file mode 100644 index 00000000..713ce63f Binary files /dev/null and b/evidence/scan-cost.png differ diff --git a/evidence/scan-cost.svg b/evidence/scan-cost.svg new file mode 100644 index 00000000..5e1f930d --- /dev/null +++ b/evidence/scan-cost.svg @@ -0,0 +1,1337 @@ + + + + + + + + 2026-11-06T19:43:24.713647 + image/svg+xml + + + Matplotlib v3.10.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..fac765a8 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8084" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..29d2ca34 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,1426 @@ +{ + "status": "PASS", + "sample_interval_seconds": 0.05, + "note": "memory.current includes page cache, kernel charges and a small sampler process; sampled peak is not a heap profile. lifetime_memory_peak includes prior requests.", + "workload": { + "status": "PASS", + "phases": [ + { + "width": 512, + "height": 512, + "size_bytes": 787271, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982275.5182576, + "finished_unix": 1793982276.575706, + "elapsed_seconds": 1.057448, + "successful_rps": 18.913, + "latency_ms": { + "p50": 102.767, + "p95": 176.198, + "p99": 183.16 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "individual_requests": [ + { + "elapsed_ms": 165.792, + "instance": "3586879492e3", + "id": "c2fdfeae-4fc7-4f18-aeea-15128a1ce778" + }, + { + "elapsed_ms": 115.745, + "instance": "3586879492e3", + "id": "dc2ffcd1-c0da-4d7d-bc54-fdf79d0e0c4b" + }, + { + "elapsed_ms": 152.416, + "instance": "2b11729802cd", + "id": "66a9427e-5c56-4616-b382-eecbf164f2a6" + }, + { + "elapsed_ms": 120.246, + "instance": "3586879492e3", + "id": "5fc86549-532d-4893-8adc-526480c8ca4a" + }, + { + "elapsed_ms": 183.16, + "instance": "3586879492e3", + "id": "f3103a13-e41d-4b7f-8101-be01e7d0b551" + }, + { + "elapsed_ms": 165.525, + "instance": "3586879492e3", + "id": "184a079f-1cf1-4486-a217-7d47f5159fc6" + }, + { + "elapsed_ms": 176.198, + "instance": "2b11729802cd", + "id": "123bb8c5-a739-460e-8048-0b74e021b1c4" + }, + { + "elapsed_ms": 98.997, + "instance": "2b11729802cd", + "id": "184c5689-8653-4598-9f22-66e833f44874" + }, + { + "elapsed_ms": 116.418, + "instance": "2b11729802cd", + "id": "84805db4-100b-4e73-b055-d5aeb1b08a20" + }, + { + "elapsed_ms": 152.617, + "instance": "2b11729802cd", + "id": "d7f12011-df5e-47ad-aff5-4048a834f21c" + }, + { + "elapsed_ms": 54.057, + "instance": "2b11729802cd", + "id": "175665e6-6045-420e-bcd4-b5666d5fcf7c" + }, + { + "elapsed_ms": 87.786, + "instance": "3586879492e3", + "id": "d1ef2b99-f69f-4622-a0ac-d4f45eb0da37" + }, + { + "elapsed_ms": 54.613, + "instance": "2b11729802cd", + "id": "4527cc17-d539-4e95-9ba7-cd0aec7b19ca" + }, + { + "elapsed_ms": 87.082, + "instance": "3586879492e3", + "id": "d3d16d58-61f0-4b3f-a205-6d696fa0cff6" + }, + { + "elapsed_ms": 102.767, + "instance": "3586879492e3", + "id": "690f4dbc-203a-4269-a0d6-f6089e1f80a7" + }, + { + "elapsed_ms": 53.332, + "instance": "3586879492e3", + "id": "c530cf03-e4f9-4476-b029-3210f1b79a07" + }, + { + "elapsed_ms": 85.73, + "instance": "2b11729802cd", + "id": "a401af64-d3eb-4c57-86b3-b0760aa8c140" + }, + { + "elapsed_ms": 54.686, + "instance": "3586879492e3", + "id": "294ed5d9-cbfd-4903-abfb-e576e497ef58" + }, + { + "elapsed_ms": 86.839, + "instance": "2b11729802cd", + "id": "9379c6d1-4be6-4984-80ed-3601463a6a7c" + }, + { + "elapsed_ms": 103.509, + "instance": "2b11729802cd", + "id": "bd49733e-f9cc-49b8-a2c5-7dfde60239d7" + } + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 91.094 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 91.312 + } + ] + }, + { + "width": 1536, + "height": 1536, + "size_bytes": 7081758, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982276.6054053, + "finished_unix": 1793982278.9907844, + "elapsed_seconds": 2.385379, + "successful_rps": 8.384, + "latency_ms": { + "p50": 617.87, + "p95": 832.535, + "p99": 896.355 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "individual_requests": [ + { + "elapsed_ms": 716.744, + "instance": "2b11729802cd", + "id": "de25dd86-bbff-4c73-b43b-b1a32ae54b58" + }, + { + "elapsed_ms": 519.935, + "instance": "2b11729802cd", + "id": "5e31a195-4588-4217-a6b7-f60e8fc354d3" + }, + { + "elapsed_ms": 113.752, + "instance": "3586879492e3", + "id": "3c2d016c-49b7-4389-828c-3b13142ae128" + }, + { + "elapsed_ms": 701.82, + "instance": "2b11729802cd", + "id": "b279556b-3662-4649-a63d-c3299cae9790" + }, + { + "elapsed_ms": 162.629, + "instance": "3586879492e3", + "id": "6f35eee0-b44c-4d8e-b9d7-941e969351c0" + }, + { + "elapsed_ms": 617.87, + "instance": "3586879492e3", + "id": "017476ec-3d6c-4a7f-9b2f-02d07308a308" + }, + { + "elapsed_ms": 609.742, + "instance": "3586879492e3", + "id": "beaf2bf4-e19a-403e-a034-d2a839b0723e" + }, + { + "elapsed_ms": 609.077, + "instance": "2b11729802cd", + "id": "265535f4-faf6-4b2e-b5ea-2c8707dc7c44" + }, + { + "elapsed_ms": 439.875, + "instance": "3586879492e3", + "id": "fa001db4-f120-463d-8f42-7043a28331a1" + }, + { + "elapsed_ms": 313.432, + "instance": "2b11729802cd", + "id": "bcb008ae-7734-4d5c-8056-f1196f785fed" + }, + { + "elapsed_ms": 832.535, + "instance": "2b11729802cd", + "id": "b06dbcf2-7db6-4466-a7b4-0e63c36a1fdc" + }, + { + "elapsed_ms": 896.355, + "instance": "2b11729802cd", + "id": "85880399-b7c2-47a9-b3f5-5d6d184f5050" + }, + { + "elapsed_ms": 827.618, + "instance": "2b11729802cd", + "id": "5bdce531-4c9f-45a6-a165-8efb291a2abb" + }, + { + "elapsed_ms": 519.467, + "instance": "3586879492e3", + "id": "7e8eaf86-9280-4422-b7fe-ff8ab6113f92" + }, + { + "elapsed_ms": 648.035, + "instance": "3586879492e3", + "id": "a2fc1c26-501f-40c6-8aac-65879c7afd90" + }, + { + "elapsed_ms": 627.053, + "instance": "2b11729802cd", + "id": "4e4f37a1-9dd5-4406-b0d3-50d92b3e38ae" + }, + { + "elapsed_ms": 421.265, + "instance": "3586879492e3", + "id": "be19fd8f-a481-4355-9392-4910191643ae" + }, + { + "elapsed_ms": 622.325, + "instance": "2b11729802cd", + "id": "bbc868dd-3437-455a-ae4b-3d893012f4c9" + }, + { + "elapsed_ms": 827.438, + "instance": "3586879492e3", + "id": "65a55f03-8eae-483f-badd-da0c71fbe96d" + }, + { + "elapsed_ms": 642.219, + "instance": "3586879492e3", + "id": "27e56e81-322a-47c7-892d-52484b4caa3d" + } + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 124.629 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 146.086 + } + ] + } + ] + }, + "api_traces": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "samples": [ + { + "at_unix": 1793982274.442563, + "memory_current_bytes": 81911808, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13053057\nuser_usec 11609593\nsystem_usec 1443464\ncore_sched.force_idle_usec 0\nnr_periods 1496\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.49357, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13053753\nuser_usec 11610212\nsystem_usec 1443541\ncore_sched.force_idle_usec 0\nnr_periods 1497\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.5449533, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13053987\nuser_usec 11610419\nsystem_usec 1443567\ncore_sched.force_idle_usec 0\nnr_periods 1497\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.59525, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13054166\nuser_usec 11610578\nsystem_usec 1443587\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.64547, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13054610\nuser_usec 11610974\nsystem_usec 1443636\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.6957629, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13054756\nuser_usec 11611104\nsystem_usec 1443652\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.746132, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13055026\nuser_usec 11611344\nsystem_usec 1443682\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.7964904, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13055254\nuser_usec 11611546\nsystem_usec 1443707\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.8469155, + "memory_current_bytes": 82042880, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13070040\nuser_usec 11626268\nsystem_usec 1443771\ncore_sched.force_idle_usec 0\nnr_periods 1498\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.8974147, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13075795\nuser_usec 11632024\nsystem_usec 1443771\ncore_sched.force_idle_usec 0\nnr_periods 1499\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.952503, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13076034\nuser_usec 11632262\nsystem_usec 1443771\ncore_sched.force_idle_usec 0\nnr_periods 1499\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.0060055, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13076181\nuser_usec 11632403\nsystem_usec 1443777\ncore_sched.force_idle_usec 0\nnr_periods 1500\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.0563178, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13079182\nuser_usec 11635404\nsystem_usec 1443777\ncore_sched.force_idle_usec 0\nnr_periods 1500\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.1073585, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13079959\nuser_usec 11636182\nsystem_usec 1443777\ncore_sched.force_idle_usec 0\nnr_periods 1501\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.160109, + "memory_current_bytes": 111951872, + "lifetime_memory_peak_bytes": 149430272, + "cpu_stat": "usage_usec 13114371\nuser_usec 11650342\nsystem_usec 1464028\ncore_sched.force_idle_usec 0\nnr_periods 1501\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.210532, + "memory_current_bytes": 149295104, + "lifetime_memory_peak_bytes": 149504000, + "cpu_stat": "usage_usec 13209095\nuser_usec 11718531\nsystem_usec 1490563\ncore_sched.force_idle_usec 0\nnr_periods 1502\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.2608743, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149557248, + "cpu_stat": "usage_usec 13236752\nuser_usec 11744244\nsystem_usec 1492508\ncore_sched.force_idle_usec 0\nnr_periods 1502\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.3112082, + "memory_current_bytes": 114237440, + "lifetime_memory_peak_bytes": 149557248, + "cpu_stat": "usage_usec 13254259\nuser_usec 11757892\nsystem_usec 1496366\ncore_sched.force_idle_usec 0\nnr_periods 1503\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.3617926, + "memory_current_bytes": 117944320, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13354081\nuser_usec 11850567\nsystem_usec 1503513\ncore_sched.force_idle_usec 0\nnr_periods 1503\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.4120207, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13356737\nuser_usec 11852138\nsystem_usec 1504598\ncore_sched.force_idle_usec 0\nnr_periods 1504\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.4623284, + "memory_current_bytes": 149368832, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13408089\nuser_usec 11891704\nsystem_usec 1516384\ncore_sched.force_idle_usec 0\nnr_periods 1505\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.5125976, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13469972\nuser_usec 11950657\nsystem_usec 1519314\ncore_sched.force_idle_usec 0\nnr_periods 1505\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.5670567, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470248\nuser_usec 11950902\nsystem_usec 1519345\ncore_sched.force_idle_usec 0\nnr_periods 1506\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.6272008, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470329\nuser_usec 11950974\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1506\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.6773808, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470619\nuser_usec 11951264\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1507\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.7281897, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470702\nuser_usec 11951347\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1507\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.7783163, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470856\nuser_usec 11951501\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1507\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.8332112, + "memory_current_bytes": 82034688, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13470974\nuser_usec 11951619\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1507\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.883939, + "memory_current_bytes": 82132992, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13476269\nuser_usec 11956914\nsystem_usec 1519354\ncore_sched.force_idle_usec 0\nnr_periods 1508\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.936938, + "memory_current_bytes": 90021888, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13513215\nuser_usec 11979973\nsystem_usec 1533241\ncore_sched.force_idle_usec 0\nnr_periods 1508\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.9873803, + "memory_current_bytes": 95518720, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13554214\nuser_usec 12013939\nsystem_usec 1540274\ncore_sched.force_idle_usec 0\nnr_periods 1509\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.037746, + "memory_current_bytes": 93552640, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13599230\nuser_usec 12056098\nsystem_usec 1543131\ncore_sched.force_idle_usec 0\nnr_periods 1509\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.087968, + "memory_current_bytes": 85962752, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13608004\nuser_usec 12060188\nsystem_usec 1547815\ncore_sched.force_idle_usec 0\nnr_periods 1510\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.1402664, + "memory_current_bytes": 85962752, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13608360\nuser_usec 12060545\nsystem_usec 1547815\ncore_sched.force_idle_usec 0\nnr_periods 1510\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.1906989, + "memory_current_bytes": 85962752, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13608676\nuser_usec 12060861\nsystem_usec 1547815\ncore_sched.force_idle_usec 0\nnr_periods 1511\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.2424896, + "memory_current_bytes": 85962752, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13608802\nuser_usec 12060986\nsystem_usec 1547815\ncore_sched.force_idle_usec 0\nnr_periods 1511\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.292725, + "memory_current_bytes": 85966848, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13608990\nuser_usec 12061174\nsystem_usec 1547815\ncore_sched.force_idle_usec 0\nnr_periods 1511\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.3473065, + "memory_current_bytes": 85966848, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13609052\nuser_usec 12061233\nsystem_usec 1547819\ncore_sched.force_idle_usec 0\nnr_periods 1511\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.3984494, + "memory_current_bytes": 85966848, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13609313\nuser_usec 12061233\nsystem_usec 1548080\ncore_sched.force_idle_usec 0\nnr_periods 1512\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.457462, + "memory_current_bytes": 85966848, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13609436\nuser_usec 12061233\nsystem_usec 1548202\ncore_sched.force_idle_usec 0\nnr_periods 1512\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.5078745, + "memory_current_bytes": 89673728, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13640350\nuser_usec 12083538\nsystem_usec 1556812\ncore_sched.force_idle_usec 0\nnr_periods 1513\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.5584888, + "memory_current_bytes": 89546752, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13659480\nuser_usec 12097962\nsystem_usec 1561517\ncore_sched.force_idle_usec 0\nnr_periods 1513\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.608678, + "memory_current_bytes": 88969216, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13672216\nuser_usec 12110633\nsystem_usec 1561583\ncore_sched.force_idle_usec 0\nnr_periods 1514\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.6590402, + "memory_current_bytes": 88969216, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13672469\nuser_usec 12110857\nsystem_usec 1561612\ncore_sched.force_idle_usec 0\nnr_periods 1514\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.7150223, + "memory_current_bytes": 88969216, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13672561\nuser_usec 12110938\nsystem_usec 1561623\ncore_sched.force_idle_usec 0\nnr_periods 1515\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.765488, + "memory_current_bytes": 88973312, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13672831\nuser_usec 12111177\nsystem_usec 1561653\ncore_sched.force_idle_usec 0\nnr_periods 1516\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.8210475, + "memory_current_bytes": 88969216, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13672927\nuser_usec 12111262\nsystem_usec 1561664\ncore_sched.force_idle_usec 0\nnr_periods 1516\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.872457, + "memory_current_bytes": 88985600, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13675732\nuser_usec 12113978\nsystem_usec 1561753\ncore_sched.force_idle_usec 0\nnr_periods 1516\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.9228036, + "memory_current_bytes": 88981504, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13675776\nuser_usec 12114017\nsystem_usec 1561758\ncore_sched.force_idle_usec 0\nnr_periods 1516\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.9730399, + "memory_current_bytes": 88981504, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13676330\nuser_usec 12114508\nsystem_usec 1561821\ncore_sched.force_idle_usec 0\nnr_periods 1517\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.0233068, + "memory_current_bytes": 99074048, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13702096\nuser_usec 12134492\nsystem_usec 1567604\ncore_sched.force_idle_usec 0\nnr_periods 1517\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.0734708, + "memory_current_bytes": 105541632, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13709181\nuser_usec 12137873\nsystem_usec 1571308\ncore_sched.force_idle_usec 0\nnr_periods 1518\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.1237204, + "memory_current_bytes": 117547008, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13761280\nuser_usec 12174959\nsystem_usec 1586320\ncore_sched.force_idle_usec 0\nnr_periods 1518\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.1741512, + "memory_current_bytes": 118476800, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13768127\nuser_usec 12179575\nsystem_usec 1588552\ncore_sched.force_idle_usec 0\nnr_periods 1519\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.2263746, + "memory_current_bytes": 119562240, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13771134\nuser_usec 12180677\nsystem_usec 1590457\ncore_sched.force_idle_usec 0\nnr_periods 1519\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.2775128, + "memory_current_bytes": 119296000, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13771326\nuser_usec 12180846\nsystem_usec 1590479\ncore_sched.force_idle_usec 0\nnr_periods 1520\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.3278463, + "memory_current_bytes": 119156736, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13798007\nuser_usec 12197592\nsystem_usec 1600414\ncore_sched.force_idle_usec 0\nnr_periods 1520\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.3815193, + "memory_current_bytes": 121430016, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13816142\nuser_usec 12210439\nsystem_usec 1605703\ncore_sched.force_idle_usec 0\nnr_periods 1521\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.432009, + "memory_current_bytes": 124145664, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13827385\nuser_usec 12214224\nsystem_usec 1613161\ncore_sched.force_idle_usec 0\nnr_periods 1521\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.4828658, + "memory_current_bytes": 124403712, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13828180\nuser_usec 12214926\nsystem_usec 1613253\ncore_sched.force_idle_usec 0\nnr_periods 1522\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.5338924, + "memory_current_bytes": 121077760, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13849415\nuser_usec 12232113\nsystem_usec 1617302\ncore_sched.force_idle_usec 0\nnr_periods 1522\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.584187, + "memory_current_bytes": 120823808, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13849775\nuser_usec 12232430\nsystem_usec 1617344\ncore_sched.force_idle_usec 0\nnr_periods 1523\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.6344275, + "memory_current_bytes": 121438208, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13904495\nuser_usec 12278268\nsystem_usec 1626227\ncore_sched.force_idle_usec 0\nnr_periods 1523\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.6878014, + "memory_current_bytes": 123244544, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13930885\nuser_usec 12295177\nsystem_usec 1635708\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.738272, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13934193\nuser_usec 12297677\nsystem_usec 1636515\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.7884378, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13934355\nuser_usec 12297820\nsystem_usec 1636534\ncore_sched.force_idle_usec 0\nnr_periods 1525\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.8446183, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13934448\nuser_usec 12297902\nsystem_usec 1636545\ncore_sched.force_idle_usec 0\nnr_periods 1525\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.894701, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13937059\nuser_usec 12300445\nsystem_usec 1636613\ncore_sched.force_idle_usec 0\nnr_periods 1526\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.9457886, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13937145\nuser_usec 12300521\nsystem_usec 1636623\ncore_sched.force_idle_usec 0\nnr_periods 1526\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.9959867, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13937538\nuser_usec 12300868\nsystem_usec 1636670\ncore_sched.force_idle_usec 0\nnr_periods 1527\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.0461764, + "memory_current_bytes": 106586112, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13937669\nuser_usec 12300983\nsystem_usec 1636685\ncore_sched.force_idle_usec 0\nnr_periods 1527\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.0968993, + "memory_current_bytes": 106688512, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13939701\nuser_usec 12303015\nsystem_usec 1636685\ncore_sched.force_idle_usec 0\nnr_periods 1528\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.14862, + "memory_current_bytes": 109469696, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13959335\nuser_usec 12320696\nsystem_usec 1638638\ncore_sched.force_idle_usec 0\nnr_periods 1528\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.201882, + "memory_current_bytes": 125444096, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 13992680\nuser_usec 12343799\nsystem_usec 1648880\ncore_sched.force_idle_usec 0\nnr_periods 1529\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.252475, + "memory_current_bytes": 130682880, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14045280\nuser_usec 12379423\nsystem_usec 1665856\ncore_sched.force_idle_usec 0\nnr_periods 1529\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.3038034, + "memory_current_bytes": 121716736, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14127276\nuser_usec 12452429\nsystem_usec 1674846\ncore_sched.force_idle_usec 0\nnr_periods 1530\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.3569474, + "memory_current_bytes": 124092416, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14156129\nuser_usec 12472106\nsystem_usec 1684023\ncore_sched.force_idle_usec 0\nnr_periods 1530\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.4082305, + "memory_current_bytes": 126844928, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14166789\nuser_usec 12475489\nsystem_usec 1691299\ncore_sched.force_idle_usec 0\nnr_periods 1531\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.4623227, + "memory_current_bytes": 126844928, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14167097\nuser_usec 12475489\nsystem_usec 1691607\ncore_sched.force_idle_usec 0\nnr_periods 1532\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.512749, + "memory_current_bytes": 125374464, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14174223\nuser_usec 12480372\nsystem_usec 1693851\ncore_sched.force_idle_usec 0\nnr_periods 1532\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.5704546, + "memory_current_bytes": 124968960, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14174566\nuser_usec 12480674\nsystem_usec 1693892\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.620688, + "memory_current_bytes": 121716736, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14174774\nuser_usec 12480857\nsystem_usec 1693917\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.6796646, + "memory_current_bytes": 121716736, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14175016\nuser_usec 12481070\nsystem_usec 1693946\ncore_sched.force_idle_usec 0\nnr_periods 1534\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.7299514, + "memory_current_bytes": 119017472, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14193385\nuser_usec 12496261\nsystem_usec 1697123\ncore_sched.force_idle_usec 0\nnr_periods 1534\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.7803354, + "memory_current_bytes": 121470976, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14252840\nuser_usec 12541810\nsystem_usec 1711030\ncore_sched.force_idle_usec 0\nnr_periods 1535\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.8355799, + "memory_current_bytes": 122150912, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14262728\nuser_usec 12545768\nsystem_usec 1716960\ncore_sched.force_idle_usec 0\nnr_periods 1535\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.886906, + "memory_current_bytes": 121888768, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14262865\nuser_usec 12545889\nsystem_usec 1716976\ncore_sched.force_idle_usec 0\nnr_periods 1536\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.9370873, + "memory_current_bytes": 114028544, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14291624\nuser_usec 12568398\nsystem_usec 1723225\ncore_sched.force_idle_usec 0\nnr_periods 1536\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.9873314, + "memory_current_bytes": 114311168, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14311723\nuser_usec 12581422\nsystem_usec 1730300\ncore_sched.force_idle_usec 0\nnr_periods 1537\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.0398889, + "memory_current_bytes": 106754048, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14313309\nuser_usec 12582048\nsystem_usec 1731260\ncore_sched.force_idle_usec 0\nnr_periods 1537\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.0901713, + "memory_current_bytes": 106754048, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14313530\nuser_usec 12582243\nsystem_usec 1731287\ncore_sched.force_idle_usec 0\nnr_periods 1538\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.1407785, + "memory_current_bytes": 106754048, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14313773\nuser_usec 12582456\nsystem_usec 1731316\ncore_sched.force_idle_usec 0\nnr_periods 1538\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.1918032, + "memory_current_bytes": 106754048, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14313919\nuser_usec 12582584\nsystem_usec 1731334\ncore_sched.force_idle_usec 0\nnr_periods 1539\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.2420332, + "memory_current_bytes": 107167744, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14325140\nuser_usec 12586701\nsystem_usec 1738438\ncore_sched.force_idle_usec 0\nnr_periods 1539\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.2923708, + "memory_current_bytes": 106774528, + "lifetime_memory_peak_bytes": 149667840, + "cpu_stat": "usage_usec 14326275\nuser_usec 12586931\nsystem_usec 1739344\ncore_sched.force_idle_usec 0\nnr_periods 1540\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + } + ], + "sampled_peak_bytes": 149368832 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "samples": [ + { + "at_unix": 1793982274.5503392, + "memory_current_bytes": 81510400, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13020026\nuser_usec 11506448\nsystem_usec 1513578\ncore_sched.force_idle_usec 0\nnr_periods 1522\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.6006935, + "memory_current_bytes": 81715200, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13020201\nuser_usec 11506603\nsystem_usec 1513598\ncore_sched.force_idle_usec 0\nnr_periods 1523\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.6509082, + "memory_current_bytes": 81715200, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13020462\nuser_usec 11506833\nsystem_usec 1513628\ncore_sched.force_idle_usec 0\nnr_periods 1523\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.7015939, + "memory_current_bytes": 81715200, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13020733\nuser_usec 11507072\nsystem_usec 1513660\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.7520585, + "memory_current_bytes": 81715200, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13021053\nuser_usec 11507355\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.8027668, + "memory_current_bytes": 81977344, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13024350\nuser_usec 11510652\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.8535888, + "memory_current_bytes": 81723392, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13048074\nuser_usec 11534376\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1524\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.9037876, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13048278\nuser_usec 11534581\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1525\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982274.9546642, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13048570\nuser_usec 11534873\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1525\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.0060227, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13048687\nuser_usec 11534990\nsystem_usec 1513697\ncore_sched.force_idle_usec 0\nnr_periods 1526\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.0584555, + "memory_current_bytes": 98746368, + "lifetime_memory_peak_bytes": 141029376, + "cpu_stat": "usage_usec 13082008\nuser_usec 11550248\nsystem_usec 1531760\ncore_sched.force_idle_usec 0\nnr_periods 1526\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.1088154, + "memory_current_bytes": 149073920, + "lifetime_memory_peak_bytes": 149094400, + "cpu_stat": "usage_usec 13174598\nuser_usec 11607477\nsystem_usec 1567121\ncore_sched.force_idle_usec 0\nnr_periods 1527\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.159955, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 149221376, + "cpu_stat": "usage_usec 13230768\nuser_usec 11660568\nsystem_usec 1570200\ncore_sched.force_idle_usec 0\nnr_periods 1527\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.2105138, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 149221376, + "cpu_stat": "usage_usec 13230924\nuser_usec 11660705\nsystem_usec 1570218\ncore_sched.force_idle_usec 0\nnr_periods 1528\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.2608125, + "memory_current_bytes": 148860928, + "lifetime_memory_peak_bytes": 149221376, + "cpu_stat": "usage_usec 13286758\nuser_usec 11700279\nsystem_usec 1586478\ncore_sched.force_idle_usec 0\nnr_periods 1528\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.3112993, + "memory_current_bytes": 81723392, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13360363\nuser_usec 11769771\nsystem_usec 1590592\ncore_sched.force_idle_usec 0\nnr_periods 1529\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.3614736, + "memory_current_bytes": 81719296, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13360489\nuser_usec 11769882\nsystem_usec 1590607\ncore_sched.force_idle_usec 0\nnr_periods 1529\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.411723, + "memory_current_bytes": 148955136, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13445183\nuser_usec 11842702\nsystem_usec 1602481\ncore_sched.force_idle_usec 0\nnr_periods 1530\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.4623811, + "memory_current_bytes": 81723392, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13487036\nuser_usec 11882260\nsystem_usec 1604776\ncore_sched.force_idle_usec 0\nnr_periods 1530\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.5131812, + "memory_current_bytes": 81739776, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13489530\nuser_usec 11883924\nsystem_usec 1605606\ncore_sched.force_idle_usec 0\nnr_periods 1531\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.5669007, + "memory_current_bytes": 81735680, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13489753\nuser_usec 11884121\nsystem_usec 1605632\ncore_sched.force_idle_usec 0\nnr_periods 1531\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.6175294, + "memory_current_bytes": 81735680, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13489872\nuser_usec 11884225\nsystem_usec 1605646\ncore_sched.force_idle_usec 0\nnr_periods 1532\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.6704443, + "memory_current_bytes": 81735680, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13490011\nuser_usec 11884348\nsystem_usec 1605663\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.7263513, + "memory_current_bytes": 81739776, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13490118\nuser_usec 11884442\nsystem_usec 1605675\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.776508, + "memory_current_bytes": 81735680, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13490290\nuser_usec 11884614\nsystem_usec 1605675\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.8299758, + "memory_current_bytes": 81735680, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13490418\nuser_usec 11884742\nsystem_usec 1605675\ncore_sched.force_idle_usec 0\nnr_periods 1533\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.8804734, + "memory_current_bytes": 81485824, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13493317\nuser_usec 11887624\nsystem_usec 1605693\ncore_sched.force_idle_usec 0\nnr_periods 1534\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.9316483, + "memory_current_bytes": 88178688, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13531314\nuser_usec 11918337\nsystem_usec 1612977\ncore_sched.force_idle_usec 0\nnr_periods 1534\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982275.9832957, + "memory_current_bytes": 95748096, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13573601\nuser_usec 11955832\nsystem_usec 1617769\ncore_sched.force_idle_usec 0\nnr_periods 1535\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.0335743, + "memory_current_bytes": 88567808, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13615431\nuser_usec 11991302\nsystem_usec 1624129\ncore_sched.force_idle_usec 0\nnr_periods 1535\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.0844526, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13625117\nuser_usec 11999665\nsystem_usec 1625452\ncore_sched.force_idle_usec 0\nnr_periods 1536\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.134897, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13625333\nuser_usec 11999855\nsystem_usec 1625478\ncore_sched.force_idle_usec 0\nnr_periods 1536\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.1852372, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13625862\nuser_usec 12000321\nsystem_usec 1625541\ncore_sched.force_idle_usec 0\nnr_periods 1537\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.2375665, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13626032\nuser_usec 12000470\nsystem_usec 1625561\ncore_sched.force_idle_usec 0\nnr_periods 1537\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.2877967, + "memory_current_bytes": 85417984, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13626296\nuser_usec 12000703\nsystem_usec 1625593\ncore_sched.force_idle_usec 0\nnr_periods 1538\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.3474078, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13626427\nuser_usec 12000834\nsystem_usec 1625593\ncore_sched.force_idle_usec 0\nnr_periods 1538\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.3976345, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13626623\nuser_usec 12001029\nsystem_usec 1625593\ncore_sched.force_idle_usec 0\nnr_periods 1539\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.4577181, + "memory_current_bytes": 85413888, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13626840\nuser_usec 12001246\nsystem_usec 1625593\ncore_sched.force_idle_usec 0\nnr_periods 1539\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.5078952, + "memory_current_bytes": 89501696, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13656898\nuser_usec 12023931\nsystem_usec 1632967\ncore_sched.force_idle_usec 0\nnr_periods 1540\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.5583322, + "memory_current_bytes": 87949312, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13675301\nuser_usec 12036218\nsystem_usec 1639082\ncore_sched.force_idle_usec 0\nnr_periods 1540\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.6114542, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13682085\nuser_usec 12040764\nsystem_usec 1641321\ncore_sched.force_idle_usec 0\nnr_periods 1541\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.662144, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13682335\nuser_usec 12040764\nsystem_usec 1641571\ncore_sched.force_idle_usec 0\nnr_periods 1541\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.7150185, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13682495\nuser_usec 12040764\nsystem_usec 1641730\ncore_sched.force_idle_usec 0\nnr_periods 1542\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.7654905, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13682732\nuser_usec 12040764\nsystem_usec 1641968\ncore_sched.force_idle_usec 0\nnr_periods 1543\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.8158686, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13682822\nuser_usec 12040764\nsystem_usec 1642058\ncore_sched.force_idle_usec 0\nnr_periods 1543\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.8666396, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13685471\nuser_usec 12042970\nsystem_usec 1642500\ncore_sched.force_idle_usec 0\nnr_periods 1544\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.9168375, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13685584\nuser_usec 12043070\nsystem_usec 1642514\ncore_sched.force_idle_usec 0\nnr_periods 1544\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982276.967401, + "memory_current_bytes": 87801856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13686282\nuser_usec 12043070\nsystem_usec 1643212\ncore_sched.force_idle_usec 0\nnr_periods 1545\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.0176845, + "memory_current_bytes": 98807808, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13702052\nuser_usec 12055200\nsystem_usec 1646851\ncore_sched.force_idle_usec 0\nnr_periods 1545\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.0700657, + "memory_current_bytes": 108711936, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13745831\nuser_usec 12084728\nsystem_usec 1661103\ncore_sched.force_idle_usec 0\nnr_periods 1546\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.1211073, + "memory_current_bytes": 117370880, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13798728\nuser_usec 12123749\nsystem_usec 1674978\ncore_sched.force_idle_usec 0\nnr_periods 1546\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.1716177, + "memory_current_bytes": 123793408, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13839513\nuser_usec 12153331\nsystem_usec 1686181\ncore_sched.force_idle_usec 0\nnr_periods 1547\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.2219195, + "memory_current_bytes": 126832640, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13854371\nuser_usec 12166849\nsystem_usec 1687521\ncore_sched.force_idle_usec 0\nnr_periods 1547\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.2743106, + "memory_current_bytes": 126570496, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13854966\nuser_usec 12167372\nsystem_usec 1687594\ncore_sched.force_idle_usec 0\nnr_periods 1548\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.3244817, + "memory_current_bytes": 122634240, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13855481\nuser_usec 12167824\nsystem_usec 1687657\ncore_sched.force_idle_usec 0\nnr_periods 1548\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.3779294, + "memory_current_bytes": 122634240, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13855813\nuser_usec 12168156\nsystem_usec 1687657\ncore_sched.force_idle_usec 0\nnr_periods 1549\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.4306517, + "memory_current_bytes": 115068928, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13875422\nuser_usec 12183993\nsystem_usec 1691429\ncore_sched.force_idle_usec 0\nnr_periods 1549\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.4811747, + "memory_current_bytes": 130555904, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13920282\nuser_usec 12213577\nsystem_usec 1706705\ncore_sched.force_idle_usec 0\nnr_periods 1550\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.5317807, + "memory_current_bytes": 132390912, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13935750\nuser_usec 12222021\nsystem_usec 1713728\ncore_sched.force_idle_usec 0\nnr_periods 1550\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.5840383, + "memory_current_bytes": 132124672, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13936059\nuser_usec 12222292\nsystem_usec 1713766\ncore_sched.force_idle_usec 0\nnr_periods 1551\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.6344206, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13943255\nuser_usec 12227694\nsystem_usec 1715560\ncore_sched.force_idle_usec 0\nnr_periods 1551\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.6862373, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13943541\nuser_usec 12227945\nsystem_usec 1715595\ncore_sched.force_idle_usec 0\nnr_periods 1552\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.7364209, + "memory_current_bytes": 115965952, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13943683\nuser_usec 12228087\nsystem_usec 1715595\ncore_sched.force_idle_usec 0\nnr_periods 1552\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.7865915, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13943838\nuser_usec 12228242\nsystem_usec 1715595\ncore_sched.force_idle_usec 0\nnr_periods 1553\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.8387625, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13943914\nuser_usec 12228318\nsystem_usec 1715595\ncore_sched.force_idle_usec 0\nnr_periods 1553\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.888881, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13947697\nuser_usec 12232091\nsystem_usec 1715606\ncore_sched.force_idle_usec 0\nnr_periods 1554\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.9390526, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13947788\nuser_usec 12232171\nsystem_usec 1715617\ncore_sched.force_idle_usec 0\nnr_periods 1554\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982277.9892638, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13948189\nuser_usec 12232522\nsystem_usec 1715667\ncore_sched.force_idle_usec 0\nnr_periods 1555\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.0397575, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13948282\nuser_usec 12232603\nsystem_usec 1715678\ncore_sched.force_idle_usec 0\nnr_periods 1555\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.0910225, + "memory_current_bytes": 115961856, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13948741\nuser_usec 12233006\nsystem_usec 1715734\ncore_sched.force_idle_usec 0\nnr_periods 1556\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.1415489, + "memory_current_bytes": 127246336, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 13983686\nuser_usec 12259001\nsystem_usec 1724685\ncore_sched.force_idle_usec 0\nnr_periods 1556\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.1934617, + "memory_current_bytes": 138948608, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 14017894\nuser_usec 12287918\nsystem_usec 1729976\ncore_sched.force_idle_usec 0\nnr_periods 1557\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.2464526, + "memory_current_bytes": 148541440, + "lifetime_memory_peak_bytes": 149356544, + "cpu_stat": "usage_usec 14096400\nuser_usec 12353303\nsystem_usec 1743097\ncore_sched.force_idle_usec 0\nnr_periods 1557\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.2967186, + "memory_current_bytes": 153001984, + "lifetime_memory_peak_bytes": 153006080, + "cpu_stat": "usage_usec 14156564\nuser_usec 12397295\nsystem_usec 1759268\ncore_sched.force_idle_usec 0\nnr_periods 1558\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.355186, + "memory_current_bytes": 153182208, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14162119\nuser_usec 12400505\nsystem_usec 1761614\ncore_sched.force_idle_usec 0\nnr_periods 1558\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.4070945, + "memory_current_bytes": 152137728, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14167034\nuser_usec 12404296\nsystem_usec 1762737\ncore_sched.force_idle_usec 0\nnr_periods 1559\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.4623606, + "memory_current_bytes": 152133632, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14167598\nuser_usec 12404296\nsystem_usec 1763301\ncore_sched.force_idle_usec 0\nnr_periods 1559\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.5127358, + "memory_current_bytes": 145731584, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14175478\nuser_usec 12408632\nsystem_usec 1766845\ncore_sched.force_idle_usec 0\nnr_periods 1560\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.5683265, + "memory_current_bytes": 147660800, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14205557\nuser_usec 12431643\nsystem_usec 1773913\ncore_sched.force_idle_usec 0\nnr_periods 1561\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.618552, + "memory_current_bytes": 147386368, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14217626\nuser_usec 12440550\nsystem_usec 1777076\ncore_sched.force_idle_usec 0\nnr_periods 1561\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.675144, + "memory_current_bytes": 150343680, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14243272\nuser_usec 12461209\nsystem_usec 1782062\ncore_sched.force_idle_usec 0\nnr_periods 1562\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.725899, + "memory_current_bytes": 147533824, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14246087\nuser_usec 12462909\nsystem_usec 1783178\ncore_sched.force_idle_usec 0\nnr_periods 1562\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.7761903, + "memory_current_bytes": 142917632, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14283641\nuser_usec 12492589\nsystem_usec 1791052\ncore_sched.force_idle_usec 0\nnr_periods 1563\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.82954, + "memory_current_bytes": 140103680, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14284074\nuser_usec 12492967\nsystem_usec 1791106\ncore_sched.force_idle_usec 0\nnr_periods 1563\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.880703, + "memory_current_bytes": 140111872, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14287146\nuser_usec 12496037\nsystem_usec 1791108\ncore_sched.force_idle_usec 0\nnr_periods 1564\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.930887, + "memory_current_bytes": 132808704, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14288747\nuser_usec 12497565\nsystem_usec 1791181\ncore_sched.force_idle_usec 0\nnr_periods 1564\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982278.981167, + "memory_current_bytes": 132808704, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14288886\nuser_usec 12497687\nsystem_usec 1791199\ncore_sched.force_idle_usec 0\nnr_periods 1565\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.0324996, + "memory_current_bytes": 132845568, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14313389\nuser_usec 12521034\nsystem_usec 1792354\ncore_sched.force_idle_usec 0\nnr_periods 1565\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.082826, + "memory_current_bytes": 132837376, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14313559\nuser_usec 12521184\nsystem_usec 1792375\ncore_sched.force_idle_usec 0\nnr_periods 1566\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.1330888, + "memory_current_bytes": 132841472, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14313868\nuser_usec 12521453\nsystem_usec 1792414\ncore_sched.force_idle_usec 0\nnr_periods 1566\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.1835585, + "memory_current_bytes": 132837376, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14314044\nuser_usec 12521607\nsystem_usec 1792436\ncore_sched.force_idle_usec 0\nnr_periods 1567\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.2337937, + "memory_current_bytes": 132837376, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14314350\nuser_usec 12521876\nsystem_usec 1792474\ncore_sched.force_idle_usec 0\nnr_periods 1567\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.283998, + "memory_current_bytes": 133427200, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14315505\nuser_usec 12523013\nsystem_usec 1792491\ncore_sched.force_idle_usec 0\nnr_periods 1568\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + }, + { + "at_unix": 1793982279.3415012, + "memory_current_bytes": 132886528, + "lifetime_memory_peak_bytes": 155426816, + "cpu_stat": "usage_usec 14320841\nuser_usec 12526155\nsystem_usec 1794686\ncore_sched.force_idle_usec 0\nnr_periods 1568\nnr_throttled 0\nthrottled_usec 0\nnr_bursts 0\nburst_usec 0\n" + } + ], + "sampled_peak_bytes": 153182208 + } + ] +} diff --git a/evidence/upload-memory.png b/evidence/upload-memory.png new file mode 100644 index 00000000..d304ec97 Binary files /dev/null and b/evidence/upload-memory.png differ diff --git a/evidence/upload-memory.svg b/evidence/upload-memory.svg new file mode 100644 index 00000000..a75f20cf --- /dev/null +++ b/evidence/upload-memory.svg @@ -0,0 +1,1630 @@ + + + + + + + + 2026-11-06T19:43:24.898663 + image/svg+xml + + + Matplotlib v3.10.0, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 00000000..d9d66fe1 --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,273 @@ +{ + "status": "PASS", + "phases": [ + { + "width": 512, + "height": 512, + "size_bytes": 787271, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982275.5182576, + "finished_unix": 1793982276.575706, + "elapsed_seconds": 1.057448, + "successful_rps": 18.913, + "latency_ms": { + "p50": 102.767, + "p95": 176.198, + "p99": 183.16 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "individual_requests": [ + { + "elapsed_ms": 165.792, + "instance": "3586879492e3", + "id": "c2fdfeae-4fc7-4f18-aeea-15128a1ce778" + }, + { + "elapsed_ms": 115.745, + "instance": "3586879492e3", + "id": "dc2ffcd1-c0da-4d7d-bc54-fdf79d0e0c4b" + }, + { + "elapsed_ms": 152.416, + "instance": "2b11729802cd", + "id": "66a9427e-5c56-4616-b382-eecbf164f2a6" + }, + { + "elapsed_ms": 120.246, + "instance": "3586879492e3", + "id": "5fc86549-532d-4893-8adc-526480c8ca4a" + }, + { + "elapsed_ms": 183.16, + "instance": "3586879492e3", + "id": "f3103a13-e41d-4b7f-8101-be01e7d0b551" + }, + { + "elapsed_ms": 165.525, + "instance": "3586879492e3", + "id": "184a079f-1cf1-4486-a217-7d47f5159fc6" + }, + { + "elapsed_ms": 176.198, + "instance": "2b11729802cd", + "id": "123bb8c5-a739-460e-8048-0b74e021b1c4" + }, + { + "elapsed_ms": 98.997, + "instance": "2b11729802cd", + "id": "184c5689-8653-4598-9f22-66e833f44874" + }, + { + "elapsed_ms": 116.418, + "instance": "2b11729802cd", + "id": "84805db4-100b-4e73-b055-d5aeb1b08a20" + }, + { + "elapsed_ms": 152.617, + "instance": "2b11729802cd", + "id": "d7f12011-df5e-47ad-aff5-4048a834f21c" + }, + { + "elapsed_ms": 54.057, + "instance": "2b11729802cd", + "id": "175665e6-6045-420e-bcd4-b5666d5fcf7c" + }, + { + "elapsed_ms": 87.786, + "instance": "3586879492e3", + "id": "d1ef2b99-f69f-4622-a0ac-d4f45eb0da37" + }, + { + "elapsed_ms": 54.613, + "instance": "2b11729802cd", + "id": "4527cc17-d539-4e95-9ba7-cd0aec7b19ca" + }, + { + "elapsed_ms": 87.082, + "instance": "3586879492e3", + "id": "d3d16d58-61f0-4b3f-a205-6d696fa0cff6" + }, + { + "elapsed_ms": 102.767, + "instance": "3586879492e3", + "id": "690f4dbc-203a-4269-a0d6-f6089e1f80a7" + }, + { + "elapsed_ms": 53.332, + "instance": "3586879492e3", + "id": "c530cf03-e4f9-4476-b029-3210f1b79a07" + }, + { + "elapsed_ms": 85.73, + "instance": "2b11729802cd", + "id": "a401af64-d3eb-4c57-86b3-b0760aa8c140" + }, + { + "elapsed_ms": 54.686, + "instance": "3586879492e3", + "id": "294ed5d9-cbfd-4903-abfb-e576e497ef58" + }, + { + "elapsed_ms": 86.839, + "instance": "2b11729802cd", + "id": "9379c6d1-4be6-4984-80ed-3601463a6a7c" + }, + { + "elapsed_ms": 103.509, + "instance": "2b11729802cd", + "id": "bd49733e-f9cc-49b8-a2c5-7dfde60239d7" + } + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 91.094 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 91.312 + } + ] + }, + { + "width": 1536, + "height": 1536, + "size_bytes": 7081758, + "concurrency": 10, + "rounds": 2, + "successful_uploads": 20, + "errors": 0, + "started_unix": 1793982276.6054053, + "finished_unix": 1793982278.9907844, + "elapsed_seconds": 2.385379, + "successful_rps": 8.384, + "latency_ms": { + "p50": 617.87, + "p95": 832.535, + "p99": 896.355 + }, + "observed_instances": [ + "2b11729802cd", + "3586879492e3" + ], + "individual_requests": [ + { + "elapsed_ms": 716.744, + "instance": "2b11729802cd", + "id": "de25dd86-bbff-4c73-b43b-b1a32ae54b58" + }, + { + "elapsed_ms": 519.935, + "instance": "2b11729802cd", + "id": "5e31a195-4588-4217-a6b7-f60e8fc354d3" + }, + { + "elapsed_ms": 113.752, + "instance": "3586879492e3", + "id": "3c2d016c-49b7-4389-828c-3b13142ae128" + }, + { + "elapsed_ms": 701.82, + "instance": "2b11729802cd", + "id": "b279556b-3662-4649-a63d-c3299cae9790" + }, + { + "elapsed_ms": 162.629, + "instance": "3586879492e3", + "id": "6f35eee0-b44c-4d8e-b9d7-941e969351c0" + }, + { + "elapsed_ms": 617.87, + "instance": "3586879492e3", + "id": "017476ec-3d6c-4a7f-9b2f-02d07308a308" + }, + { + "elapsed_ms": 609.742, + "instance": "3586879492e3", + "id": "beaf2bf4-e19a-403e-a034-d2a839b0723e" + }, + { + "elapsed_ms": 609.077, + "instance": "2b11729802cd", + "id": "265535f4-faf6-4b2e-b5ea-2c8707dc7c44" + }, + { + "elapsed_ms": 439.875, + "instance": "3586879492e3", + "id": "fa001db4-f120-463d-8f42-7043a28331a1" + }, + { + "elapsed_ms": 313.432, + "instance": "2b11729802cd", + "id": "bcb008ae-7734-4d5c-8056-f1196f785fed" + }, + { + "elapsed_ms": 832.535, + "instance": "2b11729802cd", + "id": "b06dbcf2-7db6-4466-a7b4-0e63c36a1fdc" + }, + { + "elapsed_ms": 896.355, + "instance": "2b11729802cd", + "id": "85880399-b7c2-47a9-b3f5-5d6d184f5050" + }, + { + "elapsed_ms": 827.618, + "instance": "2b11729802cd", + "id": "5bdce531-4c9f-45a6-a165-8efb291a2abb" + }, + { + "elapsed_ms": 519.467, + "instance": "3586879492e3", + "id": "7e8eaf86-9280-4422-b7fe-ff8ab6113f92" + }, + { + "elapsed_ms": 648.035, + "instance": "3586879492e3", + "id": "a2fc1c26-501f-40c6-8aac-65879c7afd90" + }, + { + "elapsed_ms": 627.053, + "instance": "2b11729802cd", + "id": "4e4f37a1-9dd5-4406-b0d3-50d92b3e38ae" + }, + { + "elapsed_ms": 421.265, + "instance": "3586879492e3", + "id": "be19fd8f-a481-4355-9392-4910191643ae" + }, + { + "elapsed_ms": 622.325, + "instance": "2b11729802cd", + "id": "bbc868dd-3437-455a-ae4b-3d893012f4c9" + }, + { + "elapsed_ms": 827.438, + "instance": "3586879492e3", + "id": "65a55f03-8eae-483f-badd-da0c71fbe96d" + }, + { + "elapsed_ms": 642.219, + "instance": "3586879492e3", + "id": "27e56e81-322a-47c7-892d-52484b4caa3d" + } + ], + "api_memory": [ + { + "container_id": "2b11729802cd7e23d8d5e26cfa9cb67478160c297bbce439c6e547b9fa3a8295", + "sampled_peak_mib": 124.629 + }, + { + "container_id": "3586879492e3830cc482e782e50c24b3a31757d2f7d9274786b4e6731650764a", + "sampled_peak_mib": 146.086 + } + ] + } + ] +} diff --git a/evidence/uploads-negative.json b/evidence/uploads-negative.json new file mode 100644 index 00000000..e9ee6693 --- /dev/null +++ b/evidence/uploads-negative.json @@ -0,0 +1,90 @@ +{ + "status": "PASS", + "cases": [ + { + "case": "corrupt_png", + "status": 415 + }, + { + "case": "truncated_png", + "status": 415 + }, + { + "case": "mime_spoof", + "status": 415 + }, + { + "case": "unsupported_mime", + "status": 415 + }, + { + "case": "empty_payload", + "status": 415 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "unsafe_filename", + "status": 400 + }, + { + "case": "foreign_chat_upload", + "status": 404 + }, + { + "case": "unauthenticated_upload", + "status": 401 + }, + { + "case": "over_40_million_pixels", + "status": 413 + }, + { + "case": "oversized_content_length_before_body", + "status": 413 + }, + { + "case": "chunked_oversize", + "status": 413 + }, + { + "case": "foreign_attachment_metadata_and_bytes", + "status": 404 + }, + { + "case": "cross_chat_and_unknown_attachment", + "status": 404 + }, + { + "case": "attachment_idempotency_and_conflict", + "status": "201/409" + }, + { + "case": "truncated_jpeg", + "status": 415 + }, + { + "case": "interrupted_http_body", + "metadata_or_object_added": false + } + ], + "count": 21 +} diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/versions.txt b/evidence/versions.txt new file mode 100644 index 00000000..946807d7 --- /dev/null +++ b/evidence/versions.txt @@ -0,0 +1,14 @@ +{ + "Python": "3.11.11", + "fastapi": "0.115.0", + "uvicorn": "0.30.6", + "psycopg": "3.3.6", + "boto3": "1.42.0", + "botocore": "1.41.6", + "Pillow": "13.0.0", + "redis": "5.1.0" +} +minio version RELEASE.2025-09-07T16-13-09Z (commit-id=07c3a429bfed433e49018cb0f78a52145d4bedeb) +Runtime: go1.24.6 linux/arm64 +License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html +Copyright: 2015-2025 MinIO, Inc. diff --git a/infra/README.md b/infra/README.md index 5a39e862..eeadbb18 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,11 @@ -# Инфраструктура студента +# Стенд lab4 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +compose.yaml — отдельный проект mtusi-lab4. Публичен только HAProxy; API, PostgreSQL, Valkey и MinIO находятся в сети Compose. PostgreSQL/Valkey/MinIO имеют собственные именованные volumes. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +- haproxy.cfg: round robin, активные readiness-проверки, Docker DNS обновление адресов. +- init-s3.sh: повторяемый приватный bucket bootstrap, отдельный API principal. +- api-policy.json: ListBucket целевого bucket, Get/Put/Delete только originals/*; административных прав нет. +- Консоль MinIO не опубликована, привязана к loopback контейнера, browser UI отключён. +- Reconciler выполняет apply с grace=300s, после прохода ждёт 60s. ops — отдельный профиль для ручной сверки/тестов. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Пароли генерируются scripts/bootstrap.sh в игнорируемом .env. Root S3 credentials не передаются API. Реальные digest/версии закреплены в Compose; запуск и ограничения — ../REPORT.md, исследование — ../RESEARCH.md. diff --git a/infra/api-policy.json b/infra/api-policy.json new file mode 100644 index 00000000..42666139 --- /dev/null +++ b/infra/api-policy.json @@ -0,0 +1,7 @@ +{ + "Version": "2012-10-17", + "Statement": [ + {"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"]}, + {"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]} + ] +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..4bf2f273 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 + maxconn 512 + +defaults + log global + mode http + timeout connect 1s + timeout client 45s + timeout server 45s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server-template api 8 api:8080 check diff --git a/infra/init-s3.sh b/infra/init-s3.sh new file mode 100755 index 00000000..8ec734e6 --- /dev/null +++ b/infra/init-s3.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +ready=false +for attempt in $(seq 1 60); do + if mc alias set local http://s3:9000 "$S3_ROOT_USER" "$S3_ROOT_PASSWORD" >/dev/null 2>&1; then + ready=true + break + fi + sleep 1 +done +[ "$ready" = true ] || exit 1 +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin user add local "$S3_API_USER" "$S3_API_PASSWORD" >/dev/null +mc admin policy create local lab4-attachments /bootstrap/api-policy.json >/dev/null +mc admin policy attach local lab4-attachments --user "$S3_API_USER" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted application principal configured' diff --git a/migrate.py b/migrate.py new file mode 100644 index 00000000..14261db8 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +from app import store + +if __name__ == '__main__': + store.pool.open(wait=True, timeout=30) + try: + store.init() + finally: + store.pool.close() diff --git a/migrations/001_initial_schema.sql b/migrations/001_initial_schema.sql new file mode 100644 index 00000000..74371785 --- /dev/null +++ b/migrations/001_initial_schema.sql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY, + token TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages(chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_chat_members_user ON chat_members(user_id, chat_id); +CREATE INDEX IF NOT EXISTS idx_chats_created ON chats(created_at, id); +CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..4a2f9591 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + sender_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + key VARCHAR(128) NOT NULL, + body TEXT NOT NULL, + response JSONB NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY(sender_id, chat_id, key) +); +CREATE INDEX message_requests_expiry ON message_requests(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..0a0a865a --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,27 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + object_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL, + state TEXT NOT NULL CHECK(state IN ('pending','ready','failed')), + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX attachments_state_age ON attachments(state, updated_at); +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9), + PRIMARY KEY(message_id, attachment_id), + UNIQUE(message_id, position) +); +CREATE INDEX message_attachments_attachment ON message_attachments(attachment_id); +-- Preserve replay of lab3 text requests if an existing lab3 DB is upgraded. +ALTER TABLE message_requests ALTER COLUMN body TYPE JSONB + USING jsonb_build_object('text', body, 'attachment_ids', '[]'::jsonb); +UPDATE message_requests SET response = response || '{"attachment_ids":[]}'::jsonb; diff --git a/reconcile.py b/reconcile.py new file mode 100644 index 00000000..1908699b --- /dev/null +++ b/reconcile.py @@ -0,0 +1,162 @@ +"""Repeatable DB/S3 reconciler. Dry-run by default; no ready object is deleted.""" +import argparse +from datetime import datetime, timedelta, timezone +import json +import time +import uuid + +from botocore.exceptions import ClientError +from psycopg.rows import dict_row +from attachments import object_lock + + +def managed_key(key): + try: + return key == 'originals/' + str(uuid.UUID(key.removeprefix('originals/'))) + except ValueError: + return False + + +def reconcile(objects, *, apply=False, grace_seconds=300, allow_short_grace=False, prefix='originals/'): + if grace_seconds < 1: + raise ValueError("grace must be positive") + if grace_seconds < 60 and not allow_short_grace: + raise ValueError('production grace must be >=60s; tests must explicitly allow a shorter grace') + if prefix != 'originals/': + raise ValueError('only the managed originals/ namespace may be scanned') + store, s3 = objects.store, objects.client + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + started = time.monotonic() + result = {'mode': 'apply' if apply else 'dry-run', 'grace_seconds': grace_seconds, + 'objects_scanned': 0, 'metadata_scanned': 0, 's3_list_calls': 0, + 's3_head_calls': 0, 's3_delete_calls': 0, 'skipped_fresh': 0, + 'skipped_busy': 0, 'skipped_in_use': 0, 'action_counts': {}, 'actions_truncated': 0, 'actions': []} + + def head(key): + result['s3_head_calls'] += 1 + try: + return s3.head_object(Bucket=objects.bucket, Key=key) + except ClientError as exc: + if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'): + return None + raise # 403/5xx/timeouts are not proof of a missing object. + + def action(kind, key, conn, row=None): + result['action_counts'][kind] = result['action_counts'].get(kind, 0) + 1 + if len(result['actions']) < 1000: + result['actions'].append({'kind': kind, 'key': key}) + else: + result['actions_truncated'] += 1 + if not apply: + return + if kind in ('delete_orphan','abort_pending'): + if head(key) is not None: + s3.delete_object(Bucket=objects.bucket, Key=key) + result['s3_delete_calls'] += 1 + if row is not None: + conn.execute("UPDATE attachments SET state='failed',error_code='incomplete_upload',updated_at=NOW() WHERE id=%s", (row['id'],)) + elif kind in ('missing_object','corrupt_object'): + conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=NOW() WHERE id=%s", (kind,row['id'])) + elif kind == 'restore_ready': + conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (row['id'],)) + + # One run at a time; object-level transaction locks are shared with uploads. + with store.connect() as guard: + acquired = guard.execute('SELECT pg_try_advisory_lock(202604)').fetchone()[0] + if not acquired: + raise RuntimeError('another reconciler is running') + try: + paginator = s3.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=objects.bucket, Prefix=prefix, PaginationConfig={'PageSize':200}): + result['s3_list_calls'] += 1 + for item in page.get('Contents', []): + result['objects_scanned'] += 1 + key = item['Key'] + if not managed_key(key): + continue + if item['LastModified'] > cutoff: + result['skipped_fresh'] += 1 + continue + with store.connect() as conn: + if not object_lock(conn,key,wait=False): + result['skipped_busy'] += 1 + continue + # Re-read BOTH stores under lock instead of trusting an old list snapshot. + aid = key.split('/')[1] + row = objects.fetch(aid,conn) + current = head(key) + if current is None: + continue + if current['LastModified'] > cutoff: + result['skipped_fresh'] += 1 + continue + if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(aid,)).fetchone(): + result['skipped_in_use'] += 1 + continue + if row is None: + action('delete_orphan',key,conn) + elif row['state']=='pending' and row['updated_at'] <= cutoff: + action('abort_pending',key,conn,row) + # Includes pending rows whose S3 PUT never occurred; keyset pages keep + # scan RAM bounded. Failed missing/corrupt rows can recover if restored. + last = '00000000-0000-0000-0000-000000000000' + while True: + with store.connect() as conn, conn.cursor(row_factory=dict_row) as cur: + cur.execute("SELECT id FROM attachments WHERE id>%s AND (state IN ('ready','pending') OR error_code IN ('missing_object','corrupt_object')) ORDER BY id LIMIT 200", (last,)) + rows = cur.fetchall() + if not rows: + break + for item in rows: + last = str(item['id']) + result['metadata_scanned'] += 1 + with store.connect() as conn: + key = 'originals/' + last + if not object_lock(conn,key,wait=False): + result['skipped_busy'] += 1 + continue + row = objects.fetch(last,conn) + if row is None: + continue + if row['updated_at'] > cutoff: + result['skipped_fresh'] += 1 + continue + current = head(key) + if row['state']=='pending': + if conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1',(last,)).fetchone(): + result['skipped_in_use'] += 1 + continue + if current is not None and current['LastModified'] > cutoff: + result['skipped_fresh'] += 1 + continue + action('abort_pending',key,conn,row) + elif current is None: + if row['state']!='failed' or row['error_code']!='missing_object': + action('missing_object',key,conn,row) + elif current['ContentLength'] != row['size_bytes'] or current.get('Metadata',{}).get('sha256') != row['sha256']: + if row['state']!='failed' or row['error_code']!='corrupt_object': + action('corrupt_object',key,conn,row) + elif row['state']=='failed': + action('restore_ready',key,conn,row) + finally: + guard.execute('SELECT pg_advisory_unlock(202604)') + result['elapsed_seconds'] = round(time.monotonic()-started,6) + return result + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--grace-seconds',type=int,default=300) + parser.add_argument('--allow-short-grace',action='store_true',help='isolated synthetic tests only') + args=parser.parse_args() + from app import store, object_store + store.pool.open(wait=True,timeout=15) + try: + print(json.dumps(reconcile(object_store,apply=args.apply,grace_seconds=args.grace_seconds, + allow_short_grace=args.allow_short_grace),indent=2)) + finally: + store.pool.close() + object_store.close() + +if __name__=='__main__': + main() diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..0f518a73 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +# Standalone figures from sanitized evidence; API runtime does not need this. +matplotlib==3.10.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..77afa678 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.0 +uvicorn==0.30.6 +psycopg[binary]==3.3.6 +psycopg-pool==3.2.6 +argon2-cffi==23.1.0 +python-dotenv==1.0.1 +redis==5.1.0 +boto3==1.42.0 +Pillow==13.0.0 diff --git a/scripts/attachment_failures.py b/scripts/attachment_failures.py new file mode 100644 index 00000000..b73fe772 --- /dev/null +++ b/scripts/attachment_failures.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Actual dependency outages. Runs inside ops with an operator-authored hook.""" +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from client import Client,Failure,check + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--dependency',choices=['s3','db'],required=True) + args=parser.parse_args() + # Host orchestrates this protocol; credentials/tokens remain inside this process. + c=Client('http://lb:8080',timeout=15) + direct=Client('http://api:8080',timeout=10) + alice,bob,eve,chat=c.bootstrap() + att,data=c.attachment(alice,chat) + from app import object_store + before=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0) + print('READY',flush=True) + assert input().strip()=='STOPPED' + result={'dependency':args.dependency} + direct.call('GET','/health/ready',503,schema='Health') + result['readiness']=503 + if args.dependency=='s3': + direct.call('GET','/api/v1/attachments/'+att['id'],actor=bob,schema='Attachment') + direct.call('GET','/api/v1/attachments/'+att['id']+'/content',503,actor=bob) + direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'}) + result.update(metadata_available=True,download=503,upload=503) + else: + direct.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',503,actor=alice,raw=data,headers={'Content-Type':'image/png'}) + after=object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('KeyCount',0) + check(before==after,'DB failure created an untracked S3 object') + result.update(upload=503,no_new_s3_objects=True) + print('CHECKED',flush=True) + assert input().strip()=='RESTORED' + deadline=time.monotonic()+60 + while True: + try: + direct.call('GET','/health/ready',schema='Health') + break + except Failure: + check(time.monotonic() .env +import secrets +for name in ('POSTGRES_PASSWORD','S3_ROOT_PASSWORD','S3_API_PASSWORD'): + print(name+'='+secrets.token_hex(24)) +print('APP_PORT=8084') +PY +fi diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 00000000..c3b2080a --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if docker compose version >/dev/null 2>&1; then + exec docker compose "$@" +fi +exec docker-compose "$@" diff --git a/scripts/container_snapshot.py b/scripts/container_snapshot.py new file mode 100644 index 00000000..b2ba8499 --- /dev/null +++ b/scripts/container_snapshot.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Sanitized real-container evidence. Never dump environment or labels wholesale.""" +import json +from pathlib import Path +import subprocess + +ROOT=Path(__file__).resolve().parents[1] +ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-a','-q'],cwd=ROOT,text=True).split() +result=[] +for item in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)): + result.append({'name':item['Name'],'id':item['Id'],'hostname':item['Config']['Hostname'], + 'service':item['Config']['Labels']['com.docker.compose.service'], + 'image_id':item['Image'],'status':item['State']['Status'], + 'started_at':item['State']['StartedAt'], + 'health':item['State'].get('Health',{}).get('Status'), + 'nano_cpus':item['HostConfig']['NanoCpus'],'memory_bytes':item['HostConfig']['Memory'], + 'published_ports':item['HostConfig']['PortBindings'], + 'volumes':[{'name':m['Name'],'destination':m['Destination']} for m in item['Mounts'] if m['Type']=='volume']}) +print(json.dumps(sorted(result,key=lambda c:c['name']),indent=2)) diff --git a/scripts/fault_after_put.py b/scripts/fault_after_put.py new file mode 100644 index 00000000..f3b7f535 --- /dev/null +++ b/scripts/fault_after_put.py @@ -0,0 +1,20 @@ +"""Isolated CLI fault: real process death after PUT, before DB ready COMMIT.""" +import hashlib +import json +import os +from pathlib import Path +import sys +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from app import store, object_store + + +def crash(aid,key): + print(json.dumps({'id':aid,'key':key}),flush=True) + os._exit(86) + +if __name__=='__main__': + store.pool.open(wait=True,timeout=15) + path=Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png' + payload=path.read_bytes() + with path.open('rb') as stream: + object_store.persist(stream,sys.argv[1],sys.argv[2],'crash.png','image/png',len(payload),hashlib.sha256(payload).hexdigest(),after_put=crash) diff --git a/scripts/plot_research.py b/scripts/plot_research.py new file mode 100644 index 00000000..3ebd5fb7 --- /dev/null +++ b/scripts/plot_research.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Rebuild CSV summaries and standalone plots from sanitized research artifacts.""" +import csv +import json +import os +from pathlib import Path +import statistics + +os.environ.setdefault('MPLCONFIGDIR','/tmp/mtusi-lab4-matplotlib') +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT=Path(__file__).resolve().parents[1] +EVIDENCE=ROOT/'evidence' + + +def main(): + research=json.loads((EVIDENCE/'reconciliation.json').read_text()) + rows=[] + for n in sorted({r['extra_orphans'] for r in research['scan_cost_runs']}): + group=[r for r in research['scan_cost_runs'] if r['extra_orphans']==n] + times=[1000*r['elapsed_seconds'] for r in group] + rows.append({'extra_orphans':n,'objects_scanned':group[0]['objects_scanned'], + 'metadata_scanned':group[0]['metadata_scanned'],'repeats':len(group), + 'mean_ms':round(statistics.mean(times),3),'min_ms':round(min(times),3), + 'max_ms':round(max(times),3),'stdev_ms':round(statistics.stdev(times),3), + 'list_calls':group[0]['s3_list_calls'],'head_calls':group[0]['s3_head_calls']}) + with (EVIDENCE/'scan-cost.csv').open('w') as stream: + writer=csv.DictWriter(stream,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows) + (EVIDENCE/'scan-cost-summary.json').write_text(json.dumps(rows,indent=2)+'\n') + plt.rcParams.update({'font.size':10,'axes.grid':True,'grid.alpha':.25}) + fig,ax=plt.subplots(figsize=(7,4),layout='constrained') + ax.errorbar([r['objects_scanned'] for r in rows],[r['mean_ms'] for r in rows], + yerr=[r['stdev_ms'] for r in rows],marker='o',capsize=5) + ax.set(xlabel='S3 objects scanned',ylabel='Dry-run time, ms (mean ± sample SD)', + title='Reconciler scan cost: fixed metadata baseline, three repeats') + for suffix in ('png','svg'): + fig.savefig(EVIDENCE/('scan-cost.'+suffix),dpi=180) + plt.close(fig) + memory=json.loads((EVIDENCE/'upload-memory.json').read_text()) + phases=memory['workload']['phases'] + fig,axes=plt.subplots(1,2,figsize=(10,4),layout='constrained') + labels=[f"{r['size_bytes']/1048576:.2f} MiB" for r in phases] + for api in range(len(phases[0]['api_memory'])): + x=[i+(api-.5)*.3 for i in range(len(phases))] + axes[0].bar(x,[r['api_memory'][api]['sampled_peak_mib'] for r in phases],width=.3,label='API '+str(api+1)) + axes[0].axhline(512,ls='--',color='tab:red',label='Per-API memory limit') + axes[0].set(xticks=range(len(labels)),xticklabels=labels,ylabel='Sampled cgroup memory peak, MiB',title='10 concurrent uploads, two rounds') + axes[0].legend(fontsize=8) + axes[1].bar(labels,[r['latency_ms']['p95'] for r in phases]) + axes[1].set(ylabel='Successful upload p95, ms',title='20 successful uploads per size') + for suffix in ('png','svg'): + fig.savefig(EVIDENCE/('upload-memory.'+suffix),dpi=180) + plt.close(fig) + print(json.dumps({'scan_cost':rows,'upload_phases':[{k:v for k,v in p.items() if k!='individual_requests'} for p in phases]},indent=2)) + +if __name__=='__main__': + main() diff --git a/scripts/reconcile_loop.py b/scripts/reconcile_loop.py new file mode 100644 index 00000000..b07ee896 --- /dev/null +++ b/scripts/reconcile_loop.py @@ -0,0 +1,25 @@ +"""Automatic, bounded-memory sweeps; errors never imply that objects are absent.""" +import json +from pathlib import Path +import sys +import time +import signal +import threading +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from app import store, object_store +from reconcile import reconcile + +stop=threading.Event() +signal.signal(signal.SIGTERM,lambda *_: stop.set()) +signal.signal(signal.SIGINT,lambda *_: stop.set()) +store.pool.open(wait=True,timeout=30) +try: + while not stop.is_set(): + try: + print(json.dumps(reconcile(object_store,apply=True,grace_seconds=300)),flush=True) + except Exception as exc: + print(json.dumps({'status':'retry_next_sweep','error_type':type(exc).__name__}),flush=True) + stop.wait(60) +finally: + store.pool.close() + object_store.close() diff --git a/scripts/research_consistency.py b/scripts/research_consistency.py new file mode 100644 index 00000000..86472ff6 --- /dev/null +++ b/scripts/research_consistency.py @@ -0,0 +1,159 @@ +"""Track 2: repeatable faults, safe repair, protection and scan-cost measurements. + +Run with automatic reconciler stopped, inside the isolated lab4 ops container. +""" +import hashlib +import json +from pathlib import Path +import statistics +import subprocess +import sys +import time +import uuid +from unittest.mock import patch +import psycopg +from botocore.exceptions import ClientError +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'tests')) +from app import store,object_store +from attachments import object_lock +from reconcile import reconcile +from client import Client,check + + +def main(): + store.pool.open(wait=True,timeout=15) + s3=object_store.client + bucket=object_store.bucket + c=Client('http://lb:8080',timeout=20) + alice,bob,eve,chat=c.bootstrap() + data=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes() + sha=hashlib.sha256(data).hexdigest() + def key(): + return 'originals/'+str(uuid.uuid4()) + def put(k,body=data): + s3.put_object(Bucket=bucket,Key=k,Body=body,ContentType='image/png',Metadata={'sha256':hashlib.sha256(body).hexdigest()}) + def age(aid): + with store.connect() as conn: + conn.execute("UPDATE attachments SET created_at=NOW()-interval '10 minutes',updated_at=NOW()-interval '10 minutes' WHERE id=%s",(aid,)) + def states(): + with store.connect() as conn: + return conn.execute('SELECT id,state,error_code FROM attachments ORDER BY id').fetchall() + def present(k): + try: + s3.head_object(Bucket=bucket,Key=k) + return True + except ClientError as exc: + if exc.response['Error']['Code'] in ('404','NoSuchKey','NotFound'): + return False + raise + def sweep(apply=False,grace=5): + return reconcile(object_store,apply=apply,grace_seconds=grace,allow_short_grace=True) + missing,_=c.attachment(alice,chat) + c.message(alice,chat,'missing-object fixture',attachments=[missing['id']]) + used,_=c.attachment(alice,chat) + c.message(alice,chat,'preserve-used fixture',attachments=[used['id']]) + damaged,_=c.attachment(alice,chat) + c.message(alice,chat,'corrupt-object fixture',attachments=[damaged['id']]) + # Simulate stale DB restore/loss of an unreferenced metadata row. + orphan,_=c.attachment(alice,chat) + orphan_key='originals/'+orphan['id'] + with store.connect() as conn: + conn.execute('DELETE FROM attachments WHERE id=%s',(orphan['id'],)) + crash=subprocess.run([sys.executable,'scripts/fault_after_put.py',alice['id'],chat['id']],capture_output=True,text=True) + check(crash.returncode==86,'fault process did not terminate at the real PUT/COMMIT boundary') + crashed=json.loads(crash.stdout) + check(present(crashed['key']),'PUT did not precede crash') + empty_id=str(uuid.uuid4()) + # Another failure shape: intent was committed, but S3 PUT never succeeded. + with store.connect() as conn: + conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,created_at,updated_at) VALUES (%s,%s,%s,%s,'no-put.png','image/png',%s,%s,'pending',NOW()-interval '10 minutes',NOW()-interval '10 minutes')", + (empty_id,chat['id'],alice['id'],'originals/'+empty_id,len(data),sha)) + for aid in (missing['id'],used['id'],damaged['id'],crashed['id']): + age(aid) + s3.delete_object(Bucket=bucket,Key='originals/'+missing['id']) + put('originals/'+damaged['id'],b'corrupt bytes') + locked_key=key();put(locked_key) + time.sleep(6) # Real S3 timestamps age; no fake server clock or lifecycle override. + fresh,_=c.attachment(alice,chat) + fresh_orphan=key();put(fresh_orphan) + before=states() + with store.connect() as uploader: + object_lock(uploader,locked_key) + dry=sweep() + check(states()==before,'dry-run modified metadata') + check(present(orphan_key) and present(crashed['key']),'dry-run deleted objects') + required={('delete_orphan',orphan_key),('abort_pending',crashed['key']), + ('abort_pending','originals/'+empty_id),('missing_object','originals/'+missing['id']), + ('corrupt_object','originals/'+damaged['id'])} + observed={(a['kind'],a['key']) for a in dry['actions']} + check(required<=observed,'dry-run missed an injected failure') + check(dry['skipped_busy']>=1,'active upload lock ignored') + put(fresh_orphan) # Freshness is measured separately at each run's cutoff. + applied=sweep(True) + check(present(locked_key),'in-flight locked key deleted') + check(present(fresh_orphan),'fresh object deleted') + check(present('originals/'+fresh['id']),'fresh ready attachment deleted') + check(present('originals/'+used['id']),'used ready attachment deleted') + check(not present(orphan_key) and not present(crashed['key']),'old orphan/pending object not cleaned') + for aid,expected in ((crashed['id'],'incomplete_upload'),(empty_id,'incomplete_upload'),(missing['id'],'missing_object'),(damaged['id'],'corrupt_object')): + with store.connect() as conn: + row=object_store.fetch(aid,conn) + check(row['state']=='failed' and row['error_code']==expected,'incorrect repair state') + c.call('GET','/api/v1/attachments/'+missing['id']+'/content',409,actor=bob) + with store.connect() as conn: + check(conn.execute('SELECT 1 FROM message_attachments WHERE attachment_id=%s',(missing['id'],)).fetchone() is not None,'missing metadata/usage link deleted') + # Known test-only locked orphan is removed after proving lock protection. + s3.delete_object(Bucket=bucket,Key=locked_key) + put(fresh_orphan) + repeated=sweep(True) + check(not repeated['actions'],'second apply is not idempotent') + put('originals/'+missing['id']) + age(missing['id']) + recovered=sweep(True) + check(any(a['kind']=='restore_ready' and a['key']=='originals/'+missing['id'] for a in recovered['actions']),'restored object not made available') + check(c.call('GET','/api/v1/attachments/'+missing['id']+'/content',actor=bob)['bytes']==data,'restoration changed original') + # Errors are not evidence of absence: both failures must stop cleanup. + before=states() + failures=[] + with patch.object(s3,'head_object',side_effect=ClientError({'Error':{'Code':'ServiceUnavailable'},'ResponseMetadata':{'HTTPStatusCode':503}},'HeadObject')): + try: + sweep(True) + raise AssertionError('S3 503 was swallowed') + except ClientError: + failures.append('s3_503_aborts_without_missing_inference') + with patch.object(store,'connect',side_effect=psycopg.OperationalError('injected database outage')): + try: + sweep(True) + raise AssertionError('DB error was swallowed') + except psycopg.OperationalError: + failures.append('db_outage_aborts_before_s3_mutation') + check(states()==before,'dependency error changed attachment states') + s3.delete_object(Bucket=bucket,Key=fresh_orphan) + protection={'fresh_orphan_preserved':True,'fresh_ready_preserved':True,'used_ready_preserved':True, + 'active_lock_preserved':True,'missing_metadata_and_links_retained':True, + 'dry_run_no_mutation':True,'second_apply_actions':len(repeated['actions'])} + # Cost: independent orphan fixtures added to the same baseline, dry-run only. + costs=[] + cost_keys=[] + time.sleep(6) # Age the fixed baseline before timing the zero-extra-object group. + try: + for target in (0,50,200): + for _ in range(target-len(cost_keys)): + k=key();put(k);cost_keys.append(k) + if target: + time.sleep(6) + for repeat in range(3): + row=sweep() + costs.append({'extra_orphans':target,'repeat':repeat+1,**row}) + finally: + for k in cost_keys: + s3.delete_object(Bucket=bucket,Key=k) + print(json.dumps({'status':'PASS','grace_seconds_in_isolated_test':5,'production_grace_seconds':300, + 'real_process_crash_exit':crash.returncode,'dry_run':dry,'applied':applied, + 'repeated':repeated,'recovered':recovered,'protection':protection, + 'dependency_failure_checks':failures,'scan_cost_runs':costs},indent=2)) + store.pool.close();object_store.close() + +if __name__=='__main__': + main() diff --git a/scripts/research_consistency_host.py b/scripts/research_consistency_host.py new file mode 100644 index 00000000..7634bc42 --- /dev/null +++ b/scripts/research_consistency_host.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Pause the normal cron worker for deterministic faults, always restore it.""" +from pathlib import Path +import subprocess + +ROOT=Path(__file__).resolve().parents[1] +COMPOSE=str(ROOT/'scripts/compose.sh') +subprocess.run([COMPOSE,'stop','reconciler'],cwd=ROOT,check=True) +try: + output=subprocess.check_output([COMPOSE,'run','--rm','-T','--no-deps','ops','python','scripts/research_consistency.py'],cwd=ROOT,text=True) + (ROOT/'evidence/reconciliation.json').write_text(output) + print(output) +finally: + subprocess.run([COMPOSE,'start','reconciler'],cwd=ROOT,check=True) diff --git a/scripts/research_upload_memory.py b/scripts/research_upload_memory.py new file mode 100644 index 00000000..16a6511f --- /dev/null +++ b/scripts/research_upload_memory.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Sample API cgroups at 50ms while ten clients upload through LB.""" +import json +from pathlib import Path +import subprocess +import uuid + +ROOT=Path(__file__).resolve().parents[1] +COMPOSE=str(ROOT/'scripts/compose.sh') +MONITOR=r''' +import json,time,sys +from pathlib import Path +stop=Path(sys.argv[1]); group=Path('/sys/fs/cgroup') +rows=[] +print('READY',flush=True) +for i in range(2400): + rows.append({'at_unix':time.time(),'memory_current_bytes':int((group/'memory.current').read_text()), + 'lifetime_memory_peak_bytes':int((group/'memory.peak').read_text()), + 'cpu_stat':(group/'cpu.stat').read_text()}) + if stop.exists(): + stop.unlink() + break + time.sleep(.05) +print(json.dumps(rows),flush=True) +''' + + +def main(): + ids=subprocess.check_output([COMPOSE,'ps','-q','api'],cwd=ROOT,text=True).split() + assert len(ids)>=2 + monitors=[] + try: + for cid in ids: + stop='/tmp/lab4-memory-probe-'+uuid.uuid4().hex + proc=subprocess.Popen(['docker','exec',cid,'python','-u','-c',MONITOR,stop],text=True,stdout=subprocess.PIPE) + assert proc.stdout.readline().strip()=='READY' + monitors.append((cid,stop,proc)) + output=subprocess.check_output([COMPOSE,'run','--rm','-T','--no-deps','ops','python','tests/concurrent_uploads.py'],cwd=ROOT,text=True,timeout=100) + workload=json.loads(output) + finally: + for cid,stop,proc in monitors: + subprocess.run(['docker','exec',cid,'touch',stop],check=True,stdout=subprocess.DEVNULL) + traces=[] + for cid,stop,proc in monitors: + data=json.loads(proc.stdout.read()) + assert proc.wait(timeout=10)==0 + traces.append({'container_id':cid,'samples':data,'sampled_peak_bytes':max(s['memory_current_bytes'] for s in data)}) + for phase in workload['phases']: + phase['api_memory']=[{'container_id':t['container_id'], + 'sampled_peak_mib':round(max(s['memory_current_bytes'] for s in t['samples'] if phase['started_unix']<=s['at_unix']<=phase['finished_unix'])/1048576,3)} for t in traces] + result={'status':'PASS','sample_interval_seconds':.05, + 'note':'memory.current includes page cache, kernel charges and a small sampler process; sampled peak is not a heap profile. lifetime_memory_peak includes prior requests.', + 'workload':workload,'api_traces':traces} + target=ROOT/'evidence/upload-memory.json' + target.write_text(json.dumps(result,indent=2)+'\n') + print(json.dumps({'status':'PASS','phases':workload['phases']},indent=2)) + +if __name__=='__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..3c3e372f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +scripts/compose.sh restart db valkey s3 api lb reconciler diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..30647955 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?Required API hostname from X-Instance-Id}" +for id in $(scripts/compose.sh ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 3 "$id" >/dev/null + exit 0 + fi +done +printf '%s\n' 'Target is not a running API replica' >&2 +exit 1 diff --git a/scripts/summarize_research.py b/scripts/summarize_research.py new file mode 100644 index 00000000..8c7cf3c2 --- /dev/null +++ b/scripts/summarize_research.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Generate the research report from the captured artifacts, without inventing data.""" +import json +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] +E=ROOT/'evidence' + + +def update_report(research_text): + report = ROOT / 'REPORT.md' + begin = '' + end = '' + results = research_text.split('## Обнаружение и исправление\n', 1)[1].split('## Воспроизведение\n', 1)[0] + results = '### Обнаружение и исправление\n' + results.replace('\n## ', '\n### ') + block = (begin + '\n## Таблицы и графики экспериментов\n\n' + 'Числа и рисунки ниже получены из сохранённых JSON. ' + 'Полная методика и исходные условия приведены в [RESEARCH.md](RESEARCH.md).\n\n' + + results.rstrip() + '\n\n' + end + '\n\n') + content = report.read_text() + if begin in content: + left, remaining = content.split(begin, 1) + _, right = remaining.split(end, 1) + content = left + block + right.lstrip('\n') + else: + content = content.replace('## Ограничения\n', block + '## Ограничения\n', 1) + content = content.replace( + 'Результаты, таблицы стоимости и графики уровня 5 — в [RESEARCH.md](RESEARCH.md).', + 'Все таблицы результатов и оба графика включены ниже в этот отчёт; ' + 'полная методика уровня 5 — в [RESEARCH.md](RESEARCH.md).') + report.write_text(content) + + +def main(): + r=json.loads((E/'reconciliation.json').read_text()) + costs=json.loads((E/'scan-cost-summary.json').read_text()) + memory=json.loads((E/'upload-memory.json').read_text()) + text='''# Исследование lab4: согласованность PostgreSQL и S3 + +Автор: Савин Александр, БВТ2303. Уровень 5, трек 2. Выполнено 6 ноября 2026 года на отдельном стенде lab4. Исходные данные находятся в evidence/, обязательные API-тесты не менялись. + +## Гипотеза + +Если устойчивое намерение создаётся до S3 PUT, а uploader и reconciler используют одну область транзакционной блокировки, можно обнаруживать незавершённые операции и расхождения без удаления свежих, используемых или ещё записываемых объектов. Повторное применение к уже исправленному состоянию не должно создавать новых изменений. + +Дополнительная гипотеза: стоимость полной сверки растёт приблизительно с числом объектов и метаданных, а не с размером каждого оригинала, поскольку сверка использует List/HEAD и не скачивает все байты. + +## Среда и методика + +macOS/Colima, Linux arm64, Docker Engine 27.4.0, VM 4 vCPU/около 8 GB RAM. Версии и caps всех компонентов закреплены в compose.yaml, SDK — boto3 1.42.0. Исследовательский ops-контейнер: 1 CPU/512 MiB; обычный reconciler: 0.25 CPU/256 MiB, поэтому их абсолютное время не обязано совпадать. MinIO: 0.5 CPU/768 MiB, PostgreSQL: 1 CPU/512 MiB. + +scripts/research_consistency_host.py временно останавливает штатный worker, запускает один изолированный эксперимент и восстанавливает worker в finally. Пользователи, чаты, файлы и fault fixtures синтетические. Прямые повреждения выполняются только исследовательским инструментом с доступом к двум хранилищам; HTTP-клиент не может включить failpoint. + +Фазы опыта: + +1. Загрузить valid PNG и связать часть вложений с сообщениями. +2. Удалить метаданные одного неиспользуемого вложения, оставив S3-объект: orphan. +3. Запустить отдельный Python-процесс, который делает настоящий PUT и завершает себя os._exit(86) до ready COMMIT. Проверить pending и существующий объект. +4. Создать pending без объекта, моделируя неуспешный PUT. +5. Удалить объект используемого вложения; заменить другой объект неверным содержимым. +6. Состарить DB timestamps, дождаться реального старения S3 timestamps. Использовать grace=5s только с явным разрешением для теста; штатный grace остаётся 300s. +7. Создать свежие ready/orphan fixtures и удерживать общий advisory lock для старого объекта, моделируя активный uploader. +8. Выполнить dry-run и сравнить состояния/наличие: мутаций быть не должно. Затем apply с повторной проверкой обеих сторон под lock. +9. Повторить apply; восстановить потерянный оригинал с его SHA metadata и вернуть ready без изменения UUID/связей. +10. Имитировать S3 503 и ошибку соединения PostgreSQL: проход должен завершиться ошибкой без вывода «объект отсутствует» и без изменения статусов. + +В тесте перед отдельными sweep обновляется timestamp test-only fresh orphan, чтобы на начало каждого прохода он действительно был моложе grace. Заблокированный тестовый orphan удаляется оператором только после проверки его защиты. Это отличает проверку текущей свежести от обещания хранить неподтверждённый orphan бесконечно. + +## Обнаружение и исправление + +''' + text+='| Действие | Количество в apply |\n| --- | --- |\n' + from collections import Counter + counts=Counter(a['kind'] for a in r['applied']['actions']) + for kind,n in sorted(counts.items()): + text+=f'| {kind} | {n} |\n' + text+=f"\nРеальный fault-процесс завершился с кодом **{r['real_process_crash_exit']}** после PUT. Последующий apply нашёл его pending и удалил объект после grace. Для исчезнувшего/повреждённого используемого вложения сохраняются метаданные и связи сообщений; меняется доступность, а не история. После восстановления исходного объекта действие restore_ready снова разрешило скачивание с тем же UUID.\n\n" + text+='| Проверка безопасности | Результат |\n| --- | --- |\n' + for name,value in r['protection'].items(): + text+=f'| {name} | {value} |\n' + text+='\nПовторный apply не предложил новых действий. Это относится к уже исправленному тестовому состоянию; при появлении новых объектов/ошибок следующий проход закономерно может выполнить новые действия. Ошибки зависимостей в отдельной проверке не изменили статусы. Реальные остановки MinIO/PostgreSQL дополнительно проверены в evidence/dependency-outages.json.\n\n' + text+='''## Стоимость обхода + +К существующему фиксированному набору метаданных добавлялись 0, 50, 200 S3-объектов без метаданных. Это отдельные известные test-only ключи. Перед нулевой группой базовые timestamps выдерживаются за пределами тестового grace; после создания каждой следующей группы также выдерживается grace. Для каждого размера выполнено три dry-run, затем известные дополнительные ключи удалены. Порядок размеров возрастающий, кэш не сбрасывался. + +Время измеряется time.monotonic внутри инструмента; запуск контейнера, подготовка fixtures и ожидание grace не входят в elapsed. Исходные отдельные результаты: reconciliation.json → scan_cost_runs. В таблице среднее/выборочное стандартное отклонение, не доверительный интервал. + +''' + text+='| Дополнительных orphan | Объектов S3 | Метаданных | Время, мс: mean ± SD | min–max, мс | List / HEAD вызовов |\n| --- | --- | --- | --- | --- | --- |\n' + for row in costs: + text+=f"| {row['extra_orphans']} | {row['objects_scanned']} | {row['metadata_scanned']} | {row['mean_ms']} ± {row['stdev_ms']} | {row['min_ms']}–{row['max_ms']} | {row['list_calls']} / {row['head_calls']} |\n" + text+=''' +![Стоимость обхода](evidence/scan-cost.png) + +При росте набора увеличиваются число point-checks и время. Список S3 и DB-метаданные читаются страницами по 200, подробный журнал ограничен 1000 действиями; остальные учитываются счётчиками. Алгоритм делает O(N+M) обход при N объектах и M подходящих метаданных, с индексированными DB point lookups. Полный SHA-аудит bytes сюда не входит: HEAD сравнивает размер и записанный uploader-ом sha256 metadata. Результат не доказывает предельную производительность хранилища или точную линейность на миллионах объектов. + +## Эксперимент уровня 4: память при параллельной загрузке + +Десять клиентов одновременно стартуют по barrier, два раунда для каждого из двух размеров. Генератор находится внутри Compose-сети и идёт через LB. Каждая загрузка проверяет 201/ready, размер и SHA-256; один оригинал каждой фазы скачивается вторым участником и сравнивается побайтно. + +Изображения — реальные синтетические RGB PNG, однотонные, сохранены без сжатия (compress_level=0), чтобы получить воспроизводимый большой файл. Это профиль передачи/памяти, а не worst-case для времени сложного JPEG/PNG-декодирования. Payload создаётся один раз на фазу и разделяется клиентскими потоками. + +В каждом API cgroup memory.current читается раз в 50ms. Измерение включает page cache, kernel charges и небольшой Python sampler; это не RSS/heap-профиль. Пики — максимумы samples внутри фазы, не математическая гарантия отсутствия краткого пропущенного пика. В raw traces отдельно есть lifetime memory.peak, который включает более ранние операции. Лимит каждой API — 512 MiB, активных upload slots=2, decoder=1. + +''' + text+='| Файл, байт | Размер, MiB | Успешных загрузок | Ошибок | p50 / p95 / p99, мс | Пики API 1 / API 2, MiB |\n| --- | --- | --- | --- | --- | --- |\n' + for phase in memory['workload']['phases']: + lat=phase['latency_ms']; peaks=' / '.join(str(v['sampled_peak_mib']) for v in phase['api_memory']) + text+=f"| {phase['size_bytes']} | {phase['size_bytes']/1048576:.3f} | {phase['successful_uploads']} | {phase['errors']} | {lat['p50']} / {lat['p95']} / {lat['p99']} | {peaks} |\n" + text+=''' +![Параллельные загрузки и память](evidence/upload-memory.png) + +Все 40 запросов прошли, в каждой фазе наблюдались две настоящие API. Измеренные пики ниже лимита. Это подтверждает выбранный профиль ≥10 конкурентных загрузок, но не доказывает пик для всех изображений вплоть до 40 млн пикселей или для одновременных массовых логинов Argon2id. + +## Воспроизведение + +```sh +./scripts/bootstrap.sh +./scripts/compose.sh up -d --build +python3 scripts/wait_ready.py --base-url http://localhost:8084 +export BASE_URL=http://localhost:8084 +make research-consistency +make research-memory +make test-dependencies +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-research.txt +.venv/bin/python scripts/plot_research.py +python3 scripts/summarize_research.py +``` + +Методика добавляет синтетические данные; не удаляет volumes. Известные дополнительные cost/orphan fixtures очищаются точечно. Готовые вложения и сообщения остаются, поскольку обязательный API не включает delete. Повтор может иметь другой размер baseline, поэтому реальные N/M сохраняются в каждой строке. + +## Вывод и границы + +Гипотеза подтверждена в заявленной модели: durable intent, grace и общие object locks позволяют корректно обработать проверенные разрывы между двумя хранилищами; dry-run наблюдаем, apply повторяем, свежие/используемые/активные объекты защищены. Штатный worker запускает ту же логику с grace=300s. + +Восстановление утраченных bytes требует внешней копии; reconciler не заменяет backup. При потере всей БД, произвольных действиях root или нарушении инварианта неизменяемых ключей нужны отдельные процедуры восстановления. Метаданные являются авторитетом для классификации orphan. Локальные три коротких повтора на небольшом наборе и тёплом кэше не дают универсального SLA; стоимость штатного worker с меньшей CPU-квотой может отличаться от ops. Нет заявления о высокой доступности S3/БД, полной побайтовой проверке всех объектов или производственном zero-downtime. +''' + (ROOT/'RESEARCH.md').write_text(text) + update_report(text) + print('Generated RESEARCH.md and synchronized tables/figures in REPORT.md') + +if __name__=='__main__': + main() diff --git a/scripts/test_dependency_outages.py b/scripts/test_dependency_outages.py new file mode 100644 index 00000000..5dd917f0 --- /dev/null +++ b/scripts/test_dependency_outages.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Stop ONLY lab4 dependencies; restore each one in finally.""" +import json +from pathlib import Path +import subprocess + +ROOT=Path(__file__).resolve().parents[1] +COMPOSE=str(ROOT/'scripts/compose.sh') + + +def main(): + rows=[] + for dependency in ('s3','db'): + proc=subprocess.Popen([COMPOSE,'run','--rm','-T','--no-deps','ops','python','scripts/attachment_failures.py','--dependency',dependency],cwd=ROOT,stdin=subprocess.PIPE,stdout=subprocess.PIPE,text=True) + try: + assert proc.stdout.readline().strip()=='READY','bootstrap failed' + subprocess.run([COMPOSE,'stop',dependency],cwd=ROOT,check=True,stdout=subprocess.DEVNULL) + proc.stdin.write('STOPPED\n');proc.stdin.flush() + assert proc.stdout.readline().strip()=='CHECKED','outage check failed' + finally: + subprocess.run([COMPOSE,'start',dependency],cwd=ROOT,check=True,stdout=subprocess.DEVNULL) + proc.stdin.write('RESTORED\n');proc.stdin.flush() + report=json.loads(proc.stdout.readline()) + assert proc.wait(timeout=90)==0 + rows.append(report) + (ROOT/'evidence/dependency-outages.json').write_text(json.dumps(rows,indent=2)+'\n') + print(json.dumps(rows,indent=2)) + +if __name__=='__main__': + main() diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..ea118b58 --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Wait for actual healthy Compose API containers and their public LB aliases.""" +import argparse +import json +from pathlib import Path +import subprocess +import time +import urllib.request + +ROOT=Path(__file__).resolve().parents[1] +parser=argparse.ArgumentParser() +parser.add_argument('--base-url',default='http://localhost:8084') +parser.add_argument('--replicas',type=int,default=2) +args=parser.parse_args() +deadline=time.monotonic()+90 +while time.monotonic()=2,'need two real processes') + proc=subprocess.Popen([str(ROOT/'scripts/stop-one.sh')],env=dict(os.environ,TARGET_INSTANCE_ID=uploader),stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + started=time.monotonic();errors=0;recovered=None;served=None + c.timeout=2 + key=uuid.uuid4().hex + while time.monotonic()-started<10: + try: + result=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(result['bytes']==data,'original bytes changed') + served=result['headers']['X-Instance-Id'] + c.message(alice,chat,'write during file failover',key=key) + if proc.poll()==0 and served!=uploader: + recovered=time.monotonic()-started + break + except Failure: + errors+=1 + time.sleep(.1) + check(proc.wait(timeout=5)==0,'uploader hook failed') + check(recovered is not None and recovered<=10,'download/write did not recover within 10 seconds') + check(confirmed in history(c,bob,chat),'confirmed attachment message lost') + snapshot=json.loads(subprocess.check_output([sys.executable,str(ROOT/'scripts/container_snapshot.py')],text=True)) + check(any(row['hostname']==uploader and row['status']=='exited' for row in snapshot),'uploader container was not actually stopped') + report={'status':'PASS','uploader_instance':uploader,'download_instance':served, + 'observed_instances_before':sorted(observed),'failed_attempts':errors, + 'first_download_write_recovery_seconds':round(recovered,3), + 'sha256':hashlib.sha256(data).hexdigest(),'same_bytes':True,'containers_after_stop':snapshot} + (ROOT/'evidence/attachment-failover.json').write_text(json.dumps(report,indent=2)+'\n') + print(json.dumps({k:v for k,v in report.items() if k!='containers_after_stop'},indent=2)) + +if __name__=='__main__': + run(main) diff --git a/tests/cache_consistency.py b/tests/cache_consistency.py new file mode 100644 index 00000000..2179b553 --- /dev/null +++ b/tests/cache_consistency.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Research track: cache consistency and stale session protection for Valkey-backed sessions.""" +import base64 +import json +import os +import statistics +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") +CACHER_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0") + + +def http_json(method, path, *, expected=200, headers=None, body=None): + req_headers = {"Accept": "application/json"} + if headers: + req_headers.update(headers) + data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8") + if data is not None: + req_headers.setdefault("Content-Type", "application/json") + request = urllib.request.Request(BASE_URL + path, data=data, headers=req_headers, method=method) + try: + with urllib.request.urlopen(request, timeout=10) as resp: + payload = resp.read() + if not payload: + return {"status": resp.status, "headers": dict(resp.headers), "body": None} + if resp.status not in (expected if isinstance(expected, (list, tuple, set)) else [expected]): + raise AssertionError(f"Expected {expected}, got {resp.status}") + return {"status": resp.status, "headers": dict(resp.headers), "body": json.loads(payload.decode("utf-8"))} + except urllib.error.HTTPError as exc: + payload = exc.read() + if payload: + try: + payload = json.loads(payload.decode("utf-8")) + except Exception: + payload = payload.decode("utf-8", errors="replace") + if exc.code not in (expected if isinstance(expected, (list, tuple, set)) else [expected]): + raise AssertionError(f"Expected status {expected}, got {exc.code} for {method} {path}: {payload}") + return {"status": exc.code, "headers": dict(exc.headers), "body": payload} + + +def call_with_latency(method, path, *, expected=200, headers=None, body=None): + start = time.perf_counter() + result = http_json(method, path, expected=expected, headers=headers, body=body) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return result, elapsed_ms + + +def main(): + import redis + + r = redis.Redis.from_url(CACHER_URL, decode_responses=True, socket_connect_timeout=2, socket_timeout=2) + + username = f"cache_{uuid.uuid4().hex[:12]}" + password = "123456789012" + body = { + "username": username, + "display_name": "Cache Consistency", + "password": password, + } + registration = http_json("POST", "/api/v1/users", expected=201, body=body) + assert registration["status"] == 201, registration + + basic = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + session = http_json("POST", "/api/v1/auth/sessions", expected=201, headers={"Authorization": basic}) + token = session["body"]["token"] + assert token, session + + key = f"session:{token}" + assert r.get(key), "session was not populated in Valkey" + ttl = r.ttl(key) + assert ttl and ttl > 0, f"session TTL must be positive, got {ttl}" + + latencies = [] + for _ in range(10): + _, elapsed_ms = call_with_latency("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + latencies.append(elapsed_ms) + avg = sum(latencies) / len(latencies) + p95 = sorted(latencies)[int(len(latencies) * 0.95) - 1] + + stale_payload = { + "token": token, + "user_id": str(uuid.uuid4()), + "expires_at": "2000-01-01T00:00:00Z", + "revoked_at": None, + "user": {"id": str(uuid.uuid4()), "username": "stale", "display_name": "stale", "created_at": "2020-01-01T00:00:00Z"}, + } + r.setex(key, 120, json.dumps(stale_payload, separators=(",", ":"), ensure_ascii=False)) + stale_result = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + assert stale_result["status"] == 200, stale_result + assert stale_result["body"]["username"] == username, stale_result + + # An unexpired payload must not be able to substitute another identity. + stale_payload["expires_at"] = "2100-01-01T00:00:00Z" + r.setex(key, 120, json.dumps(stale_payload)) + forged = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert forged["body"]["id"] == session["body"]["user"]["id"], forged + assert forged["body"]["username"] == username, forged + + r.setex(key, 120, json.dumps(stale_payload)) + def read_identity(_): + return http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})["body"]["id"] + with ThreadPoolExecutor(max_workers=8) as executor: + identities = list(executor.map(read_identity, range(8))) + assert identities == [session["body"]["user"]["id"]] * 8, identities + + # Syntactically valid JSON with missing fields must be treated as a miss. + r.setex(key, 120, "{}") + malformed = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert malformed["body"]["username"] == username, malformed + + r.setex(key, 1, json.dumps({ + "token": token, + "user_id": str(uuid.uuid4()), + "expires_at": "2100-01-01T00:00:00Z", + "revoked_at": None, + "user": {"id": str(uuid.uuid4()), "username": "ttl-recovery", "display_name": "ttl-recovery", "created_at": "2020-01-01T00:00:00Z"}, + }, separators=(",", ":"), ensure_ascii=False)) + time.sleep(2) + short_ttl = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + assert short_ttl["status"] == 200, short_ttl + assert short_ttl["body"]["username"] == username, short_ttl + + revoke = http_json("DELETE", "/api/v1/auth/sessions/current", expected=204, headers={"Authorization": "Bearer " + token}) + assert revoke["status"] == 204, revoke + assert not r.get(key), "revoked session key should be deleted from cache" + + after_logout = http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token}) + assert after_logout["status"] == 401, after_logout + + # Simulate a writer repopulating the cache after logout's DELETE. + stale_payload.update(session["body"]) + stale_payload["user_id"] = session["body"]["user"]["id"] + stale_payload["revoked_at"] = None + r.setex(key, 120, json.dumps(stale_payload)) + http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token}) + assert not r.exists(key), "stale key after logout must be removed" + + print("cache_key_exists=True") + print(f"ttl_seconds={ttl}") + print(f"avg_latency_ms={avg:.2f}") + print(f"p95_latency_ms={p95:.2f}") + print("stale_cache_rejected=True") + print("forged_identity_rejected=True") + print("concurrent_identity_checks=8/8") + print("malformed_cache_recovers=True") + print("repopulated_revoked_cache_rejected=True") + print("short_ttl_recovers_from_db=True") + print("logout_revokes_cache=True") + + +if __name__ == "__main__": + main() diff --git a/tests/concurrent_uploads.py b/tests/concurrent_uploads.py new file mode 100644 index 00000000..8a01bc65 --- /dev/null +++ b/tests/concurrent_uploads.py @@ -0,0 +1,53 @@ +"""Two payload sizes, ten simultaneous uploads, two rounds per size.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import io +import json +from pathlib import Path +import sys +import threading +import time +from PIL import Image +sys.path.insert(0,str(Path(__file__).resolve().parent)) +from client import Client,check,run + + +def main(): + c=Client('http://lb:8080',timeout=30) + alice,bob,eve,chat=c.bootstrap() + phases=[] + for width in (512,1536): + stream=io.BytesIO() + Image.new('RGB',(width,width),'navy').save(stream,'PNG',compress_level=0) + payload=stream.getvalue() + checksum=hashlib.sha256(payload).hexdigest() + samples=[] + started=time.time() + for repeat in range(2): + barrier=threading.Barrier(10) + def worker(index): + local=Client(c.base,timeout=45) + barrier.wait() + t=time.monotonic() + result=local.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=concurrent.png',201, + actor=alice,raw=payload,headers={'Content-Type':'image/png'},schema='Attachment') + att=result['json'] + check(att['sha256']==checksum and att['size_bytes']==len(payload),'upload metadata mismatch') + return {'elapsed_ms':round((time.monotonic()-t)*1000,3),'instance':result['headers']['X-Instance-Id'],'id':att['id']} + with ThreadPoolExecutor(max_workers=10) as pool: + samples.extend(pool.map(worker,range(10))) + finished=time.time() + instances=sorted({r['instance'] for r in samples}) + check(len(instances)>=2,'did not observe two upload processes') + downloaded=c.call('GET','/api/v1/attachments/'+samples[-1]['id']+'/content',actor=bob) + check(downloaded['bytes']==payload,'concurrent upload changed original') + latency=sorted(r['elapsed_ms'] for r in samples) + phases.append({'width':width,'height':width,'size_bytes':len(payload),'concurrency':10,'rounds':2, + 'successful_uploads':len(samples),'errors':0,'started_unix':started,'finished_unix':finished, + 'elapsed_seconds':round(finished-started,6),'successful_rps':round(len(samples)/(finished-started),3), + 'latency_ms':{'p50':latency[9],'p95':latency[18],'p99':latency[19]}, + 'observed_instances':instances,'individual_requests':samples}) + print(json.dumps({'status':'PASS','phases':phases},indent=2)) + +if __name__=='__main__': + run(main) diff --git a/tests/idempotency_concurrent.py b/tests/idempotency_concurrent.py new file mode 100644 index 00000000..7a65c2c8 --- /dev/null +++ b/tests/idempotency_concurrent.py @@ -0,0 +1,37 @@ +"""Concurrent shared-key requests through LB, real instance headers and history.""" +from concurrent.futures import ThreadPoolExecutor +import json +import threading +import uuid +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + key = uuid.uuid4().hex + barrier = threading.Barrier(16) + + def worker(_): + local = Client(args.base_url) + barrier.wait() + return local.message(alice, chat, ' concurrent body ', key=key) + + with ThreadPoolExecutor(max_workers=16) as pool: + responses = list(pool.map(worker, range(16))) + ids = {r['json']['id'] for r in responses} + instances = {r['headers']['X-Instance-Id'] for r in responses} + check(len(ids) == 1, 'concurrent duplicate messages') + check(len(instances) >= 2, 'concurrent requests did not reach two replicas') + path = '/api/v1/chats/' + chat['id'] + '/messages' + c.call('POST', path, 409, actor=alice, body={'text': 'concurrent body'}, headers={'Idempotency-Key': key}) + again = c.call('POST', path, 201, actor=alice, body={'attachment_ids': [], 'text': ' concurrent body '}, headers={'Idempotency-Key': key}, schema='Message') + check(again['json'] == responses[0]['json'], 'semantic replay differs') + page = c.call('GET', path, actor=bob, schema='MessagePage')['json'] + check(len(page['items']) == 1, 'history contains duplicate effects') + c.message(bob, chat, 'other user', key=key) + print(json.dumps({'parallel_requests': 16, 'observed_instances': sorted(instances), 'unique_messages': len(ids), 'conflict': 409, 'semantic_replay': True, 'user_scope': True})) + +if __name__ == '__main__': + run(main) diff --git a/tests/private_storage.py b/tests/private_storage.py new file mode 100644 index 00000000..136fc25d --- /dev/null +++ b/tests/private_storage.py @@ -0,0 +1,52 @@ +"""Anonymous access outside API and least-privilege negative checks.""" +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request +from botocore.exceptions import ClientError +sys.path.insert(0,str(Path(__file__).resolve().parent)) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from client import Client,check,run + + +def main(): + from app import store,object_store + c=Client('http://lb:8080',timeout=15) + alice,bob,eve,chat=c.bootstrap() + att,_=c.attachment(alice,chat) + key='originals/'+att['id'] + anonymous=[] + for suffix in ('','/'+key): + try: + urllib.request.urlopen('http://s3:9000/'+object_store.bucket+suffix,timeout=3) + raise AssertionError('private bucket allowed anonymous access') + except urllib.error.HTTPError as exc: + check(exc.code==403,'anonymous access failed for an unexpected reason') + anonymous.append(exc.code) + # MinIO returns a filtered ListBuckets response to a ListBucket principal. + visible=[b['Name'] for b in object_store.client.list_buckets().get('Buckets',[])] + check(set(visible)<={object_store.bucket},'unrelated bucket names exposed') + denied=[] + for name,operation in [('create_bucket',lambda:object_store.client.create_bucket(Bucket='must-not-create-lab4')), + ('get_bucket_policy',lambda:object_store.client.get_bucket_policy(Bucket=object_store.bucket)), + ('write_outside_prefix',lambda:object_store.client.put_object(Bucket=object_store.bucket,Key='outside-policy',Body=b'no'))]: + try: + operation() + raise AssertionError('application credentials have excessive privileges: '+name) + except ClientError as exc: + check(exc.response['Error']['Code']=='AccessDenied','unexpected permission error') + denied.append(name) + # The console is bound to 127.0.0.1 inside S3 and browser UI is disabled. + import socket + try: + socket.create_connection(('s3',9001),timeout=1).close() + raise AssertionError('S3 console reachable from another container') + except OSError: + pass + print(json.dumps({'status':'PASS','anonymous_bucket_and_object_status':anonymous, + 'application_admin_operations_denied':denied,'visible_bucket_names':visible,'console_reachable':False},indent=2)) + object_store.close() + +if __name__=='__main__': + run(main) diff --git a/tests/uploads_negative.py b/tests/uploads_negative.py new file mode 100644 index 00000000..e3e765e0 --- /dev/null +++ b/tests/uploads_negative.py @@ -0,0 +1,127 @@ +"""Contract boundaries and object authorization; runs in Compose ops network.""" +import hashlib +import http.client +import io +import json +from pathlib import Path +import socket +import struct +import sys +import time +from urllib.parse import quote, urlsplit +import uuid +import zlib +from PIL import Image +sys.path.insert(0,str(Path(__file__).resolve().parent)) +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from client import Client, check, run + + +def main(): + c=Client('http://lb:8080',timeout=15) + alice,bob,eve,chat=c.bootstrap() + path='/api/v1/chats/'+chat['id']+'/attachments' + fixture=(Path(__file__).parent/'fixtures/sample.png').read_bytes() + cases=[] + def bad(label,status,data=fixture,mime='image/png',filename='test.png',actor=None): + c.call('POST',path+'?filename='+quote(filename,safe=''),status,actor=actor or alice,raw=data,headers={'Content-Type':mime}) + cases.append({'case':label,'status':status}) + bad('corrupt_png',415,b'not an image') + bad('truncated_png',415,fixture[:50]) + bad('mime_spoof',415,mime='image/jpeg') + bad('unsupported_mime',415,mime='application/octet-stream') + bad('empty_payload',415,b'') + for filename in ('../secret.png','a\\b.png','x\x00.png','x\n.png','x\u202e.png','x'*256): + bad('unsafe_filename',400,filename=filename) + bad('foreign_chat_upload',404,actor=eve) + c.call('POST',path+'?filename=x.png',401,raw=fixture,headers={'Content-Type':'image/png'}) + cases.append({'case':'unauthenticated_upload','status':401}) + # Oversized dimensions are rejected using headers before a full raster decode. + data=bytearray(fixture) + data[16:24]=struct.pack('>II',10000,4001) + data[29:33]=struct.pack('>I',zlib.crc32(data[12:29])&0xffffffff) + bad('over_40_million_pixels',413,bytes(data)) + # No large allocation or body transfer: reject declared size before reading. + parsed=urlsplit(c.base) + conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=10) + conn.putrequest('POST',path+'?filename=big.png') + for k,v in {'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png','Content-Length':str(10485761)}.items(): + conn.putheader(k,v) + conn.endheaders() + response=conn.getresponse() + check(response.status==413,'oversized content-length accepted') + response.read();conn.close() + cases.append({'case':'oversized_content_length_before_body','status':413}) + # True streamed overflow, no Content-Length; response is read after the last + # chunk, even if the peer stopped accepting bytes once it knew the limit. + conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=15) + conn.putrequest('POST',path+'?filename=chunked.png') + conn.putheader('Authorization','Bearer '+alice['token']) + conn.putheader('Content-Type','image/png') + conn.putheader('Transfer-Encoding','chunked') + conn.endheaders() + try: + block=b'x'*65536 + for _ in range(161): + conn.send(b'10000\r\n'+block+b'\r\n') + conn.send(b'0\r\n\r\n') + except (BrokenPipeError,ConnectionResetError): + pass + response=conn.getresponse() + check(response.status==413,'streamed byte limit not enforced') + response.read();conn.close() + cases.append({'case':'chunked_oversize','status':413}) + att,content=c.attachment(alice,chat) + # Another member can reference an attachment; ownership is not required. + msg=c.message(bob,chat,'',attachments=[att['id']])['json'] + check(msg['attachment_ids']==[att['id']],'member could not link attachment') + for suffix in ('','/content'): + c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve) + c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',404,actor=alice) + cases.append({'case':'foreign_attachment_metadata_and_bytes','status':404}) + other=c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'other','member_ids':[alice['id']]},schema='Chat')['json'] + other_path='/api/v1/chats/'+other['id']+'/messages' + for aid in (att['id'],str(uuid.uuid4())): + c.call('POST',other_path,404,actor=alice,body={'text':'','attachment_ids':[aid]},headers={'Idempotency-Key':uuid.uuid4().hex}) + cases.append({'case':'cross_chat_and_unknown_attachment','status':404}) + message_path='/api/v1/chats/'+chat['id']+'/messages' + for ids in ([att['id'],att['id']],['bad-id'],[str(uuid.uuid4()) for _ in range(11)]): + c.call('POST',message_path,400,actor=alice,body={'text':'','attachment_ids':ids},headers={'Idempotency-Key':uuid.uuid4().hex}) + key=uuid.uuid4().hex + first=c.message(alice,chat,'',key=key,attachments=[att['id']])['json'] + second=c.message(alice,chat,'',key=key,attachments=[att['id']])['json'] + check(first==second,'attachment message replay changed') + c.call('POST',message_path,409,actor=alice,body={'text':'different','attachment_ids':[att['id']]},headers={'Idempotency-Key':key}) + cases.append({'case':'attachment_idempotency_and_conflict','status':'201/409'}) + # Full JPEG validation, including truncated entropy-coded body. + stream=io.BytesIO() + Image.new('RGB',(320,200),'blue').save(stream,'JPEG') + jpeg=stream.getvalue() + uploaded=c.call('POST',path+'?filename=photo.jpg',201,actor=alice,raw=jpeg,headers={'Content-Type':'image/jpeg'},schema='Attachment')['json'] + downloaded=c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',actor=bob) + check(downloaded['bytes']==jpeg,'JPEG original changed') + bad('truncated_jpeg',415,jpeg[:-20],mime='image/jpeg') + # A disconnect before receiving all bytes creates neither intent nor object. + from app import store,object_store + store.pool.open(wait=True,timeout=15) + try: + def counts(): + with store.connect() as db: + n=db.execute('SELECT count(*) FROM attachments').fetchone()[0] + return n,len(object_store.client.list_objects_v2(Bucket=object_store.bucket,Prefix='originals/').get('Contents',[])) + before=counts() + conn=http.client.HTTPConnection(parsed.hostname,parsed.port,timeout=5) + conn.putrequest('POST',path+'?filename=interrupted.png') + conn.putheader('Authorization','Bearer '+alice['token']) + conn.putheader('Content-Type','image/png') + conn.putheader('Content-Length',str(len(fixture))) + conn.endheaders();conn.send(fixture[:50]);conn.close() + time.sleep(1) + check(counts()==before,'aborted upload persisted partial data') + cases.append({'case':'interrupted_http_body','metadata_or_object_added':False}) + finally: + store.pool.close();object_store.close() + print(json.dumps({'status':'PASS','cases':cases,'count':len(cases)},ensure_ascii=False,indent=2)) + +if __name__=='__main__': + run(main)