From dbc700fafe3dd824b2ff9fc6ac6f03e194eb44b4 Mon Sep 17 00:00:00 2001 From: Khakimov Dmitrij Date: Wed, 28 Oct 2026 22:41:10 +0300 Subject: [PATCH] lab4 done --- .github/pull_request_template.md | 37 +- Dockerfile | 3 + Dockerfile.storage | 34 + Makefile | 21 +- PORTING.md | 117 +- REPORT.md | 262 +++- cmd/main.go | 11 +- cmd/reconcile/main.go | 57 + compose.yaml | 62 +- contracts/reconciliation.md | 39 + dto/create_message.go | 16 +- dto/create_message_json.go | 33 + dto/create_message_json_test.go | 21 + dto/list_messages.go | 11 +- evidence/lab4-acceptance.json | 104 ++ evidence/lab4-acceptance.txt | 2 + evidence/lab4-bucket-bootstrap.txt | 8 + evidence/lab4-build.txt | 112 ++ evidence/lab4-deploy.txt | 313 +++++ evidence/lab4-environment.json | 81 ++ evidence/lab4-failover.txt | 4 + evidence/lab4-go-tests.txt | 13 + evidence/lab4-integration.txt | 338 +++++ evidence/lab4-persistence.txt | 3 + evidence/lab4-reconciliation.json | 1226 +++++++++++++++++ evidence/lab4-reconciliation.txt | 7 + evidence/lab4-restart-verification.json | 54 + evidence/lab4-scaling.txt | 6 + evidence/lab4-smoke.txt | 2 + evidence/lab4-status-after-restart.json | 146 ++ evidence/lab4-status-before-restart.json | 146 ++ evidence/lab4-status-final.json | 146 ++ evidence/lab4-status-initial.json | 146 ++ evidence/lab4-update.txt | 3 + evidence/lab4-upload-memory.json | 105 ++ evidence/lab4-upload-memory.txt | 4 + evidence/lab4-validation.txt | 8 + evidence/swarm-grade4.json | 72 + go.mod | 19 +- go.sum | 39 +- infra/README.md | 17 +- infra/haproxy.cfg | 4 +- infra/minio-entrypoint.sh | 5 + infra/s3-bootstrap.sh | 11 + infra/s3-policy.json | 21 + internal/app/chat/attachments.go | 68 + internal/app/chat/create_message.go | 11 +- internal/app/chat/errors.go | 4 + internal/app/chat/list_messages.go | 2 +- internal/app/chat/readiness.go | 5 + internal/app/chat/service.go | 15 +- internal/attachment/image.go | 128 ++ internal/attachment/image_test.go | 139 ++ internal/attachment/json.go | 18 + internal/attachment/json_test.go | 37 + internal/attachment/reconcile.go | 222 +++ internal/attachment/s3.go | 44 + internal/attachment/service.go | 132 ++ internal/config/config.go | 19 + internal/pkg/domain/message.go | 1 + internal/pkg/repository/postgres/lab4_test.go | 341 +++++ internal/pkg/repository/postgres/messages.go | 79 +- .../pkg/repository/postgres/storage_test.go | 7 + internal/usecase/chat/service.go | 28 +- migrations/003_attachments.sql | 20 + migrations/004_message_attachments.sql | 13 + scripts/bootstrap.sh | 9 +- scripts/measure_uploads.py | 85 ++ scripts/swarm.py | 55 +- stack.yaml | 47 +- tests/own/README.md | 13 +- tests/own/fixtures/sample.jpg | Bin 0 -> 4140 bytes tests/own/helpers.py | 116 ++ tests/own/lab4.py | 154 +++ tests/own/reconcile.py | 137 ++ 75 files changed, 5611 insertions(+), 227 deletions(-) create mode 100644 Dockerfile.storage create mode 100644 cmd/reconcile/main.go create mode 100644 contracts/reconciliation.md create mode 100644 dto/create_message_json.go create mode 100644 dto/create_message_json_test.go create mode 100644 evidence/lab4-acceptance.json create mode 100644 evidence/lab4-acceptance.txt create mode 100644 evidence/lab4-bucket-bootstrap.txt create mode 100644 evidence/lab4-build.txt create mode 100644 evidence/lab4-deploy.txt create mode 100644 evidence/lab4-environment.json create mode 100644 evidence/lab4-failover.txt create mode 100644 evidence/lab4-go-tests.txt create mode 100644 evidence/lab4-integration.txt create mode 100644 evidence/lab4-persistence.txt create mode 100644 evidence/lab4-reconciliation.json create mode 100644 evidence/lab4-reconciliation.txt create mode 100644 evidence/lab4-restart-verification.json create mode 100644 evidence/lab4-scaling.txt create mode 100644 evidence/lab4-smoke.txt create mode 100644 evidence/lab4-status-after-restart.json create mode 100644 evidence/lab4-status-before-restart.json create mode 100644 evidence/lab4-status-final.json create mode 100644 evidence/lab4-status-initial.json create mode 100644 evidence/lab4-update.txt create mode 100644 evidence/lab4-upload-memory.json create mode 100644 evidence/lab4-upload-memory.txt create mode 100644 evidence/lab4-validation.txt create mode 100644 evidence/swarm-grade4.json create mode 100755 infra/minio-entrypoint.sh create mode 100755 infra/s3-bootstrap.sh create mode 100644 infra/s3-policy.json create mode 100644 internal/app/chat/attachments.go create mode 100644 internal/attachment/image.go create mode 100644 internal/attachment/image_test.go create mode 100644 internal/attachment/json.go create mode 100644 internal/attachment/json_test.go create mode 100644 internal/attachment/reconcile.go create mode 100644 internal/attachment/s3.go create mode 100644 internal/attachment/service.go create mode 100644 internal/pkg/repository/postgres/lab4_test.go create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_message_attachments.sql create mode 100644 scripts/measure_uploads.py create mode 100644 tests/own/fixtures/sample.jpg create mode 100644 tests/own/helpers.py create mode 100644 tests/own/lab4.py create mode 100644 tests/own/reconcile.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec547..cb6ccb2 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,27 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: заполняет студент. +Целевая оценка: **5**. +Трек 5: **2 — согласованность PostgreSQL и S3**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Оригиналы PNG/JPEG хранятся в приватном MinIO, метаданные и ссылки сообщений — в PostgreSQL. +Авторизованное скачивание сохраняет байты после остановки загрузившей API-реплики и полного перезапуска стенда. +Из lab3 сохранены пользователи, чаты, сессии Redis, пагинация, идемпотентность и две API-реплики через HAProxy. +Reconciler проверяет orphan/missing, поддерживает dry-run и повторяемое исправление с grace period и блокировками. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `make up`, затем `make test`. +Проверки: `make check validate test-own`, `make test-integration`, `make test-lab4`, `make test-reconcile`, +`make measure-uploads`, `make test-persistence`, `make test-failover`, `make test-grade4`. +Вывод сохранён в `evidence/lab4-*`. Подробности и измерения: [REPORT.md](../REPORT.md). + +Ограничения: один Docker-host, нерезервированные PostgreSQL/Redis/S3/LB; сверка запускается оператором. +Десять загрузок проверены на двух размерах; предельная пара больших PNG требует более объёмной Docker VM. +Источники: контракт курса, MinIO/minio-go, PostgreSQL advisory locks, Docker Swarm. +Реализация и проверки подготовлены с помощью Codex. Коммит и PR не создавались по указанию пользователя. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile index 24d8a7f..7cd6ea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ -trimpath \ -ldflags="-s -w" \ -o /out/messenger ./cmd +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile FROM builder AS tests COPY --from=goose-builder /out/goose /usr/local/bin/goose @@ -43,6 +44,8 @@ RUN addgroup -S messenger \ COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger +COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ENV PORT=8080 EXPOSE 8080 diff --git a/Dockerfile.storage b/Dockerfile.storage new file mode 100644 index 0000000..8b7f7d0 --- /dev/null +++ b/Dockerfile.storage @@ -0,0 +1,34 @@ +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder +ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . +RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + +FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder +ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz +RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . +RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio +LABEL org.opencontainers.image.source="https://github.com/minio/minio" \ + org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" \ + org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +RUN mkdir /data && chown 10001:10001 /data +COPY --from=minio-builder /out/minio /usr/local/bin/minio +COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio +USER 10001:10001 +EXPOSE 9000 +ENTRYPOINT ["/usr/local/bin/start-minio"] + +FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS mc +LABEL org.opencontainers.image.source="https://github.com/minio/mc" \ + org.opencontainers.image.revision="d6541ea280b73a834b64d4097e21f2be77676104" \ + org.opencontainers.image.version="RELEASE.2025-08-13T08-35-41Z" \ + org.opencontainers.image.licenses="AGPL-3.0" +COPY --from=mc-builder /out/mc /usr/local/bin/mc +USER 10001:10001 +ENTRYPOINT ["mc"] diff --git a/Makefile b/Makefile index 074d47f..8c7ec35 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,11 @@ IMAGE ?= mtusi-messenger:lab4 CA_FILE ?= ACTION_SCRIPT ?= scripts/restart.sh FAILOVER_SCRIPT ?= scripts/stop-one.sh +RECONCILE_ARGS ?= export PYTHONDONTWRITEBYTECODE=1 TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help run docker-build docker-run check validate test test-own test-all load test-persistence bootstrap up down migrate test-integration test-grade4 measure-load migrate-status rolling-update test-failover wait +.PHONY: help run docker-build docker-run check validate test test-own test-all load test-persistence bootstrap up down migrate test-integration test-grade4 test-lab4 test-reconcile measure-uploads measure-load migrate-status rolling-update test-failover wait bootstrap-s3 reconcile reconcile-apply export PORT BASE_URL @@ -67,6 +68,24 @@ test-integration: test-grade4: $(PYTHON) tests/own/swarm_grade4.py +test-lab4: + $(PYTHON) tests/own/lab4.py + +test-reconcile: + $(PYTHON) tests/own/reconcile.py + +measure-uploads: + $(PYTHON) scripts/measure_uploads.py + +bootstrap-s3: + $(PYTHON) scripts/swarm.py bootstrap-s3 + +reconcile: + $(PYTHON) scripts/swarm.py reconcile $(RECONCILE_ARGS) + +reconcile-apply: + $(PYTHON) scripts/swarm.py reconcile --apply $(RECONCILE_ARGS) + measure-load: $(PYTHON) scripts/measure_load.py --base-url "$(BASE_URL)" diff --git a/PORTING.md b/PORTING.md index 6e9092b..f6cb10e 100644 --- a/PORTING.md +++ b/PORTING.md @@ -1,110 +1,15 @@ # Перенос реализации lab3 -В репозиторий перенесена логика из `mtusi2026lab3`. Исходный проект сохранён без изменений. +Проект содержит Go API, миграции PostgreSQL, сессии Redis, HAProxy и Docker Swarm из `mtusi2026lab3`. +Исходный проект сохранён без изменений. Go module path оставлен для совместимости импортов. +Образы, volumes, Swarm secrets и метка узла этого проекта используют имена lab4. -## Состав +В lab4 добавлены приватный MinIO, PNG/JPEG-вложения и сообщения с `attachment_ids`. +Трек 2 включает reconciler с grace period, dry-run и повторяемыми исправлениями. +Исходные миграции 001/002 сохранены; новые миграции 003/004 дополняют схему. +Обязательный контракт и публичные tests lab4 не изменены. -- `cmd/`, `internal/`, `dto/`: HTTP API, пользователи, чаты, сообщения, права доступа и пагинация. -- `internal/pkg/session/`: общие сессии в Redis. -- `internal/pkg/repository/postgres/`: PostgreSQL, транзакции и идемпотентная отправка сообщений. -- `migrations/`: исходные миграции goose `001_initial.sql` и `002_idempotency.sql` без изменений. -- `contracts/embed.go`: встраивание текущего OpenAPI в бинарный файл для Swagger UI. -- `Dockerfile`: сборка API, миграций и контейнера интеграционных тестов. -- `stack.yaml`, `infra/`: две API-реплики, HAProxy, PostgreSQL и Redis AOF. -- `scripts/`: запуск, миграции, перезапуск, остановка реплики, последовательное обновление API и измерение нагрузки. -- `tests/own/` и Go-тесты: собственные проверки предыдущих лабораторных. - -Контракт, примеры запросов и публичные проверки lab4 сохранены. Swagger UI показывает контракт lab4. -S3, вложения и поле `attachment_ids` пока не реализованы. Публичные HTTP-проверки lab4 требуют этих возможностей. -Результаты экспериментов lab3 не перенесены: они относятся к другому стенду. - -## Запуск - -Установите Docker Engine с поддержкой Swarm, Python 3.9+ и Make. -Для локальных Go-тестов установите Go 1.26.4 или новее. -Dockerfile использует Go 1.26.4 и goose 3.27.1. - -Запустите стенд на одном Docker-узле: - -```sh -make up -make wait -``` - -Команда создаёт локальные секреты, собирает образы и применяет миграции перед запуском API. -Повторный запуск сохраняет данные и применяет новые миграции перед обновлением API. -Секреты создаются отдельно для этого репозитория; копирование секретов lab3 не требуется. - -API доступен по адресу , Swagger UI — . -По умолчанию используется стек `mtusi-lab4`, образы с тегом `lab4` и метка узла `mtusi_lab4_storage`. -PostgreSQL и Redis используют отдельные volumes этого стека. Go module path сохранён для совместимости исходных импортов. - -Если порт 8080 занят, задайте другой порт: - -```sh -PORT=8081 make up -PORT=8081 make wait -``` - -Передавайте одинаковые `PORT`, `STACK_NAME`, `API_IMAGE` и `API_REPLICAS` всем командам, если меняете их значения. -Параметры передаются через переменные процесса. Скрипты не загружают файлы окружения автоматически. - -## Проверки - -Выполните проверки исходников: - -```sh -make check -make test-own -goose -env /dev/null -dir migrations validate -``` - -Для полной проверки контракта установите зависимости из `requirements-dev.txt` в виртуальное окружение. -Затем выполните `make validate PYTHON=.venv/bin/python`. - -После запуска стенда выполните интеграционные Go-тесты: - -```sh -make test-integration -``` - -Без `INTEGRATION_TESTS=1` Go-тесты пропускают сценарии с PostgreSQL. -Команда `make test-integration` запускает их в отдельном Swarm job с временной схемой БД. -Она проверяет миграции, транзакции, ограничения, пагинацию и конкурентные запросы. - -Команды `make test`, `make test-persistence` и `make test-failover` используют публичный профиль lab4. -Для их полного прохождения добавьте S3 и вложения. -Команда `make test-grade4` перенесена из lab3 и проверяет масштабирование; она не подтверждает критерии оценки lab4. - -## Управление стендом - -Проверьте состояние: - -```sh -python3 scripts/swarm.py status -make migrate-status -``` - -Перезапустите API, HAProxy, PostgreSQL и Redis с сохранением данных: - -```sh -./scripts/restart.sh -``` - -Выполните последовательное обновление API: - -```sh -make rolling-update -``` - -Остановите стенд: - -```sh -make down -``` - -Команда сохраняет volumes и Swarm secrets. Стенд выдерживает отказ API-реплики в пределах одного узла. -Отказ узла, PostgreSQL, Redis или HAProxy приводит к недоступности соответствующих операций. - -`compose.yaml` и `scripts/compose.sh` сохраняют вспомогательный вариант из предыдущих лабораторных. -Он запускает одну API-реплику без HAProxy. Основной стенд использует Swarm и `stack.yaml`. +Запуск: `make up`. Проверка API: `make test`. Остановка с сохранением данных: `make down`. +Полная инструкция и результаты находятся в [REPORT.md](REPORT.md). +Протокол согласованности описан в [reconciliation.md](contracts/reconciliation.md). +Старые результаты экспериментов lab3 не используются как доказательства lab4. diff --git a/REPORT.md b/REPORT.md index 9ad2b84..dfefd39 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,253 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Целевая оценка: **5**, трек 2 — согласованность PostgreSQL и S3. +Автор и группа: Хакимов Дмитрий БВТ2301 -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Установите Docker Engine с поддержкой Swarm, Make и Python 3.9+. +Для локальных Go-тестов установите Go 1.26.4 или новее. +Для одновременной обработки предельных изображений выделите Docker VM минимум 6 GiB RAM. -## Архитектура и решения +```sh +make up +make wait +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`make up` создаёт секреты, собирает образы и запускает PostgreSQL, Redis, MinIO и HAProxy. +Миграции goose применяются перед запуском API. Отдельная задача создаёт приватный bucket и учётную запись API. +Повторный запуск сохраняет данные. Обновление API проходит последовательно, по одной реплике. +API: . Swagger UI: . -## Доказательства критериев +Если порт занят, выполните `PORT=8081 make up`. +Передавайте одинаковые `PORT` и `STACK_NAME` последующим командам. +Скрипты получают настройки через переменные процесса. Файлы окружения автоматически не загружаются. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make check validate test-own +make test-integration +make test-lab4 +make measure-uploads +make test-reconcile +make test-persistence +make wait +make test-failover +make wait +make test-grade4 +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Запускайте сценарии отказов, сохранности и обновления последовательно. +Проверки создают синтетические данные. Токены остаются в памяти тестовых процессов. +`test-reconcile` создаёт условный тестовый trigger и временно отключает S3/БД. +Эксперимент ждёт 92 секунды: возраст настоящих S3-объектов должен превысить grace period. -## Эксперименты и отказы +Остановите стенд командой `make down`. Команда сохраняет volumes и Swarm secrets. +Для намеренного удаления **всех учебных данных** после остановки выполните: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```sh +docker volume rm mtusi-lab4_postgres_data mtusi-lab4_redis_data mtusi-lab4_s3_data +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Эта команда необратима. При другом `STACK_NAME` имена volumes отличаются. +`compose.yaml` предназначен для разработки с одной API-репликой; доказательства масштабирования используют `stack.yaml`. -## Ограничения и следующие шаги +## Компоненты -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +| Роль | Сервис / версия | Готовность | Постоянные данные | Сеть / секреты | +| --- | --- | --- | --- | --- | +| API ×2 | Go 1.26.4, Echo, pgx, minio-go | PostgreSQL, Redis, S3 через /health/ready | Нет обязательных локальных данных | backend + data; пароль БД, ключ курсоров, S3 API keys | +| PostgreSQL | postgres:17.6-alpine3.22 | pg_isready | postgres_data | data; пароль БД | +| Redis | redis:7.4.5-alpine3.21 | PING | redis_data; AOF appendfsync always | data | +| MinIO | RELEASE.2025-09-07T16-13-09Z | /minio/health/live | s3_data | data; root keys | +| HAProxy | haproxy:3.2.6-alpine | Readiness API | Нет | backend; порт 8080 | +| Bootstrap | mc RELEASE.2025-08-13T08-35-41Z | Успешное завершение задачи | Нет | data; root/API keys на время настройки | + +Go и Alpine в Dockerfiles закреплены digest. MinIO и mc собираются из исходников через `Dockerfile.storage`. +Готовые образы MinIO оказались недоступны в использованных реестрах. +Commit, SHA-256 архива и адрес исходников указаны в Dockerfile. `latest` не используется. +Идентификаторы образов и оборудование указаны в `evidence/lab4-environment.json`. + +Лимиты: PostgreSQL 1 CPU/512M, Redis 0.5 CPU/128M, MinIO 1 CPU/512M, HAProxy 0.5 CPU/128M. +Каждая API-реплика ограничена 1 CPU/1536M, `GOMEMLIMIT=1152MiB`, `GOGC=50`. +Bootstrap ограничен 0.5 CPU/512M, интеграционные тесты — 1 CPU/1G. +Это лимиты, а не резервирование памяти. + +Секреты создаются в закрытом каталоге `secrets/` и передаются через `/run/secrets`. +Они не входят в Git или build context. API не получает root keys MinIO. + +| Настройка | Значение в стенде | +| --- | --- | +| POSTGRES_HOST / POSTGRES_DB / POSTGRES_USER | postgres / messenger / messenger | +| POSTGRES_PASSWORD_FILE | /run/secrets/postgres_password | +| CURSOR_KEY_FILE | /run/secrets/cursor_key | +| REDIS_ADDR / SESSION_TTL_SECONDS | redis:6379 / 3600 | +| S3_ENDPOINT / S3_BUCKET / S3_USE_SSL | minio:9000 / attachments / false | +| S3_ACCESS_KEY_FILE / S3_SECRET_KEY_FILE | /run/secrets/s3_access_key / /run/secrets/s3_secret_key | +| INSTANCE_ID | Swarm Task ID | + +## Данные, права и память + +PostgreSQL хранит метаданные Attachment и ссылки сообщения `attachment_ids`. +Bucket `attachments` хранит оригиналы по ключам `original/`. +Имя файла используется только для отображения. Публичный API не возвращает object key. + +API проверяет сессию и участие в чате перед чтением тела. +Он принимает сырые PNG/JPEG, проверяет MIME по содержимому и полностью декодирует изображение для проверки повреждений. +Ограничение 10 MiB действует и без Content-Length. DecodeConfig проверяет максимум 40 млн пикселей до полного декодирования. +Имена с разделителями пути, NUL и управляющими символами получают 400. + +Одна API-реплика принимает и декодирует один файл одновременно. +Остальные запросы ждут свободного места в пределах общего срока 45 секунд. +Оригинал проходит через ограниченный временный файл; SHA-256 вычисляется во время копирования. +Файл удаляется при завершении запроса. После 201 API не зависит от локального оригинала. +Download читает S3 потоком через авторизованный API без redirect. + +Предельный 16-bit RGBA PNG и строки декодера могут занимать примерно 960 MB. +Поэтому лимит контейнера превышает 1 GiB. Один декодер ограничивает одновременное потребление. +Расчёт находится в [reconciliation.md](contracts/reconciliation.md). +Измерение десяти клиентов проверяет обычные изображения двух размеров, а не все предельные формы PNG. + +POST сообщения проверяет вложения своего чата и атомарно сохраняет ссылки вместе с сообщением. +Идемпотентность сравнивает текст и упорядоченный список ID; отсутствие списка равно `[]`. +PostgreSQL advisory lock сериализует конкурентные запросы одного ключа. +Повтор подтверждённого сообщения сохраняет исходный результат даже после утраты оригинала. + +Bootstrap создаёт bucket, отключает anonymous access и назначает политику отдельному API-пользователю. +API получает Get/Put/Delete для `attachments/original/*`, LIST этого префикса и GetBucketLocation. +API не может создавать bucket, менять политику или писать в другой префикс. +Консоль MinIO слушает только loopback контейнера. Только HAProxy публикует порт наружу. +Anonymous GET проверяется напрямую через S3, без API. + +## Трек 2: согласованность + +Гипотеза: возраст объектов и общая блокировка позволяют повторять сверку без удаления свежих или используемых данных. +Стоимость обхода растёт с числом объектов и метаданных, без скачивания бинарного содержимого. + +Upload держит транзакционную advisory lock ключа от S3 PUT до COMMIT метаданных. +Только успешные PUT и COMMIT разрешают вернуть 201 ready. +При неопределённом COMMIT API не удаляет объект: запись могла завершиться успешно. +После отказа БД неподтверждённый объект остаётся до следующей сверки. + +```sh +# Dry-run по умолчанию: +make reconcile +make reconcile RECONCILE_ARGS='--grace=90s' +# Применение исправлений: +make reconcile-apply +``` + +Grace по умолчанию равен двум минутам; минимум — 90 секунд при сроке upload 45 секунд. +Оператор запускает `make reconcile-apply` каждые пять минут через cron или планировщик стенда. +API не запускает сверку автоматически. Без запуска reconciler отложенная очистка не происходит. + +Reconciler читает страницы S3 LIST и страницы метаданных по 100 строк. +Он проверяет кандидата под той же advisory lock; занятую блокировку пропускает до следующего запуска. +Он удаляет только старый S3-объект без метаданных после повторного HEAD и проверки БД. +Объект с метаданными не удаляется независимо от ссылки сообщения. + +Для отсутствующего оригинала старые ready-метаданные получают внутренний статус `failed/original_missing`. +Строка и ссылки сообщений сохраняются. Metadata/content возвращают 409: схема lab4 допускает только ready-метаданные. +Восстановление байтов требует резервной копии; сверка не создаёт их заново. +Таймаут и отказ доступа S3 не считаются отсутствием объекта. + +Dry-run не меняет данные. Повторный apply после исправления не добавляет изменений. +JSON-отчёт содержит счётчики обхода, SQL/HEAD, время и не более 100 примеров действий. +`bytes_scanned` означает логический размер объектов из LIST; содержимое не скачивается. +`list_requests_estimate` оценивает число страниц. `db_queries` считает явные запросы без BEGIN/ROLLBACK. + +## Доказательства + +Результаты находятся в `evidence/lab4-*`. Публичные tests и обязательный OpenAPI не изменены. + +| Требование | Команда | Файл | +| --- | --- | --- | +| Контракт и исходники | make check validate test-own | lab4-validation.txt, lab4-go-tests.txt | +| API lab3 и вложения lab4 | make test | lab4-smoke.txt | +| Транзакции, гонки, миграции | make test-integration | lab4-integration.txt | +| PNG/JPEG, лимиты, повреждения, MIME, чужие ID/чат | make test-lab4 | lab4-acceptance.json | +| Private bucket, политика, повтор bootstrap | make test-lab4 | lab4-acceptance.json | +| Обрыв клиента, отказ S3, остановка загрузившей реплики | make test-lab4 | lab4-acceptance.json | +| Полный restart, прежний токен и оригинал | make test-persistence | lab4-persistence.txt, lab4-status-before/after-restart.json | +| Failover без дублей | make test-failover | lab4-failover.txt | +| Прежние проверки масштабирования | make test-grade4 | lab4-scaling.txt, swarm-grade4.json | +| ≥10 одновременных загрузок, память | make measure-uploads | lab4-upload-memory.json | +| Отказ между S3/БД, grace, locks, dry-run/apply/repeat | make test-reconcile | lab4-reconciliation.json | +| Свежий и полный обход, 100/1100 объектов | make test-reconcile | lab4-reconciliation.json | + +## Наблюдения и результаты + +Проверки выполнены на MacBookPro18,3: 8 CPU, 16 GiB RAM; Docker VM: 2 CPU и 2053644288 байт RAM. +Docker Engine 29.5.2, Ubuntu 24.04.4 LTS, aarch64. Версии и image IDs сохранены в `lab4-environment.json`. + +Публичный smoke прошёл. Интеграционные тесты проверили миграции с нуля, повторный запуск, откат и прежнюю схему. +Негативные HTTP-сценарии получили 400/401/404/413/415 без 500. +После обрыва клиента не появились метаданные или временные файлы. +Прямой anonymous S3 GET получил 403. Повторный bootstrap дважды сохранил оригинал. +После остановки загрузившей API-реплики download через другую восстановился за 0.510 секунды с прежним SHA-256. +При отказе S3 liveness остался 200, readiness вернул 503; стенд восстановился за 19.285 секунды. +Полный restart заменил все задачи API, HAProxy, PostgreSQL, Redis и MinIO с прежними volumes. +Публичная проверка сохранила исходный токен, сообщения и оригинал файла. +Публичный failover подтвердил 112 записей с двумя переходными ошибками; первая успешная запись восстановилась за 2.061 секунды. +Дополнительная проверка остановила обе исходные реплики по очереди и сохранила подтверждённые сообщения без дублей. +Последовательное обновление под нагрузкой завершило 3362 записи без ошибок за 44.156 секунды. + +Эксперимент загрузок: один прогревочный файл, затем один burst из десяти одновременных клиентов для каждого размера. +PNG содержит синтетический шум, чтобы размер сжатого файла оставался существенным. +Download каждого результата проверил SHA-256. Все двадцать загрузок завершились успешно. + +| Размер PNG, байт | Клиенты / ошибки | Burst, с | p50 / p95, мс | Kernel memory.peak, максимум на реплику, MiB | +| --- | --- | --- | --- | --- | +| 787252 | 10 / 0 | 0.325 | 120.287 / 172.127 | 46.44 | +| 2360847 | 10 / 0 | 0.401 | 258.573 / 324.694 | 47.61 | + +Это пик за жизнь контейнера, включая предыдущие запросы, а не отдельный профиль Go heap. +Сырые baseline, выборочные замеры и kernel peaks находятся в `lab4-upload-memory.json`. +Десять HTTP-запросов активны одновременно; декодирование внутри каждой реплики последовательно. + +При ошибке INSERT после успешного PUT API вернул 503 и оставил один неподтверждённый объект. +До истечения grace apply сохранил этот объект. Занятая advisory lock также запретила его удаление. +После grace dry-run обнаружил один orphan и один missing, не изменив данные. +Apply удалил orphan и отметил missing как failed; ссылка сообщения сохранилась. +Повторный apply выполнил ноль удалений и ноль изменений метаданных. +Живой оригинал со ссылкой сообщения сохранился. Отказы S3 и БД завершили сверку с ошибкой без разрушительных исправлений. + +Стоимость обхода измерена для двух наборов с дополнительными объектами по одному байту. +В bucket также находились 30 исходных объектов; метаданных было 30. +Для полного обхода объекты выдержаны более 90 секунд. Содержимое оригиналов не скачивалось. + +| Дополнительные объекты / возраст | Всего объектов | HEAD | SQL | Время, мс | +| --- | --- | --- | --- | --- | +| 100 / свежие | 130 | 7 | 30 | 27 | +| 1100 / свежие | 1130 | 7 | 30 | 64 | +| 100 / старше grace | 130 | 131 | 323 | 129 | +| 1100 / старше grace | 1130 | 1131 | 2323 | 999 | + +При 1130 объектах LIST занял две страницы вместо одной. +Результаты согласуются с линейным ростом числа HEAD/SQL для старых объектов. +Свежие объекты требуют LIST, но пропускают отдельные проверки ключа. +На каждую точку выполнен один проход; дисперсия и максимальная производительность не оценивались. + +## Ограничения + +Стенд использует один Docker-host и локальные volumes на закреплённом узле. +Отказ host, PostgreSQL, Redis, S3 или HAProxy не покрывается резервированием этих компонентов. +Readiness учитывает критичные зависимости; liveness проверяет только процесс. +Краткие ошибки переключения API учитываются проверками. + +Локальный нагрузочный клиент не определяет максимальный production RPS. +Выборочные `memory.current` замеры дают нижнюю оценку кратковременного пика. +Kernel `memory.peak` хранит максимум за жизнь контейнера и может включать предыдущие запросы. +Экспериментальная VM имеет меньше памяти, чем суммарные лимиты; пара предельных PNG на ней не проверяется. +Обход не является общей snapshot-транзакцией двух хранилищ. Следующий запуск обнаруживает изменения после прохода. +Протокол предполагает синхронизированные часы API и S3; в проверенном стенде контейнеры используют часы одного узла. + +## Источники + +- README, OpenAPI и семантика API этого репозитория. +- [MinIO server](https://github.com/minio/minio/tree/01ce918d8279a20e4706b96a64396146894adee4). +- [MinIO client](https://github.com/minio/mc/tree/d6541ea280b73a834b64d4097e21f2be77676104). +- [S3 PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html). +- [PostgreSQL advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS). diff --git a/cmd/main.go b/cmd/main.go index aa05fec..6f1f721 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ import ( "github.com/redis/go-redis/v9" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session" @@ -58,12 +59,16 @@ func main() { repo := postgres.New(pool, cfg.QueryTimeout) sessions := session.NewRedis(redisClient, cfg.QueryTimeout) useCase := chatusecase.New(repo, sessions, cfg.SessionTTL) - handler := chat.NewChat(useCase, cfg.CursorKey) + objectStore, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + handler := chat.NewChat(useCase, cfg.CursorKey).WithAttachments(attachment.New(pool, objectStore)) port := cfg.Port e := handler.Router() e.Server.ReadHeaderTimeout = 5 * time.Second - e.Server.ReadTimeout = 10 * time.Second - e.Server.WriteTimeout = 15 * time.Second + e.Server.ReadTimeout = 50 * time.Second + e.Server.WriteTimeout = 55 * time.Second e.Server.IdleTimeout = 60 * time.Second serverErrors := make(chan error, 1) diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 0000000..2534aa6 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "log" + "os" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + apply := flag.Bool("apply", false, "apply safe repairs; default dry-run") + grace := flag.Duration("grace", 2*time.Minute, "minimum object/metadata age (at least 90s)") + timeout := flag.Duration("timeout", 10*time.Minute, "whole scan deadline") + flag.Parse() + if *grace < attachment.MinimumGrace { + log.Fatal("grace must be at least 90s") + } + cfg, err := config.Load() + if err != nil { + log.Fatal(err) + } + pc, err := pgxpool.ParseConfig("sslmode=disable") + if err != nil { + log.Fatal("invalid DB config") + } + pc.ConnConfig.Host = cfg.PostgresHost + pc.ConnConfig.Port = cfg.PostgresPort + pc.ConnConfig.User = cfg.PostgresUser + pc.ConnConfig.Password = cfg.PostgresPassword + pc.ConnConfig.Database = cfg.PostgresDatabase + pc.ConnConfig.ConnectTimeout = 3 * time.Second + pc.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + pc.MaxConns = 2 + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, pc) + if err != nil { + log.Fatal("cannot create DB pool") + } + defer pool.Close() + store, err := attachment.NewStore(cfg.S3Endpoint, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3UseSSL) + if err != nil { + log.Fatal("invalid S3 config") + } + result, err := attachment.New(pool, store).Reconcile(ctx, *apply, *grace) + _ = json.NewEncoder(os.Stdout).Encode(result) + if err != nil { + log.Print(err) + os.Exit(1) + } +} diff --git a/compose.yaml b/compose.yaml index 59c6371..4c221cd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,6 +30,41 @@ services: retries: 30 cpus: 0.5 mem_limit: 128m + minio: + image: mtusi-minio:RELEASE.2025-09-07T16-13-09Z + build: + context: . + dockerfile: Dockerfile.storage + target: minio + secrets: [s3_root_access_key, s3_root_secret_key] + volumes: [s3_data:/data] + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live'] + interval: 2s + timeout: 3s + start_period: 15s + retries: 30 + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 20s + s3_bootstrap: + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + build: + context: . + dockerfile: Dockerfile.storage + target: mc + entrypoint: [/bin/sh, /bootstrap.sh] + configs: + - source: s3_bootstrap + target: /bootstrap.sh + - source: s3_policy + target: /policy.json + secrets: [s3_root_access_key, s3_root_secret_key, s3_access_key, s3_secret_key] + depends_on: + minio: + condition: service_healthy + cpus: 0.5 + mem_limit: 512m migrations: image: mtusi-migrations:lab4 build: @@ -59,9 +94,18 @@ services: CURSOR_KEY_FILE: /run/secrets/cursor_key SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} QUERY_TIMEOUT_MS: '2000' + S3_ENDPOINT: minio:9000 + S3_BUCKET: attachments + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + S3_USE_SSL: 'false' + GOMEMLIMIT: 1152MiB + GOGC: '50' secrets: &api_secrets - postgres_password - cursor_key + - s3_access_key + - s3_secret_key ports: - '127.0.0.1:${PORT:-8080}:8080' depends_on: @@ -71,8 +115,10 @@ services: condition: service_healthy migrations: condition: service_completed_successfully + s3_bootstrap: + condition: service_completed_successfully cpus: 1.0 - mem_limit: 256m + mem_limit: 1536m stop_grace_period: 10s tests: image: mtusi-messenger-tests:lab4 @@ -91,7 +137,20 @@ services: condition: service_healthy cpus: 1.0 mem_limit: 1g +configs: + s3_bootstrap: + file: ./infra/s3-bootstrap.sh + s3_policy: + file: ./infra/s3-policy.json secrets: + s3_root_access_key: + file: ./secrets/s3_root_access_key + s3_root_secret_key: + file: ./secrets/s3_root_secret_key + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key postgres_password: file: ./secrets/postgres_password cursor_key: @@ -99,3 +158,4 @@ secrets: volumes: postgres_data: redis_data: + s3_data: diff --git a/contracts/reconciliation.md b/contracts/reconciliation.md new file mode 100644 index 0000000..e1ce8c5 --- /dev/null +++ b/contracts/reconciliation.md @@ -0,0 +1,39 @@ +# Согласованность PostgreSQL и S3 (исследовательский трек 2) + +API хранит бинарный оригинал только в приватном S3 bucket. PostgreSQL хранит метаданные и упорядоченный массив UUID в сообщении. Пустой массив и отсутствие `attachment_ids` эквивалентны; перестановка UUID меняет тело идемпотентного запроса. + +## Запись и граница подтверждения + +До декодирования API записывает не более 10 MiB + 1 байта во временный файл. Файл служит только для валидации и передачи: API удаляет его при любом завершении операции. Один слот на процесс ограничивает число одновременных декодирований. PNG/JPEG полностью декодируется после проверки размера не более 40 миллионов пикселей. Дедлайн запроса — 45 секунд, включая ожидание слота. + +После проверки API генерирует ключ `original/`, открывает транзакцию PostgreSQL и получает `pg_advisory_xact_lock(hashtextextended(object_key,0))`. Под этой блокировкой API выполняет S3 PUT, INSERT метаданных и COMMIT. Ответ 201 возможен только после успешного COMMIT. + +Если S3 PUT завершён, а INSERT/COMMIT не подтверждён, объект остаётся для сверки. API не удаляет его при неизвестном результате COMMIT: сервер БД мог успеть зафиксировать метаданные. Прерывание процесса освобождает блокировку. Уникальные серверные ключи не переиспользуются. + +## Повторяемая сверка + +Запустите `/usr/local/bin/reconcile --grace=2m` в контейнере API. По умолчанию это dry-run. Флаг `--apply` включает исправления. Минимальное grace — 90 секунд: больше максимального времени загрузки 45 секунд. Продолжительность полного обхода ограничивает `--timeout`, по умолчанию 10 минут. + +1. Reconciler получает список объектов только с prefix `original/`. Свежие объекты пропускает по серверному LastModified S3. +2. Для старого объекта получает неблокирующую транзакционную advisory-блокировку по ключу. Занятые ключи пропускает. Повторно проверяет наличие метаданных, существование и возраст объекта. +3. Если метаданных нет, dry-run регистрирует orphan. Apply удаляет этот объект. Если метаданные есть, объект сохраняется независимо от наличия сообщения. +4. Затем reconciler читает метаданные страницами по 100 строк. Свежие и уже failed строки пропускает. Для старой ready строки получает ту же блокировку и блокировку строки FOR UPDATE, повторно проверяет возраст/статус и выполняет S3 HEAD. +5. Только подтверждённый NoSuchKey переводит метаданные в `failed` с `error_code=original_missing`. Ошибка доступа, timeout и отказ S3 прерывают обход без такого исправления. Метаданные и ссылки сообщений не удаляются. Отдельный счётчик показывает missing, на которые ссылаются сообщения. + +В обязательном контракте lab4 схема Attachment разрешает только status=ready. Поэтому GET метаданных и content для внутренней failed-записи возвращают 409. Попытка прикрепить её к новому сообщению также возвращает 409. Внутреннее состояние доступно в PostgreSQL и JSON-результате reconciler. Повтор уже подтверждённого сообщения сохраняет прежний результат, даже если оригинал позже пропал. + +Dry-run не изменяет S3 или PostgreSQL. Повтор apply не размножает исправления: удалённый объект отсутствует в списке, failed-строка больше не кандидат. Внешнее удаление используемого оригинала исправляется сохранением failed-метаданных, а не удалением сообщения. Reconciler не восстанавливает потерянные байты; для восстановления нужна резервная копия. + +## Граница памяти декодирования + +Один upload-слот на реплику ограничивает одновременно живые декодеры. Ограничение 40 миллионов пикселей допускает 16-bit RGBA PNG: до 320 000 000 байт для результата. Стандартный Go PNG decoder дополнительно выделяет две строки по `1 + ceil(bits_per_pixel * width / 8)` байт. Для допустимого PNG размером 40 000 000 × 1 это ещё 640 000 002 байта. Поэтому консервативная оценка живой памяти самого декодера — около 960 MB (916 MiB), а не только 320 MB. Для Adam7 нужна также память промежуточного прохода; при заданном числе пикселей эта оценка покрывает текущую реализацию декодера Go. + +Контейнер API имеет лимит 1536 MiB, `GOMEMLIMIT=1152MiB` и `GOGC=50`. Это оставляет запас для HTTP, SDK, GC и до четырёх Argon2 операций по 19 MiB. GOMEMLIMIT — мягкий ориентир GC, а не жёсткая гарантия RSS; жёсткий предел задаёт контейнер. Временный файл ограничен 10 MiB + 1 байтом и не входит в обязательное постоянное хранилище. Ожидающие запросы не начинают читать тело до освобождения слота. Выбранная конфигурация сохраняет поддержку крайних соотношений сторон без дополнительного ограничения ширины, которого нет в контракте. + +Измеренный пик для обычных fixtures и десяти конкурентных запросов приведён в отчёте. Он не заменяет оценку худшего случая для всех корректных PNG. Код стандартного decoder и параметры его памяти нужно проверять при обновлении Go. + +## Стоимость и границы + +JSON содержит число объектов и строк метаданных, сумму размеров объектов и время обхода. Отдельные счётчики показывают HEAD, SQL, страницы LIST, кандидатов, исправления и пропуски. Сумма размеров — объём перечисленных оригиналов, а не переданные байты: бинарные объекты при сверке не скачиваются. Счётчик SQL не включает BEGIN/ROLLBACK/COMMIT. LIST оценивается по страницам до 1000 элементов. Обход имеет линейную стоимость по количеству объектов и строк, без загрузки оригиналов в RAM. Страницы метаданных ограничены 100 строками, список действий в отчёте ограничен 100 примерами. + +Безопасность конкурентной работы гарантируется для API и reconciler, которые соблюдают этот протокол. Администратор с root credentials может нарушить инварианты прямым изменением S3/БД. Ошибка сети может оставить объект без метаданных до следующего успешного запуска reconciler. Планировщик должен регулярно повторять обход; для лабораторной запуск выполняется явно и сохраняет доказательства. diff --git a/dto/create_message.go b/dto/create_message.go index 7cfeb14..f89c57c 100644 --- a/dto/create_message.go +++ b/dto/create_message.go @@ -1,14 +1,16 @@ package dto type CreateMessageRequest struct { - ChatID string `json:"-"` - Text string `json:"text"` + ChatID string `json:"-"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` } type CreateMessageResponse struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt Timestamp `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` } diff --git a/dto/create_message_json.go b/dto/create_message_json.go new file mode 100644 index 0000000..b92e99d --- /dev/null +++ b/dto/create_message_json.go @@ -0,0 +1,33 @@ +package dto + +import ( + "bytes" + "encoding/json" + "errors" +) + +// Keep absent attachment_ids equivalent to [], while rejecting JSON null and +// wrong types rather than quietly treating them as an empty message body. +func (r *CreateMessageRequest) UnmarshalJSON(data []byte) error { + type payload CreateMessageRequest + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil || fields == nil { + return errors.New("Ожидается JSON-объект") + } + text, exists := fields["text"] + if !exists || bytes.Equal(bytes.TrimSpace(text), []byte("null")) { + return errors.New("Требуется строка text") + } + if ids, exists := fields["attachment_ids"]; exists && bytes.Equal(bytes.TrimSpace(ids), []byte("null")) { + return errors.New("attachment_ids должен быть массивом") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var decoded payload + if err := decoder.Decode(&decoded); err != nil { + return err + } + decoded.ChatID = r.ChatID + *r = CreateMessageRequest(decoded) + return nil +} diff --git a/dto/create_message_json_test.go b/dto/create_message_json_test.go new file mode 100644 index 0000000..0d97cd0 --- /dev/null +++ b/dto/create_message_json_test.go @@ -0,0 +1,21 @@ +package dto + +import ( + "encoding/json" + "testing" +) + +func TestMessageStrictFields(t *testing.T) { + for _, body := range []string{`null`, `[]`, `{}`, `{"text":null}`, `{"text":"x","attachment_ids":null}`, `{"text":"x","attachment_ids":"x"}`, `{"text":"x","extra":1}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err == nil { + t.Fatalf("accepted %s", body) + } + } + for _, body := range []string{`{"text":"x"}`, `{"text":"x","attachment_ids":[]}`, `{"text":"","attachment_ids":["id"]}`} { + var r CreateMessageRequest + if err := json.Unmarshal([]byte(body), &r); err != nil { + t.Fatalf("rejected %s: %v", body, err) + } + } +} diff --git a/dto/list_messages.go b/dto/list_messages.go index a7d82f6..35a3842 100644 --- a/dto/list_messages.go +++ b/dto/list_messages.go @@ -12,9 +12,10 @@ type ListMessagesResponse struct { } type ListMessagesItem struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt Timestamp `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + CreatedAt Timestamp `json:"created_at"` } diff --git a/evidence/lab4-acceptance.json b/evidence/lab4-acceptance.json new file mode 100644 index 0000000..8f45301 --- /dev/null +++ b/evidence/lab4-acceptance.json @@ -0,0 +1,104 @@ +{ + "negative_cases": [ + { + "case": "oversized_bytes", + "status": 413 + }, + { + "case": "damaged_png", + "status": 415 + }, + { + "case": "corrupt_crc", + "status": 415 + }, + { + "case": "mime_spoof", + "status": 415 + }, + { + "case": "not_image", + "status": 415 + }, + { + "case": "unsupported_mime", + "status": 415 + }, + { + "case": "pixel_budget", + "status": 413 + }, + { + "case": "unsafe_slash", + "status": 400 + }, + { + "case": "unsafe_backslash", + "status": 400 + }, + { + "case": "unsafe_control", + "status": 400 + }, + { + "case": "empty_filename", + "status": 400 + }, + { + "case": "foreign_attachment", + "status": 404 + }, + { + "case": "foreign_chat_link", + "status": 404 + }, + { + "case": "unknown_attachment", + "status": 404 + }, + { + "case": "duplicate_ids", + "status": 400 + }, + { + "case": "missing_authentication", + "status": 401 + }, + { + "case": "unknown_variant", + "status": 404 + }, + { + "case": "chunked_oversized", + "status": 413 + } + ], + "failures": { + "client_disconnect": { + "metadata_added": 0, + "spool_files_remaining": 0 + }, + "upload_replica_failure": { + "upload_instance": "alixbvpbjnzvtqvo5hhfpokc6", + "download_instance": "h6fc30dhoabp0jcfl6zrpjdzx", + "recovery_seconds": 0.51, + "sha256": "8502afe2cc5bab33ac834c834874369d29e7896599b5a256d68d80c5395d99c3" + }, + "s3_outage": { + "liveness": 200, + "readiness": 503, + "recovery_seconds": 19.285 + } + }, + "checksums": { + "image/png": "8502afe2cc5bab33ac834c834874369d29e7896599b5a256d68d80c5395d99c3", + "image/jpeg": "41feb40d75a595386c72cfc8a5bc013c2dedc445b73316ebbd3b3eac13491c5d" + }, + "bucket_security": { + "anonymous_get_status": 403, + "create_bucket_denied": true, + "write_outside_prefix_denied": true, + "console_published": false, + "bootstrap_repeats": 2 + } +} diff --git a/evidence/lab4-acceptance.txt b/evidence/lab4-acceptance.txt new file mode 100644 index 0000000..3a0ea76 --- /dev/null +++ b/evidence/lab4-acceptance.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/own/lab4.py +PASS: PNG/JPEG, bounded uploads, MIME/damage/ACL/link errors, disconnect, private S3, repeat bootstrap, S3 outage. diff --git a/evidence/lab4-bucket-bootstrap.txt b/evidence/lab4-bucket-bootstrap.txt new file mode 100644 index 0000000..533378d --- /dev/null +++ b/evidence/lab4-bucket-bootstrap.txt @@ -0,0 +1,8 @@ +image mtusi-mc:RELEASE.2025-08-13T08-35-41Z could not be accessed on a registry to record +its digest. Each node will access mtusi-mc:RELEASE.2025-08-13T08-35-41Z independently, +possibly leading to different nodes running different +versions of the image. + +0pza81k652zzumhh8w5zfuxw8 +mtusi-lab4_s3-bootstrap.0.jjp4j3jx92l1@colima | PASS: private bucket and restricted API account configured. +mtusi-lab4_s3-bootstrap diff --git a/evidence/lab4-build.txt b/evidence/lab4-build.txt new file mode 100644 index 0000000..4130195 --- /dev/null +++ b/evidence/lab4-build.txt @@ -0,0 +1,112 @@ +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 401.9kB +Step 1/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/35 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/35 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/35 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/35 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/35 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/35 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/35 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/35 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/35 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/35 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/35 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/35 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/35 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/35 : COPY internal ./internal + ---> 2f61826e3774 +Step 19/35 : COPY migrations ./migrations + ---> d96dd811bc83 +Step 20/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Running in 3d7cd429d659 + ---> Removed intermediate container 3d7cd429d659 + ---> 2ad69db4c288 +Step 21/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Running in 7b9a510ed472 + ---> Removed intermediate container 7b9a510ed472 + ---> 4a49dc2719a3 +Step 22/35 : FROM builder AS tests + ---> 4a49dc2719a3 +Step 23/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> 5df1413dfc44 +Step 24/35 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Running in c30eee45a5e7 + ---> Removed intermediate container c30eee45a5e7 + ---> 173325b59e7d +Step 25/35 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Running in 384e08125582 + ---> Removed intermediate container 384e08125582 + ---> 187ea6b3bcac +Step 26/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 27/35 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 33192b561bfc +Step 28/35 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> 37a18e0c6edf +Step 29/35 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> b0e79d0416ba +Step 30/35 : ENV PORT=8080 + ---> Running in 48da45f81d52 + ---> Removed intermediate container 48da45f81d52 + ---> a7c04e9ff97b +Step 31/35 : EXPOSE 8080 + ---> Running in 671ff92f3fad + ---> Removed intermediate container 671ff92f3fad + ---> e56b07b42f04 +Step 32/35 : USER messenger:messenger + ---> Running in 8003e4ea3b5d + ---> Removed intermediate container 8003e4ea3b5d + ---> 48b4424b156b +Step 33/35 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Running in c05d11001e2b + ---> Removed intermediate container c05d11001e2b + ---> 563fbe4dcd7d +Step 34/35 : STOPSIGNAL SIGTERM + ---> Running in 8fb80a0e0f6c + ---> Removed intermediate container 8fb80a0e0f6c + ---> c5a0a88ba147 +Step 35/35 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Running in 9e51b29d5939 + ---> Removed intermediate container 9e51b29d5939 + ---> c28dcd6e4410 +Successfully built c28dcd6e4410 +Successfully tagged mtusi-messenger:lab4 diff --git a/evidence/lab4-deploy.txt b/evidence/lab4-deploy.txt new file mode 100644 index 0000000..6a3e6ce --- /dev/null +++ b/evidence/lab4-deploy.txt @@ -0,0 +1,313 @@ +.venv/bin/python scripts/swarm.py up +vphhh79vd5ti8501q6m9lzulj +Local secrets are ready. +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.8kB +Step 1/20 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder + ---> 18b460dd1754 +Step 2/20 : ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz + + + ---> Using cache + ---> 4a2b9310629e +Step 3/20 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> 5166de215d05 +Step 4/20 : WORKDIR /src + ---> Using cache + ---> 1f12c7444ff0 +Step 5/20 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . + ---> Using cache + ---> 8477e9d7324c +Step 6/20 : RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> 94162de69c8c +Step 7/20 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder + ---> 18b460dd1754 +Step 8/20 : ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz + + + ---> Using cache + ---> 5e135fc72a2c +Step 9/20 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> cbffdcf35dd5 +Step 10/20 : WORKDIR /src + ---> Using cache + ---> 3322378e2ebd +Step 11/20 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . + ---> Using cache + ---> 0aba858f1d48 +Step 12/20 : RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> af0a57fdfed1 +Step 13/20 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio + ---> 075c2c1a4068 +Step 14/20 : LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 57200264604d +Step 15/20 : RUN mkdir /data && chown 10001:10001 /data + ---> Using cache + ---> c7024406f9a9 +Step 16/20 : COPY --from=minio-builder /out/minio /usr/local/bin/minio + ---> Using cache + ---> 0d1965ab7125 +Step 17/20 : COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio + ---> Using cache + ---> e5c21da70a4f +Step 18/20 : USER 10001:10001 + ---> Using cache + ---> 9bc52122d9f2 +Step 19/20 : EXPOSE 9000 + ---> Using cache + ---> 2ca448e77273 +Step 20/20 : ENTRYPOINT ["/usr/local/bin/start-minio"] + ---> Using cache + ---> 3a0797aa5ead +Successfully built 3a0797aa5ead +Successfully tagged mtusi-minio:RELEASE.2025-09-07T16-13-09Z +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.8kB +Step 1/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS minio-builder + ---> 18b460dd1754 +Step 2/25 : ADD --checksum=sha256:8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 https://codeload.github.com/minio/minio/tar.gz/01ce918d8279a20e4706b96a64396146894adee4 /tmp/source.tar.gz + + + ---> Using cache + ---> 4a2b9310629e +Step 3/25 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> 5166de215d05 +Step 4/25 : WORKDIR /src + ---> Using cache + ---> 1f12c7444ff0 +Step 5/25 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/minio . + ---> Using cache + ---> 8477e9d7324c +Step 6/25 : RUN echo '8b11129f6a6830768bcdebab569290ad956618f5ff6ef4857b633acb64f95126 /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> 94162de69c8c +Step 7/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS mc-builder + ---> 18b460dd1754 +Step 8/25 : ADD --checksum=sha256:4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e https://codeload.github.com/minio/mc/tar.gz/d6541ea280b73a834b64d4097e21f2be77676104 /tmp/source.tar.gz + + + ---> Using cache + ---> 5e135fc72a2c +Step 9/25 : RUN mkdir /src && tar -xzf /tmp/source.tar.gz -C /src --strip-components=1 + ---> Using cache + ---> cbffdcf35dd5 +Step 10/25 : WORKDIR /src + ---> Using cache + ---> 3322378e2ebd +Step 11/25 : RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/mc . + ---> Using cache + ---> 0aba858f1d48 +Step 12/25 : RUN echo '4cea8ed209c8f96e59ea1a5cc4901019b54c2be6d6f5f16c6749ea8bdcf4172e /tmp/source.tar.gz' | sha256sum -c - + ---> Using cache + ---> af0a57fdfed1 +Step 13/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS minio + ---> 075c2c1a4068 +Step 14/25 : LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.revision="01ce918d8279a20e4706b96a64396146894adee4" org.opencontainers.image.version="RELEASE.2025-09-07T16-13-09Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 57200264604d +Step 15/25 : RUN mkdir /data && chown 10001:10001 /data + ---> Using cache + ---> c7024406f9a9 +Step 16/25 : COPY --from=minio-builder /out/minio /usr/local/bin/minio + ---> Using cache + ---> 0d1965ab7125 +Step 17/25 : COPY infra/minio-entrypoint.sh /usr/local/bin/start-minio + ---> Using cache + ---> e5c21da70a4f +Step 18/25 : USER 10001:10001 + ---> Using cache + ---> 9bc52122d9f2 +Step 19/25 : EXPOSE 9000 + ---> Using cache + ---> 2ca448e77273 +Step 20/25 : ENTRYPOINT ["/usr/local/bin/start-minio"] + ---> Using cache + ---> 3a0797aa5ead +Step 21/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS mc + ---> 075c2c1a4068 +Step 22/25 : LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.revision="d6541ea280b73a834b64d4097e21f2be77676104" org.opencontainers.image.version="RELEASE.2025-08-13T08-35-41Z" org.opencontainers.image.licenses="AGPL-3.0" + ---> Using cache + ---> 9257dca2de26 +Step 23/25 : COPY --from=mc-builder /out/mc /usr/local/bin/mc + ---> Using cache + ---> 4bea21e69e8d +Step 24/25 : USER 10001:10001 + ---> Using cache + ---> 45d00b9fce3e +Step 25/25 : ENTRYPOINT ["mc"] + ---> Using cache + ---> 0713832dbf4a +Successfully built 0713832dbf4a +Successfully tagged mtusi-mc:RELEASE.2025-08-13T08-35-41Z +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/35 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/35 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/35 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/35 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/35 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/35 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/35 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/35 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/35 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/35 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/35 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/35 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/35 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/35 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/35 : COPY internal ./internal + ---> Using cache + ---> 72552b328db4 +Step 19/35 : COPY migrations ./migrations + ---> Using cache + ---> a68ac4a31860 +Step 20/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> 7303ceff230d +Step 21/35 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> e6cc7991dac9 +Step 22/35 : FROM builder AS tests + ---> e6cc7991dac9 +Step 23/35 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 5749aa2abe35 +Step 24/35 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> ca0a6f733ba8 +Step 25/35 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> c643755b8aac +Step 26/35 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 + ---> 075c2c1a4068 +Step 27/35 : RUN addgroup -S messenger && adduser -S -D -H -u 10001 -G messenger messenger + ---> Using cache + ---> 33192b561bfc +Step 28/35 : COPY --from=builder --chown=messenger:messenger /out/messenger /usr/local/bin/messenger + ---> Using cache + ---> 136320eda1d3 +Step 29/35 : COPY --from=builder --chown=messenger:messenger /out/reconcile /usr/local/bin/reconcile + ---> Using cache + ---> 4acf2a8fc638 +Step 30/35 : ENV PORT=8080 + ---> Using cache + ---> 6fbea06a8135 +Step 31/35 : EXPOSE 8080 + ---> Using cache + ---> 333a62acf202 +Step 32/35 : USER messenger:messenger + ---> Using cache + ---> 60a29210a7f2 +Step 33/35 : HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 CMD wget -q -O /dev/null "http://127.0.0.1:${PORT}/health/ready" || exit 1 + ---> Using cache + ---> 803ac1f944e5 +Step 34/35 : STOPSIGNAL SIGTERM + ---> Using cache + ---> 40299d0a0f9f +Step 35/35 : ENTRYPOINT ["/usr/local/bin/messenger"] + ---> Using cache + ---> c81ae0702578 +Successfully built c81ae0702578 +Successfully tagged mtusi-messenger:lab4 +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/9 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/9 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/9 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/9 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/9 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/9 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/9 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/9 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/9 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Successfully built 841d9bb239cd +Successfully tagged mtusi-migrations:lab4 +kkk3r78zn9bb09xudyehchs4p +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 001_initial.sql (17.71ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 002_idempotency.sql (2.54ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 003_attachments.sql (5.13ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 OK 004_message_attachments.sql (2.34ms) +mtusi-lab4_migrations.0.mzx2ds4dsbyh@colima | 2026/10/28 19:14:47 goose: successfully migrated database to version: 4 +mtusi-lab4_migrations +Since --detach=false was not specified, tasks will be created in the background. +In a future release, --detach=false will become the default. +Updating service mtusi-lab4_redis (id: zlhenabo1vgqarsazgqa6d897) +Updating service mtusi-lab4_minio (id: ygvoo1fje6vqlxw02ugw7awa8) +Updating service mtusi-lab4_api (id: xiueb8lcewfukmzn8wocqcmjh) +Updating service mtusi-lab4_lb (id: uo65o8allz5azrqbx4zchi811) +Updating service mtusi-lab4_postgres (id: vxi6yu73ah85o0xog5w8r1t5k) +f6qbximmjnxx8ezv05jxgsngz +mtusi-lab4_s3-bootstrap.0.p9vksdlsfz2l@colima | PASS: private bucket and restricted API account configured. +mtusi-lab4_s3-bootstrap +mtusi-lab4_api scaled to 2 +mtusi-lab4_api +PASS: rolling update completed +PASS: Swarm stack ready at http://localhost:8080 diff --git a/evidence/lab4-environment.json b/evidence/lab4-environment.json new file mode 100644 index 0000000..d285695 --- /dev/null +++ b/evidence/lab4-environment.json @@ -0,0 +1,81 @@ +{ + "docker_engine": { + "cpus": 2, + "memory_bytes": 2053644288, + "os": "Ubuntu 24.04.4 LTS", + "kernel": "6.8.0-117-generic", + "architecture": "aarch64", + "version": "29.5.2" + }, + "host": { + "architecture": "arm64", + "system": "Darwin", + "model": "MacBookPro18,3", + "cpu_cores": 8, + "memory_bytes": 17179869184 + }, + "host_go": "go version go1.27.2 darwin/arm64", + "host_python": "3.9.6", + "images": [ + { + "tag": "mtusi-messenger:lab4", + "image_id": "sha256:c28dcd6e4410ee011a827852286b93296c8a0f20ddb268524b4d92371014403e", + "repo_digests": [ + "mtusi-messenger@sha256:c28dcd6e4410ee011a827852286b93296c8a0f20ddb268524b4d92371014403e" + ], + "source_labels": {} + }, + { + "tag": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "image_id": "sha256:3a0797aa5eadafe23e8a7b205c39511a9280b7500da6ef0e5d604cc74f46c3ed", + "repo_digests": [ + "mtusi-minio@sha256:3a0797aa5eadafe23e8a7b205c39511a9280b7500da6ef0e5d604cc74f46c3ed" + ], + "source_labels": { + "org.opencontainers.image.licenses": "AGPL-3.0", + "org.opencontainers.image.revision": "01ce918d8279a20e4706b96a64396146894adee4", + "org.opencontainers.image.source": "https://github.com/minio/minio", + "org.opencontainers.image.version": "RELEASE.2025-09-07T16-13-09Z" + } + }, + { + "tag": "mtusi-mc:RELEASE.2025-08-13T08-35-41Z", + "image_id": "sha256:0713832dbf4ab9a3b716ab0a4cca6629fec96224bc1425f26ab6ba42c5463a99", + "repo_digests": [ + "mtusi-mc@sha256:0713832dbf4ab9a3b716ab0a4cca6629fec96224bc1425f26ab6ba42c5463a99" + ], + "source_labels": { + "org.opencontainers.image.licenses": "AGPL-3.0", + "org.opencontainers.image.revision": "d6541ea280b73a834b64d4097e21f2be77676104", + "org.opencontainers.image.source": "https://github.com/minio/mc", + "org.opencontainers.image.version": "RELEASE.2025-08-13T08-35-41Z" + } + }, + { + "tag": "postgres:17.6-alpine3.22", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "repo_digests": [ + "postgres@sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94" + ], + "source_labels": {} + }, + { + "tag": "redis:7.4.5-alpine3.21", + "image_id": "sha256:bb186d083732f669da90be8b0f975a37812b15e913465bb14d845db72a4e3e08", + "repo_digests": [ + "redis@sha256:bb186d083732f669da90be8b0f975a37812b15e913465bb14d845db72a4e3e08" + ], + "source_labels": {} + }, + { + "tag": "haproxy:3.2.6-alpine", + "image_id": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "repo_digests": [ + "haproxy@sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c" + ], + "source_labels": {} + } + ], + "go_container_version": "1.26.4", + "test_data": "synthetic; no credentials/tokens saved" +} diff --git a/evidence/lab4-failover.txt b/evidence/lab4-failover.txt new file mode 100644 index 0000000..946745b --- /dev/null +++ b/evidence/lab4-failover.txt @@ -0,0 +1,4 @@ +.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 2, "first_recovery_seconds": 2.061} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-go-tests.txt b/evidence/lab4-go-tests.txt new file mode 100644 index 0000000..d915cb2 --- /dev/null +++ b/evidence/lab4-go-tests.txt @@ -0,0 +1,13 @@ +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto (cached) +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 13.184s +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 2.905s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password (cached) +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 2.388s +? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat (cached) diff --git a/evidence/lab4-integration.txt b/evidence/lab4-integration.txt new file mode 100644 index 0000000..9b82a00 --- /dev/null +++ b/evidence/lab4-integration.txt @@ -0,0 +1,338 @@ +.venv/bin/python scripts/swarm.py test-integration +DEPRECATED: The legacy builder is deprecated and will be removed in a future release. + Install the buildx component to build images with BuildKit: + https://docs.docker.com/go/buildx/ + +Sending build context to Docker daemon 398.3kB +Step 1/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS goose-builder + ---> 18b460dd1754 +Step 2/25 : RUN GOBIN=/out CGO_ENABLED=0 go install -tags='no_clickhouse no_libsql no_mssql no_mysql no_sqlite3 no_vertica no_ydb no_spanner' github.com/pressly/goose/v3/cmd/goose@v3.27.1 + ---> Using cache + ---> 7012ef857c0c +Step 3/25 : FROM alpine:3.23@sha256:075c2c1a4068c1e251228ca2b0ea8b163f835ca5d66e758b93c7fc89b1995616 AS migrations + ---> 075c2c1a4068 +Step 4/25 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 6a0ad52336c1 +Step 5/25 : COPY migrations /migrations + ---> Using cache + ---> 19018c7bfac9 +Step 6/25 : COPY infra/migrate.sh /usr/local/bin/migrate + ---> Using cache + ---> 0cc075ba3459 +Step 7/25 : USER 10001:10001 + ---> Using cache + ---> 37de51af32cd +Step 8/25 : ENTRYPOINT ["/usr/local/bin/migrate"] + ---> Using cache + ---> e7539a967679 +Step 9/25 : CMD ["up"] + ---> Using cache + ---> 841d9bb239cd +Step 10/25 : FROM golang:1.26.4-alpine3.23@sha256:18b460dd17542c2ba43299a633cf6ebfc1115101509531471d7cfce1019af083 AS builder + ---> 18b460dd1754 +Step 11/25 : WORKDIR /src + ---> Using cache + ---> 1ba013278a4a +Step 12/25 : RUN apk add --no-cache gcc musl-dev + ---> Using cache + ---> a0ea65f0bded +Step 13/25 : COPY go.mod go.sum ./ + ---> Using cache + ---> e98ccecfb093 +Step 14/25 : RUN go mod download + ---> Using cache + ---> 91d546c71507 +Step 15/25 : COPY cmd ./cmd + ---> Using cache + ---> b4e431a9d5f2 +Step 16/25 : COPY contracts ./contracts + ---> Using cache + ---> 958477d6c98e +Step 17/25 : COPY dto ./dto + ---> Using cache + ---> d115fe2e1af4 +Step 18/25 : COPY internal ./internal + ---> Using cache + ---> 72552b328db4 +Step 19/25 : COPY migrations ./migrations + ---> Using cache + ---> a68ac4a31860 +Step 20/25 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd + ---> Using cache + ---> 7303ceff230d +Step 21/25 : RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/reconcile ./cmd/reconcile + ---> Using cache + ---> e6cc7991dac9 +Step 22/25 : FROM builder AS tests + ---> e6cc7991dac9 +Step 23/25 : COPY --from=goose-builder /out/goose /usr/local/bin/goose + ---> Using cache + ---> 5749aa2abe35 +Step 24/25 : ENV GOOSE_MIGRATION_DIR=/src/migrations + ---> Using cache + ---> ca0a6f733ba8 +Step 25/25 : CMD ["go", "test", "-p=1", "-race", "-count=1", "-v", "./..."] + ---> Using cache + ---> c643755b8aac +Successfully built c643755b8aac +Successfully tagged mtusi-messenger-tests:lab4 +qs9c5xskdu6nyjiv287eg68b8 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/cmd/reconcile [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/contracts [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestMessageStrictFields +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestMessageStrictFields (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestTimestampFixedMicroseconds +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestTimestampFixedMicroseconds (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/dto 1.013s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsMalformedJSON +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:40 request_id=94533522-1e48-4953-9857-91c107c2aac7 method=POST path=/api/v1/users status=400 response_bytes=140 latency=130.671µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsMalformedJSON (0.01s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestHidesForeignChat +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:40 request_id=8b136f4b-a1b6-4756-9dda-92871979845c method=POST path=/api/v1/users status=201 response_bytes=141 latency=369.112982ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=df98ca4b-5f1f-4ef1-a8eb-7708ccfe2367 method=POST path=/api/v1/auth/sessions status=201 response_bytes=269 latency=397.145342ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=c535a54a-6ce6-49b0-8ec7-47e70ea04799 method=POST path=/api/v1/users status=201 response_bytes=147 latency=364.216754ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=fea0410a-7349-4fc3-8541-f479066736b7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=275 latency=359.930021ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=b7201791-c957-4ee8-8237-2aab675777ec method=POST path=/api/v1/chats status=201 response_bytes=163 latency=174.381µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:41 request_id=30b379c6-e33c-42bc-963a-54de252a1e44 method=GET path=/api/v1/chats/:chat_id status=404 response_bytes=126 latency=53.002µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestHidesForeignChat (1.50s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsEmptyMessage +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=92d81694-3fcf-452b-be3d-24e21f938fb8 method=POST path=/api/v1/users status=201 response_bytes=157 latency=411.707889ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=792cc573-4974-4744-86bc-7ce29f78c5a2 method=POST path=/api/v1/auth/sessions status=201 response_bytes=285 latency=357.462544ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=4aec085d-f3fe-4b56-a0f2-b6bb55c32fd5 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=135.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=bae49af4-9cb9-414e-8227-7991f680a0ab method=POST path=/api/v1/chats/:chat_id/messages status=400 response_bytes=172 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsEmptyMessage (0.77s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestRejectsDuplicateUsername +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:42 request_id=c6f9b126-b2a0-4672-b4de-a59183cffe86 method=POST path=/api/v1/users status=201 response_bytes=149 latency=370.366912ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:43 request_id=c174982c-6a8d-4fc7-8d4b-6cbe68b34d70 method=POST path=/api/v1/auth/sessions status=201 response_bytes=277 latency=365.829696ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:43 request_id=b8771b83-4e05-4335-a6bd-4de115eb5e20 method=POST path=/api/v1/users status=409 response_bytes=170 latency=358.448604ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestRejectsDuplicateUsername (1.10s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestPaginatesMessagesAcrossMultiplePages +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=fb5f658b-7688-4017-9758-bea1d60af1cf method=POST path=/api/v1/users status=201 response_bytes=151 latency=366.583521ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=ddf8c2e9-34d2-4b17-aaa5-66ee679b9a4b method=POST path=/api/v1/auth/sessions status=201 response_bytes=279 latency=393.325397ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=32abde1e-d2e7-46fe-ad10-210533f83bd5 method=POST path=/api/v1/chats status=201 response_bytes=163 latency=136.838µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=227b702f-3d9d-4307-a1a8-30ab4f7aa8f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=217.05µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=edc8c38d-6beb-4353-b6c1-d1c8f337282b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=132.629µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=27b25c6f-8b85-4f74-9a26-bc175b6e490f method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=120.337µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=148f8872-b793-4898-a57f-78f66d57fbb6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=115.879µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=d2b4fae7-2633-4406-920a-af6bad40e7c9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=113.254µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=65190391-2026-4f9e-b907-b52379ad8afe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=141.005µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=094c52b2-7bb7-4ba0-be47-2695d64f4867 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=228 latency=181.464µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=ebf5592b-9a0e-4ea4-8fa7-b7d2eb14639e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=304.594µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=c0d34ec3-f7c3-4639-8160-d6e32d09ff42 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=180.132µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=0e2daae8-eec7-4e5e-a222-2f11927517a0 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=725 latency=168.339µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=b8dc3150-f39e-4d7b-a0af-38ffa5a8936d method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=259 latency=110.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestPaginatesMessagesAcrossMultiplePages (0.77s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestConcurrentMessagesAreUnique +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:44 request_id=a35f6c93-191f-4fe2-8640-6390ceab09fc method=POST path=/api/v1/users status=201 response_bytes=153 latency=372.343576ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=4dd49805-f2ef-4166-b188-ebbbc1924d9a method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=371.622652ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=745e9e01-1f34-4e95-81fe-6a3ae7cf9ea4 method=POST path=/api/v1/users status=201 response_bytes=153 latency=394.884355ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:45 request_id=95ac49aa-2560-42da-90c1-902d76341ccd method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=358.911214ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:46 request_id=249bef9d-e9fb-462a-99c3-1946037d1d0c method=POST path=/api/v1/users status=201 response_bytes=153 latency=374.597959ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:46 request_id=ef3349aa-4db5-4ab6-998e-50fa2f3262bb method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=364.93841ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=3007dbff-cdb7-42b9-ac50-77dc7c2ebc4f method=POST path=/api/v1/users status=201 response_bytes=153 latency=396.618954ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=01f269ef-e3c5-4240-9070-37312feb0e6b method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=372.262695ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:47 request_id=ab7c6c66-e836-4704-a516-3c4b51322e74 method=POST path=/api/v1/users status=201 response_bytes=153 latency=358.500208ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:48 request_id=b5f8edd3-d534-401f-8e68-fac601795f3f method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=369.620285ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:48 request_id=1362a848-874b-4b9b-b7fe-735ae74cc76d method=POST path=/api/v1/users status=201 response_bytes=153 latency=401.141119ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=953f6f2b-9fd3-49c5-b660-94a56615c24c method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=376.048273ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=56532a32-ca19-4238-b539-486b339d0bbb method=POST path=/api/v1/users status=201 response_bytes=153 latency=365.0437ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:49 request_id=89c7cf01-2523-4ae6-9a85-dd1a76a61039 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=362.393272ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=0af98040-fde9-4694-a646-c60f794e097f method=POST path=/api/v1/users status=201 response_bytes=153 latency=360.604883ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=92c16b19-7579-47aa-9fb0-a3595b644fc7 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=373.615906ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:50 request_id=460293e1-83ce-4a90-9255-cea43854fe3f method=POST path=/api/v1/users status=201 response_bytes=153 latency=366.494027ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f564332d-7da7-40c0-9842-20cf89412495 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=375.265646ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=62b8365d-f837-4354-8383-1456f1b72154 method=POST path=/api/v1/users status=201 response_bytes=153 latency=358.727985ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a2a37a46-5e62-41e6-bfa3-2bce4c2e3ad8 method=POST path=/api/v1/auth/sessions status=201 response_bytes=281 latency=395.19545ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=5267f89c-aaf1-47b4-8c6a-9714ca727915 method=POST path=/api/v1/chats status=201 response_bytes=514 latency=188.799µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=c5b24f54-0511-46a1-82e5-0061ec77cb32 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.924µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f72ec60f-87ae-4bf6-b996-ce192a118119 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=159.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8afe7b58-5dce-42a1-adf2-1145ac360d45 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=148.63µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7b09b2e8-26b4-42af-bee4-016a8e269a6b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=81276d1e-5a76-4f9b-a7aa-f9857012dc26 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=260.717µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8904e3d2-be95-47dc-a2fa-d3eaf50a3578 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.714µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a03ae327-db89-4091-b3af-578a51335de2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=216.716µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a498db7b-49fb-4935-bb51-891c908fa4f8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=139.089µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=2c981e76-6d36-494c-89ba-3336d357095d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=140.255µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=b9b68e40-c117-4bc7-8f41-b16dd197571a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7223bf26-18da-419a-a853-3b50f5f04a4b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=132.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=16cc739c-a6a1-4c4c-a0bb-ec393bab1534 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.423µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0aef904a-3e95-41de-b5b4-0bff73ed6722 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.297µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=25534c61-60fe-4374-bfda-db1354ea6012 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.714µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=53c00161-1317-49e3-84db-eb7c99300efa method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=1adf4594-5a16-4773-bff0-1b097ab75276 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=133.63µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=78902daf-2bbe-4307-a4d3-e6e9042a6291 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.09µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=7d53cfa6-047a-48c8-8fef-59024e131a96 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=185.173µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=b2942d4f-3e6a-4c91-8dbf-a32c6af07f43 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=206.424µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=5760c94c-8ecd-4067-832b-11909584e440 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.593µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=2b89e564-7bdd-4d5f-9e78-78445d1cbf70 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f8fe6364-dab9-4122-9364-cc8a4e8e3d75 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.757µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=3af65a8b-625c-4fc7-ac62-bfbc353ac7a6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.214µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=755c991e-1d9e-4279-b187-d607b13823ff method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=159.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=c36f80e6-63d2-4dc5-a2ee-01e6ab546ef3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=245.509µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e03d69f4-1738-49f5-b579-cdf4a632cc45 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=153.131µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=05255c0a-5b54-4b53-b714-08f15260687e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=180.423µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=70f18e4c-0cdf-40a9-9e45-048f73171b1d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=126.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ed2a3480-d27e-44eb-bc85-bd088d0cd3a3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=192.631µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=51c5b7ad-f0a6-4127-aca2-1d300b973d21 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=214.091µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e8718af1-4fbd-4b3b-8189-9c066828773c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=151.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=cd61dc79-f672-45c5-91a5-797ac5bd8680 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=170.214µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e9226bfb-5572-4513-9a97-0e90e532cec1 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.838µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8286f6ba-c4cc-4eb9-8b10-da07785cf32a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=164.173µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=d85b2fd6-05bf-4f9b-818a-09ca9ae2df48 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=142.005µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0ba6f4c9-0162-48d9-b610-bf28b56d39cd method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=144.339µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=296d49e0-2b8b-4493-9835-bee0c4f23261 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=243.426µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9cf7e296-e25f-4d63-a1db-de1c262e2b8c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=208.632µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=8fa7d17d-40d0-4dc6-a4bd-b6c514bceb4a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=133.171µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9e1b6e9b-7487-446f-b948-175f7d88e714 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=163.506µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=3cf21baa-455a-4dac-b8c2-732150d32af8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=323.762µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=688c501c-30c9-46bc-ba4d-9d6a9202200d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=572.521µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=65eefba1-b010-405b-b534-8e30d577d755 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=503.852µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=988807ed-875a-4d13-be2c-eb083546e583 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=193.299µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=0ae6f088-d7cb-4d57-a3db-bedb1d222bc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=129.088µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=038588a3-1ed6-4c91-bb8c-de5e55113175 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=320.887µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ff151126-07e7-4100-a2af-e9765d071ca2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=762.652µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=4cf0b177-670d-41b1-aa74-dafd209611e3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.549µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f893e111-2ccc-49a7-a34e-38c8248d5f3b method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.422µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ece5e0f5-e795-4d1e-8bfe-2722feed55f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.047µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=45f9158b-a65f-4527-a741-2a8b3af800ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.089µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=ddc311c8-9d3f-4977-a159-a4d1daff9bf9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=166.298µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=a0ab6cc9-4a7d-471e-9b65-932e3089cbc7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.84µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=76e8f478-d9dc-4443-9bb8-310b8e9dd45a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.51µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=96cb53f0-70bd-45aa-b735-6b5003310461 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=353.721µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=f1dda2fb-4016-440f-9fb6-c95a024038e6 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=169.048µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=e553ec3c-0ab2-4007-8363-48f94f36f000 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=141.796µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=78043ec3-1ec7-4c62-970c-950382757345 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=187.298µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=6222f780-9cdf-4d85-b99b-2897e61cbc43 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=808.862µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=73702805-906e-46ed-b38a-0206be2646dc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=695.567µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=1186e970-9cb0-4db8-a44e-6d823c940ba7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=410.557µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=98aaedba-db19-4504-a633-8a057f94d54e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=311.22µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=92a3bfc9-1577-4a57-9bdd-351208fd7d93 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.151249ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=015c2bd0-266c-478a-99c8-516c482aeebe method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=266.801µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=664b2b6c-39ae-46d5-bed9-72caa9acb7f2 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=249.759µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=286c54ea-04d6-4af2-b00e-8ab00720b2ac method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=273.885µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=89ab422c-a181-43b7-8e4c-cbcdfe3c18b0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.459927ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=9bbdd304-3f76-4f00-85bb-3c964aefb553 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=334.178µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:51 request_id=52dcf62b-df81-4c8f-8b60-c54965aaa140 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.887567ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=68f8ee3e-6eec-4f40-9428-dbeff99fd58a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=1.955945ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=efa4cdeb-c02b-4332-a745-9ba14b490f76 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.642094ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=73f2e45a-8d59-4540-b193-98f1a4dca76a method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=486.101µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=c74d1c30-bde3-46d2-9b37-566419bf5376 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=507.602µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=65789068-cb57-4684-82d8-e201be139913 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=2.130868ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=fcffe674-7c32-4ee1-b34f-3841ac041030 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=191.966µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5ea31e02-1c46-4305-ad02-1fa2217b9738 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=4.155481ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=b281c62b-cbeb-4e81-9c96-40757c5a88bf method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=150.547µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5aceee05-4b98-45ca-974a-1e919fdde991 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=131.963µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=f82374a1-7d40-403d-aa70-5de8f913d2cc method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=5a725d61-e66a-445b-9b40-57ba6183a195 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=207.841µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8ab66e91-8556-454b-913e-7b2c130dfd8e method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=152.631µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=423f02cb-0169-42fd-a1ed-5b6952cb262d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=132.88µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=7da7d9d5-13e8-4495-9ec8-18ef6966bd41 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=160.839µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=870557ee-8823-4a7a-b78e-a440a1d8a0f7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.588µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=52f91333-d9c9-4365-9453-cdfcb0c3f5a7 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=146.172µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=af8b17fd-7edc-4208-a40d-84ca21e160f3 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=167.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=d4c625e7-5ebf-4eea-8d60-45af463edf4c method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=165.798µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=083768b9-8f2a-4681-bcd4-5ae52228e0bb method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=186.59µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=46101c43-7641-481a-869d-3543f25ef54d method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=436.099µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=ba6ff758-a13f-4580-8ba2-da63a2351ca8 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=162.255µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e6379b05-1acd-4b5a-b5a6-99e7bccc5f01 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=134.672µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=d9fb6747-2b9e-41cf-b53c-1a7750fc91f0 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=202.966µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e5e5ec36-88eb-412f-8c2c-3f82c87c4590 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=171.006µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8c2d9c31-df32-49d8-9c38-780f4fab1746 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=184.465µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=95fd52ea-b8d8-45df-a68a-ceba9d27b133 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=135.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=ae9ee763-096d-480c-b24b-cd349cb18978 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=384.431µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=088337f5-bb46-4911-80fd-fa855dc98200 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.463µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=2aa7e674-07c8-4282-8798-fcf641d08133 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=138.797µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=e0339717-77bb-4bd7-a321-34eaf1984fb9 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=163.923µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=74331de8-09a1-4057-8604-bd04d68465f4 method=POST path=/api/v1/chats/:chat_id/messages status=201 response_bytes=239 latency=145.922µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=da1793af-1f78-46eb-8751-52961c9bd4af method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=865.156µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=41cc821a-71d8-48f3-a2ec-618e8d6370b3 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=796.404µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=58db51d8-2b41-4a61-9f48-e28a79b88470 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=728.859µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=6b05dcf0-ee67-43cf-96d4-ef406c94761e method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=616.272µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=8bbd35fc-09c2-4ede-af22-3fb7147d5620 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=4332 latency=611.938µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:52 request_id=10bfa4c2-537d-442a-8e5b-0da397c81cc0 method=GET path=/api/v1/chats/:chat_id/messages status=200 response_bytes=3616 latency=446.391µs +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestConcurrentMessagesAreUnique (7.52s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/app/chat 12.755s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestSpoolOriginalAndFullValidation +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestSpoolOriginalAndFullValidation (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestSpoolLimitsBeforeDecode +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestSpoolLimitsBeforeDecode (0.02s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestFilenameAndCancelledUpload +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestFilenameAndCancelledUpload (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestReconcileRejectsUnsafeGrace +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestReconcileRejectsUnsafeGrace (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestValid16BitWidePNG +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestValid16BitWidePNG (0.08s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment 1.160s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/config [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestUnicodeAndSalt +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestUnicodeAndSalt (2.23s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password 3.266s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestS3WireFixture +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestS3WireFixture (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:84: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (24.32ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.98ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (4.14ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (1.77ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:85: goose up: 2026/10/28 19:17:59 goose: no migrations to run. current version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/registration_race +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:121: 12 concurrent INSERTs: 1 success, 11 unique conflicts +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/rollback +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:139: failed second member INSERT left 0 chats and 0 members +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/lab4_attachments_and_reconciliation +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | lab4_test.go:248: ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | lab4_test.go:316: dry-run={Apply:false GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:94 ListRequestsEstimate:1 StatRequests:3 DBQueries:12 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:0 MarkedFailed:0 DurationMS:7 Actions:[{Kind:orphan Key:original/9da5bf1b-bbc2-4c8e-8816-d473c908db4a Applied:false} {Kind:missing Key:original/65f5dff4-bc2f-469b-8b6b-6bc1275f8d6a Applied:false}]} apply={Apply:true GraceSeconds:90 ObjectsScanned:4 MetadataScanned:2 BytesScanned:94 ListRequestsEstimate:1 StatRequests:3 DBQueries:13 Orphans:1 Missing:1 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:1 Deleted:1 MarkedFailed:1 DurationMS:5 Actions:[{Kind:orphan Key:original/9da5bf1b-bbc2-4c8e-8816-d473c908db4a Applied:true} {Kind:missing Key:original/65f5dff4-bc2f-469b-8b6b-6bc1275f8d6a Applied:true}]} repeat={Apply:true GraceSeconds:90 ObjectsScanned:3 MetadataScanned:2 BytesScanned:88 ListRequestsEstimate:1 StatRequests:1 DBQueries:7 Orphans:0 Missing:0 FreshSkipped:1 LockedSkipped:1 ReferencedMissing:0 Deleted:0 MarkedFailed:0 DurationMS:2 Actions:[]}; fresh, locked and referenced objects retained +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/keyset_and_explain +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:193: EXPLAIN on 10,000 messages, cursor near the end (9,000): +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Limit (cost=0.29..18.05 rows=51 width=142) (actual time=0.011..0.032 rows=51 loops=1) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: -> Index Scan using messages_history_idx on messages m (cost=0.29..348.68 rows=1000 width=142) (actual time=0.011..0.028 rows=51 loops=1) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Index Cond: ((chat_id = '3945a437-ac3b-4af6-a505-e156a979dd08'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.9+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Buffers: shared hit=10 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Planning Time: 0.031 ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:199: Execution Time: 0.061 ms +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:204: keyset traversal returned 10,000 distinct messages, including equal timestamps +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/idempotency_race +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:254: 16 concurrent PostgreSQL requests: one effect, identical replay, changed body conflicts +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestIntegration/goose_legacy_and_down +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:268: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (1.62ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.13ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (583.4µs) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (2.74ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:269: goose up: 2026/10/28 19:17:59 goose: no migrations to run. current version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:276: adopted legacy schema: all existing messages are unchanged +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:277: goose reset: 2026/10/28 19:17:59 OK 004_message_attachments.sql (5.92ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (3.96ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (2.73ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 001_initial.sql (9.78ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:282: goose up: 2026/10/28 19:17:59 OK 001_initial.sql (10.62ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 002_idempotency.sql (1.69ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 003_attachments.sql (3.22ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 OK 004_message_attachments.sql (1.21ms) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | 2026/10/28 19:17:59 goose: successfully migrated database to version: 4 +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | storage_test.go:286: down and up succeeded in isolated schema +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration (0.64s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/registration_race (0.04s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/rollback (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/lab4_attachments_and_reconciliation (0.04s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/keyset_and_explain (0.08s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/idempotency_race (0.01s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestIntegration/goose_legacy_and_down (0.12s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres 1.675s +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ? gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/session [no test files] +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | === RUN TestPasswordContract +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | --- PASS: TestPasswordContract (0.00s) +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | PASS +mtusi-lab4_tests.0.lks3m3xxmw5s@colima | ok gitflic.ru/akselbant/mtusi2026lab1-fork/internal/usecase/chat 1.011s +mtusi-lab4_tests diff --git a/evidence/lab4-persistence.txt b/evidence/lab4-persistence.txt new file mode 100644 index 0000000..4ba6d9c --- /dev/null +++ b/evidence/lab4-persistence.txt @@ -0,0 +1,3 @@ +.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-reconciliation.json b/evidence/lab4-reconciliation.json new file mode 100644 index 0000000..0a57b41 --- /dev/null +++ b/evidence/lab4-reconciliation.json @@ -0,0 +1,1226 @@ +{ + "hypothesis": "fixed grace plus per-object transaction lock permits repeatable repair without deleting fresh or linked data", + "grace_seconds": 90, + "faults": { + "s3_put_db_insert_failure": { + "upload_status": 503, + "objects_without_metadata": 1 + }, + "minio_unavailable": { + "exit_nonzero": true, + "summary": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 0, + "metadata_scanned": 0, + "bytes_scanned": 0, + "list_requests_estimate": 0, + "stat_requests": 0, + "db_queries": 0, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 201, + "actions": [] + } + }, + "postgres_unavailable": { + "exit_nonzero": true, + "summary": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 1, + "metadata_scanned": 0, + "bytes_scanned": 837, + "list_requests_estimate": 1, + "stat_requests": 0, + "db_queries": 0, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 48, + "actions": [] + } + } + }, + "cost": [ + { + "phase": "fresh", + "synthetic_objects": 100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 130, + "metadata_scanned": 30, + "bytes_scanned": 31825703, + "list_requests_estimate": 1, + "stat_requests": 7, + "db_queries": 30, + "orphans": 0, + "missing": 0, + "fresh_skipped": 146, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 27, + "actions": [] + }, + { + "phase": "fresh", + "synthetic_objects": 1100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 1130, + "metadata_scanned": 30, + "bytes_scanned": 31826703, + "list_requests_estimate": 2, + "stat_requests": 7, + "db_queries": 30, + "orphans": 0, + "missing": 0, + "fresh_skipped": 1146, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 64, + "actions": [] + }, + { + "phase": "aged", + "synthetic_objects": 1100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 1130, + "metadata_scanned": 30, + "bytes_scanned": 31826703, + "list_requests_estimate": 2, + "stat_requests": 1131, + "db_queries": 2323, + "orphans": 1101, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 999, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/1", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/10", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/100", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/1000", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/101", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/102", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/103", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/104", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/105", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/106", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/107", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/108", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/109", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/11", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/110", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/111", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/112", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/113", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/114", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/115", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/116", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/117", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/118", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/119", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/12", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/120", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/121", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/122", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/123", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/124", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/125", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/126", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/127", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/128", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/129", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/13", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/130", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/131", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/132", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/133", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/134", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/135", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/136", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/137", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/138", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/139", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/14", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/140", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/141", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/142", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/143", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/144", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/145", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/146", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/147", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/148", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/149", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/15", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/150", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/151", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/152", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/153", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/154", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/155", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/156", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/157", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/158", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/159", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/16", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/160", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/161", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/162", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/163", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/164", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/165", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/166", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/167", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/168", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/169", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/17", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/170", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/171", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/172", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/173", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/174", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/175", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/176", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/177", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/178", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/179", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/18", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/180", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/181", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/182", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/183", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/184", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/185", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/186", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/extra/187", + "applied": false + } + ] + }, + { + "phase": "aged", + "synthetic_objects": 100, + "apply": false, + "grace_seconds": 90, + "objects_scanned": 130, + "metadata_scanned": 30, + "bytes_scanned": 31825703, + "list_requests_estimate": 1, + "stat_requests": 131, + "db_queries": 323, + "orphans": 101, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 129, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/1", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/10", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/100", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/11", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/12", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/13", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/14", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/15", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/16", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/17", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/18", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/19", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/2", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/20", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/21", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/22", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/23", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/24", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/25", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/26", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/27", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/28", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/29", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/3", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/30", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/31", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/32", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/33", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/34", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/35", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/36", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/37", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/38", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/39", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/4", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/40", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/41", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/42", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/43", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/44", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/45", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/46", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/47", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/48", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/49", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/5", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/50", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/51", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/52", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/53", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/54", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/55", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/56", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/57", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/58", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/59", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/6", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/60", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/61", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/62", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/63", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/64", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/65", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/66", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/67", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/68", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/69", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/7", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/70", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/71", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/72", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/73", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/74", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/75", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/76", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/77", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/78", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/79", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/8", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/80", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/81", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/82", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/83", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/84", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/85", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/86", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/87", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/88", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/89", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/9", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/90", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/91", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/92", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/93", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/94", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/95", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/96", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/97", + "applied": false + }, + { + "kind": "orphan", + "key": "original/cost-7c25c21102134be3935726555c66ed93/small/98", + "applied": false + } + ] + } + ], + "fresh_orphan_preserved": true, + "fresh_metadata_preserved": true, + "locked_candidate_preserved": true, + "dry_run": { + "apply": false, + "grace_seconds": 90, + "objects_scanned": 30, + "metadata_scanned": 30, + "bytes_scanned": 31825603, + "list_requests_estimate": 1, + "stat_requests": 31, + "db_queries": 123, + "orphans": 1, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 54, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": false + }, + { + "kind": "missing", + "key": "original/780437df-ef67-4bc1-82e1-209c619b6a3a", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 30, + "metadata_scanned": 30, + "bytes_scanned": 31825603, + "list_requests_estimate": 1, + "stat_requests": 31, + "db_queries": 124, + "orphans": 1, + "missing": 1, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 1, + "deleted": 1, + "marked_failed": 1, + "duration_ms": 54, + "actions": [ + { + "kind": "orphan", + "key": "original/337521a8-44be-4941-a1b1-0eec2ada2b3c", + "applied": true + }, + { + "kind": "missing", + "key": "original/780437df-ef67-4bc1-82e1-209c619b6a3a", + "applied": true + } + ] + }, + "repeat_apply": { + "apply": true, + "grace_seconds": 90, + "objects_scanned": 29, + "metadata_scanned": 30, + "bytes_scanned": 31824766, + "list_requests_estimate": 1, + "stat_requests": 29, + "db_queries": 118, + "orphans": 0, + "missing": 0, + "fresh_skipped": 0, + "locked_skipped": 0, + "referenced_missing": 0, + "deleted": 0, + "marked_failed": 0, + "duration_ms": 40, + "actions": [] + }, + "live_linked_preserved": true, + "missing_linked_reference_preserved": true +} diff --git a/evidence/lab4-reconciliation.txt b/evidence/lab4-reconciliation.txt new file mode 100644 index 0000000..7a1e6fb --- /dev/null +++ b/evidence/lab4-reconciliation.txt @@ -0,0 +1,7 @@ +.venv/bin/python tests/own/reconcile.py +Fresh scan cost: {"objects_scanned": 130, "metadata_scanned": 30, "list_requests_estimate": 1, "stat_requests": 7, "db_queries": 30, "duration_ms": 27} +Fresh scan cost: {"objects_scanned": 1130, "metadata_scanned": 30, "list_requests_estimate": 2, "stat_requests": 7, "db_queries": 30, "duration_ms": 64} +Waiting92s for actual S3 timestamps: measuring aged-object locking, DB and HEAD cost next. +Aged scan cost: {"objects_scanned": 1130, "metadata_scanned": 30, "list_requests_estimate": 2, "stat_requests": 1131, "db_queries": 2323, "duration_ms": 999} +Aged scan cost: {"objects_scanned": 130, "metadata_scanned": 30, "list_requests_estimate": 1, "stat_requests": 131, "db_queries": 323, "duration_ms": 129} +PASS: real two-store faults, grace, lock, dry-run/apply/repeat, linked data preserved and paginated scan cost. diff --git a/evidence/lab4-restart-verification.json b/evidence/lab4-restart-verification.json new file mode 100644 index 0000000..9702392 --- /dev/null +++ b/evidence/lab4-restart-verification.json @@ -0,0 +1,54 @@ +{ + "api": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "05qp70hqicx1m3uten0t88c9p", + "1diq18xi0jmwpzqzhhq39hjad" + ], + "after": [ + "g2s34c3zuztbxehcxyypsyrxy", + "rz3qizaxp0s2bguzbs57me0op" + ] + }, + "lb": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "psq3iyz4kraq9u3n2mtkyfijf" + ], + "after": [ + "oaiciqwjik1afkel0hdgwvhwy" + ] + }, + "postgres": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "7xbetpe47jvp1uuu2di2uk7vy" + ], + "after": [ + "i3scdtmsi49zwytqye6hc3k15" + ] + }, + "redis": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "x8ejqk47qyz4rgtgygq48wos7" + ], + "after": [ + "tqq4m5ygv0nohjefvbdprapzz" + ] + }, + "minio": { + "all_original_tasks_replaced": true, + "mounts_unchanged": true, + "before": [ + "ww19llqoe6yjvteqx3by0i4r1" + ], + "after": [ + "lfmr13mco26ik0a37xk2ntene" + ] + } +} diff --git a/evidence/lab4-scaling.txt b/evidence/lab4-scaling.txt new file mode 100644 index 0000000..9dad541 --- /dev/null +++ b/evidence/lab4-scaling.txt @@ -0,0 +1,6 @@ +.venv/bin/python tests/own/swarm_grade4.py +PASS: shared-key race on two replicas; replay/conflict/scope/validation/access/logout +{"action": "stop-one", "target": "rz3qizaxp0s2bguzbs57me0op", "duration_seconds": 15.458, "successful_writes": 1078, "failed_attempts": 7, "successful_rps": 69.739, "first_recovery_seconds": 1.349, "max_success_gap_seconds": 1.055, "instances_before": ["rz3qizaxp0s2bguzbs57me0op", "xcgz2hubz1xrzmrkoyrgwq0tq"], "instances_after": ["j7awgfnedlt9ne1esywrappxx", "xcgz2hubz1xrzmrkoyrgwq0tq"], "confirmed_messages_checked": 1229, "duplicate_bodies": 0} +{"action": "stop-one", "target": "xcgz2hubz1xrzmrkoyrgwq0tq", "duration_seconds": 15.467, "successful_writes": 1091, "failed_attempts": 7, "successful_rps": 70.537, "first_recovery_seconds": 1.384, "max_success_gap_seconds": 1.057, "instances_before": ["j7awgfnedlt9ne1esywrappxx", "xcgz2hubz1xrzmrkoyrgwq0tq"], "instances_after": ["5vy560hk6k0q3eicqodd9dp2z", "j7awgfnedlt9ne1esywrappxx"], "confirmed_messages_checked": 1251, "duplicate_bodies": 0} +{"action": "update", "target": null, "duration_seconds": 44.156, "successful_writes": 3362, "failed_attempts": 0, "successful_rps": 76.139, "first_recovery_seconds": null, "max_success_gap_seconds": 0.152, "instances_before": ["5vy560hk6k0q3eicqodd9dp2z", "j7awgfnedlt9ne1esywrappxx"], "instances_after": ["j201rqxerkxdq0vbcw3armc3m", "uqyjmps3u9fyl976lpj944pc5"], "confirmed_messages_checked": 3519, "duplicate_bodies": 0} +PASS: each original replica stopped and replaced; rolling update under load; confirmations and sessions preserved diff --git a/evidence/lab4-smoke.txt b/evidence/lab4-smoke.txt new file mode 100644 index 0000000..607ff81 --- /dev/null +++ b/evidence/lab4-smoke.txt @@ -0,0 +1,2 @@ +.venv/bin/python tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab4-status-after-restart.json b/evidence/lab4-status-after-restart.json new file mode 100644 index 0000000..be72be0 --- /dev/null +++ b/evidence/lab4-status-after-restart.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "oaiciqwjik1afkel0hdgwvhwy", + "state": "running", + "container": "ace6b5a9ab2ac3a291bca557e7eb89ba6354ed586625332b99dc3d1cdfaab6b5" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "g2s34c3zuztbxehcxyypsyrxy", + "state": "running", + "container": "303dc53972fde0fe71990f644cb1b56658441a4d1cb6410ffc351ccceb0cd3a7" + }, + { + "id": "rz3qizaxp0s2bguzbs57me0op", + "state": "running", + "container": "f69bf37c2237e13f08bf4aa397936d4fed84a17395410f0af469b2cbc3e41699" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "i3scdtmsi49zwytqye6hc3k15", + "state": "running", + "container": "1622ddce8784f7dc1092ba808f05583898e08ba57a5c390d227265e4243744dc" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "tqq4m5ygv0nohjefvbdprapzz", + "state": "running", + "container": "dfe7c35a3887061560c608484e33ce1036222d8f683a5729599fb0a4e33065f3" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "lfmr13mco26ik0a37xk2ntene", + "state": "running", + "container": "8110ff5d0c7a776005653182ecfd927fe59a8b38b02b4645e1bd03fa1c6fdeff" + } + ] + } + } +} diff --git a/evidence/lab4-status-before-restart.json b/evidence/lab4-status-before-restart.json new file mode 100644 index 0000000..e2dd380 --- /dev/null +++ b/evidence/lab4-status-before-restart.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "psq3iyz4kraq9u3n2mtkyfijf", + "state": "running", + "container": "c9246107dbeb5aa78d3574865ddb2e9ac2d117c87830ca7be5214bdc8e8768b4" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "05qp70hqicx1m3uten0t88c9p", + "state": "running", + "container": "0beaa8438aa0f61c069729d0d3e83220df07e49288451658d3c92f698c1b7fbc" + }, + { + "id": "1diq18xi0jmwpzqzhhq39hjad", + "state": "running", + "container": "bc1d5078efd2145c84e341ae457360f490f32539d7e4f6d5155378c918da69c8" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "7xbetpe47jvp1uuu2di2uk7vy", + "state": "running", + "container": "7a24978349b4630091ba38d365fcd74b20c60bd9a27fc15d5cb967e3874e09de" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "x8ejqk47qyz4rgtgygq48wos7", + "state": "running", + "container": "33a57e377891ce0c82807b12757cf0103ef9461502d1a64efdeec506744d9c27" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "ww19llqoe6yjvteqx3by0i4r1", + "state": "running", + "container": "d71ce1a062fe9ba9060f5266a9f00d89a34583f133a609769bb2510deaa52470" + } + ] + } + } +} diff --git a/evidence/lab4-status-final.json b/evidence/lab4-status-final.json new file mode 100644 index 0000000..73fc9e0 --- /dev/null +++ b/evidence/lab4-status-final.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "oaiciqwjik1afkel0hdgwvhwy", + "state": "running", + "container": "ace6b5a9ab2ac3a291bca557e7eb89ba6354ed586625332b99dc3d1cdfaab6b5" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "uqyjmps3u9fyl976lpj944pc5", + "state": "running", + "container": "2f98c5810a5f8557476af364186cb66e02c46003fece8827e200313d27f2fd74" + }, + { + "id": "j201rqxerkxdq0vbcw3armc3m", + "state": "running", + "container": "68730a58baf57c19b9919e752e34b7ad247ff29d94918f28fb7ede200c1a95f1" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "i3scdtmsi49zwytqye6hc3k15", + "state": "running", + "container": "1622ddce8784f7dc1092ba808f05583898e08ba57a5c390d227265e4243744dc" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "tqq4m5ygv0nohjefvbdprapzz", + "state": "running", + "container": "dfe7c35a3887061560c608484e33ce1036222d8f683a5729599fb0a4e33065f3" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "lfmr13mco26ik0a37xk2ntene", + "state": "running", + "container": "8110ff5d0c7a776005653182ecfd927fe59a8b38b02b4645e1bd03fa1c6fdeff" + } + ] + } + } +} diff --git a/evidence/lab4-status-initial.json b/evidence/lab4-status-initial.json new file mode 100644 index 0000000..8098235 --- /dev/null +++ b/evidence/lab4-status-initial.json @@ -0,0 +1,146 @@ +{ + "stack": "mtusi-lab4", + "docker": "29.5.2", + "services": { + "lb": { + "image": "haproxy:3.2.6-alpine", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [ + { + "Protocol": "tcp", + "TargetPort": 8080, + "PublishedPort": 8080, + "PublishMode": "host" + } + ], + "secret_names": [], + "mounts": [], + "tasks": [ + { + "id": "psq3iyz4kraq9u3n2mtkyfijf", + "state": "running", + "container": "c9246107dbeb5aa78d3574865ddb2e9ac2d117c87830ca7be5214bdc8e8768b4" + } + ] + }, + "api": { + "image": "mtusi-messenger:lab4", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 1610612736 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_cursor_key", + "mtusi-lab4_postgres_password", + "mtusi-lab4_s3_access_key", + "mtusi-lab4_s3_secret_key" + ], + "mounts": [], + "tasks": [ + { + "id": "0gwv6ha33wg3dao0mqxl9gu0n", + "state": "running", + "container": "f37c59b0556d4bc5b3f5c0432e8c58da4888ca5286635954131ea37c0714ead1" + }, + { + "id": "rbtm0mbmnbzr2oqcz8icho3qr", + "state": "running", + "container": "17d5f64c6fdd70ac3cdc27e655308909d4670ca0a7b743384ac7fe34e172d428" + } + ] + }, + "postgres": { + "image": "postgres:17.6-alpine3.22", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_postgres_password" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_postgres_data", + "Target": "/var/lib/postgresql/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "uy9m8wn36msy9qzttodwobkez", + "state": "running", + "container": "41c7c4bb6a37b443982f168a8e8494847581c2e30d1d6f5bf817284aebaf96b9" + } + ] + }, + "redis": { + "image": "redis:7.4.5-alpine3.21", + "limits": { + "NanoCPUs": 500000000, + "MemoryBytes": 134217728 + }, + "published_ports": [], + "secret_names": [], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_redis_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "x8ejqk47qyz4rgtgygq48wos7", + "state": "running", + "container": "33a57e377891ce0c82807b12757cf0103ef9461502d1a64efdeec506744d9c27" + } + ] + }, + "minio": { + "image": "mtusi-minio:RELEASE.2025-09-07T16-13-09Z", + "limits": { + "NanoCPUs": 1000000000, + "MemoryBytes": 536870912 + }, + "published_ports": [], + "secret_names": [ + "mtusi-lab4_s3_root_access_key", + "mtusi-lab4_s3_root_secret_key" + ], + "mounts": [ + { + "Type": "volume", + "Source": "mtusi-lab4_s3_data", + "Target": "/data", + "VolumeOptions": { + "Labels": { + "com.docker.stack.namespace": "mtusi-lab4" + } + } + } + ], + "tasks": [ + { + "id": "jafk96quou17emokdyfk1qi4t", + "state": "running", + "container": "a04bf799191faea4386533680968586c90d67adeef8f37166824aa4723f8ac03" + } + ] + } + } +} diff --git a/evidence/lab4-update.txt b/evidence/lab4-update.txt new file mode 100644 index 0000000..2c2ec68 --- /dev/null +++ b/evidence/lab4-update.txt @@ -0,0 +1,3 @@ +.venv/bin/python scripts/swarm.py update +mtusi-lab4_api +PASS: rolling update completed diff --git a/evidence/lab4-upload-memory.json b/evidence/lab4-upload-memory.json new file mode 100644 index 0000000..5b2026b --- /dev/null +++ b/evidence/lab4-upload-memory.json @@ -0,0 +1,105 @@ +{ + "concurrency": 10, + "warmup_uploads": 1, + "memory_method": "cgroup v2 memory.current sampled every >=100ms; values include file cache", + "runs": [ + { + "width": 512, + "height": 512, + "size_bytes": 787252, + "sha256": "1a09c45b4a706ff8009c1adf0e9f10e428d8059d335e074cf4ad456680d4b473", + "requests": 10, + "successful": 10, + "errors": 0, + "duration_seconds": 0.325, + "successful_rps": 30.816, + "latency_ms": { + "0.5": 120.287, + "0.95": 172.127, + "0.99": 172.127 + }, + "baseline_bytes": { + "a43c3f7e6e92": 45240320, + "007b52fbba64": 45174784 + }, + "sampled_peak_bytes": { + "a43c3f7e6e92": 47005696, + "007b52fbba64": 46907392 + }, + "kernel_lifetime_peak_before_bytes": { + "a43c3f7e6e92": 46804992, + "007b52fbba64": 46718976 + }, + "kernel_lifetime_peak_after_bytes": { + "a43c3f7e6e92": 48623616, + "007b52fbba64": 48693248 + }, + "sampled_peak_sum_bytes": 93913088, + "samples": [ + { + "seconds": 0.045, + "bytes": { + "a43c3f7e6e92": 47005696, + "007b52fbba64": 46907392 + } + }, + { + "seconds": 0.325, + "bytes": { + "a43c3f7e6e92": 46116864, + "007b52fbba64": 46080000 + } + } + ] + }, + { + "width": 1024, + "height": 768, + "size_bytes": 2360847, + "sha256": "ad94bc1d209135dea4d3caa0712cee92cfaa97ba3ccdd5797aff6664fcce4a2a", + "requests": 10, + "successful": 10, + "errors": 0, + "duration_seconds": 0.401, + "successful_rps": 24.957, + "latency_ms": { + "0.5": 258.573, + "0.95": 324.694, + "0.99": 324.694 + }, + "baseline_bytes": { + "a43c3f7e6e92": 46067712, + "007b52fbba64": 46063616 + }, + "sampled_peak_bytes": { + "a43c3f7e6e92": 41107456, + "007b52fbba64": 13651968 + }, + "kernel_lifetime_peak_before_bytes": { + "a43c3f7e6e92": 48623616, + "007b52fbba64": 48693248 + }, + "kernel_lifetime_peak_after_bytes": { + "a43c3f7e6e92": 49119232, + "007b52fbba64": 49922048 + }, + "sampled_peak_sum_bytes": 54759424, + "samples": [ + { + "seconds": 0.043, + "bytes": { + "a43c3f7e6e92": 41107456, + "007b52fbba64": 13651968 + } + }, + { + "seconds": 0.401, + "bytes": { + "a43c3f7e6e92": 13234176, + "007b52fbba64": 10956800 + } + } + ] + } + ] +} diff --git a/evidence/lab4-upload-memory.txt b/evidence/lab4-upload-memory.txt new file mode 100644 index 0000000..8077fe9 --- /dev/null +++ b/evidence/lab4-upload-memory.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/measure_uploads.py +{"width": 512, "height": 512, "size_bytes": 787252, "sha256": "1a09c45b4a706ff8009c1adf0e9f10e428d8059d335e074cf4ad456680d4b473", "requests": 10, "successful": 10, "errors": 0, "duration_seconds": 0.325, "successful_rps": 30.816, "latency_ms": {"0.5": 120.287, "0.95": 172.127, "0.99": 172.127}, "baseline_bytes": {"a43c3f7e6e92": 45240320, "007b52fbba64": 45174784}, "sampled_peak_bytes": {"a43c3f7e6e92": 47005696, "007b52fbba64": 46907392}, "kernel_lifetime_peak_before_bytes": {"a43c3f7e6e92": 46804992, "007b52fbba64": 46718976}, "kernel_lifetime_peak_after_bytes": {"a43c3f7e6e92": 48623616, "007b52fbba64": 48693248}, "sampled_peak_sum_bytes": 93913088} +{"width": 1024, "height": 768, "size_bytes": 2360847, "sha256": "ad94bc1d209135dea4d3caa0712cee92cfaa97ba3ccdd5797aff6664fcce4a2a", "requests": 10, "successful": 10, "errors": 0, "duration_seconds": 0.401, "successful_rps": 24.957, "latency_ms": {"0.5": 258.573, "0.95": 324.694, "0.99": 324.694}, "baseline_bytes": {"a43c3f7e6e92": 46067712, "007b52fbba64": 46063616}, "sampled_peak_bytes": {"a43c3f7e6e92": 41107456, "007b52fbba64": 13651968}, "kernel_lifetime_peak_before_bytes": {"a43c3f7e6e92": 48623616, "007b52fbba64": 48693248}, "kernel_lifetime_peak_after_bytes": {"a43c3f7e6e92": 49119232, "007b52fbba64": 49922048}, "sampled_peak_sum_bytes": 54759424} +PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved. diff --git a/evidence/lab4-validation.txt b/evidence/lab4-validation.txt new file mode 100644 index 0000000..f973c72 --- /dev/null +++ b/evidence/lab4-validation.txt @@ -0,0 +1,8 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +/Users/dkhakimovig/mtusi2026lab4/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +PASS: goose3.27.1 migration syntax validated. +PASS: mandatory lab4 contract and public checks unchanged. diff --git a/evidence/swarm-grade4.json b/evidence/swarm-grade4.json new file mode 100644 index 0000000..419dfe1 --- /dev/null +++ b/evidence/swarm-grade4.json @@ -0,0 +1,72 @@ +{ + "idempotency": { + "parallel_requests": 16, + "instances": [ + "rz3qizaxp0s2bguzbs57me0op", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "unique_message_ids": 1 + }, + "failovers": [ + { + "action": "stop-one", + "target": "rz3qizaxp0s2bguzbs57me0op", + "duration_seconds": 15.458, + "successful_writes": 1078, + "failed_attempts": 7, + "successful_rps": 69.739, + "first_recovery_seconds": 1.349, + "max_success_gap_seconds": 1.055, + "instances_before": [ + "rz3qizaxp0s2bguzbs57me0op", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "instances_after": [ + "j7awgfnedlt9ne1esywrappxx", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "confirmed_messages_checked": 1229, + "duplicate_bodies": 0 + }, + { + "action": "stop-one", + "target": "xcgz2hubz1xrzmrkoyrgwq0tq", + "duration_seconds": 15.467, + "successful_writes": 1091, + "failed_attempts": 7, + "successful_rps": 70.537, + "first_recovery_seconds": 1.384, + "max_success_gap_seconds": 1.057, + "instances_before": [ + "j7awgfnedlt9ne1esywrappxx", + "xcgz2hubz1xrzmrkoyrgwq0tq" + ], + "instances_after": [ + "5vy560hk6k0q3eicqodd9dp2z", + "j7awgfnedlt9ne1esywrappxx" + ], + "confirmed_messages_checked": 1251, + "duplicate_bodies": 0 + } + ], + "rolling_update": { + "action": "update", + "target": null, + "duration_seconds": 44.156, + "successful_writes": 3362, + "failed_attempts": 0, + "successful_rps": 76.139, + "first_recovery_seconds": null, + "max_success_gap_seconds": 0.152, + "instances_before": [ + "5vy560hk6k0q3eicqodd9dp2z", + "j7awgfnedlt9ne1esywrappxx" + ], + "instances_after": [ + "j201rqxerkxdq0vbcw3armc3m", + "uqyjmps3u9fyl976lpj944pc5" + ], + "confirmed_messages_checked": 3519, + "duplicate_bodies": 0 + } +} diff --git a/go.mod b/go.mod index f5cadee..7e3203b 100644 --- a/go.mod +++ b/go.mod @@ -6,26 +6,37 @@ require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.6 github.com/labstack/echo/v4 v4.13.4 + github.com/minio/minio-go/v7 v7.0.95 github.com/redis/go-redis/v9 v9.12.1 github.com/swaggest/swgui v1.8.9 - golang.org/x/crypto v0.38.0 + golang.org/x/crypto v0.39.0 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vearutop/statigz v1.4.0 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sync v0.14.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index bbb344c..f791326 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -23,6 +29,11 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= @@ -31,10 +42,20 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -42,23 +63,25 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggest/swgui v1.8.9 h1:cxAgIwouPpZPlvX68jY5fpwarzLbkc8/IL6DMj+H460= github.com/swaggest/swgui v1.8.9/go.mod h1:eTJfgwudbyw9xMwqO26vs82ei2u6//JnUAofx2vGB3M= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vearutop/statigz v1.4.0 h1:RQL0KG3j/uyA/PFpHeZ/L6l2ta920/MxlOAIGEOuwmU= github.com/vearutop/statigz v1.4.0/go.mod h1:LYTolBLiz9oJISwiVKnOQoIwhO1LWX1A7OECawGS8XE= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/infra/README.md b/infra/README.md index cb4f6fa..82e0281 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,11 +1,14 @@ # Инфраструктура студента -Из lab3 перенесены HAProxy, Redis AOF и скрипт запуска goose. -Основной стенд использует `stack.yaml`: HAProxy → две API-реплики → PostgreSQL и Redis. -Инструкция запуска находится в [PORTING.md](../PORTING.md). S3 пока не добавлено. +Основной стенд использует `stack.yaml`: HAProxy → две API-реплики → PostgreSQL, Redis AOF и приватный MinIO. +Запуск: `make up`. Остановка с сохранением данных: `make down`. +Версии, секреты, ограничения и проверки описаны в [REPORT.md](../REPORT.md). -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +`s3-bootstrap.sh` идемпотентно создаёт bucket и отдельного API-пользователя по `s3-policy.json`. +`minio-entrypoint.sh` читает root keys из Docker secrets. Консоль слушает loopback контейнера. +`Dockerfile.storage` собирает MinIO и mc из исходников с фиксированными commit и SHA-256. +`haproxy.cfg` проверяет readiness реплик. `redis.conf` включает AOF и запрещает eviction. +`migrate.sh` запускает goose без автоматической загрузки файлов окружения. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Restart hook перезапускает API, HAProxy, PostgreSQL, Redis и MinIO с сохранением volumes. +Failover hook останавливает API-реплику по `TARGET_INSTANCE_ID`; Swarm создаёт новую task с другим ID. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index 3c378e8..6a3fb1f 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -6,8 +6,8 @@ defaults log global mode http timeout connect 1s - timeout client 20s - timeout server 20s + timeout client 55s + timeout server 55s timeout check 1s retries 0 option http-server-close diff --git a/infra/minio-entrypoint.sh b/infra/minio-entrypoint.sh new file mode 100755 index 0000000..ba24993 --- /dev/null +++ b/infra/minio-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_access_key)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_secret_key)" +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100755 index 0000000..4251f8f --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +export MC_CONFIG_DIR=/tmp/mc +mc alias set admin http://minio:9000 "$(cat /run/secrets/s3_root_access_key)" "$(cat /run/secrets/s3_root_secret_key)" >/dev/null +mc ready admin >/dev/null +mc mb --ignore-existing admin/attachments >/dev/null +mc anonymous set none admin/attachments >/dev/null +mc admin policy create admin attachments-api /policy.json >/dev/null +mc admin user add admin "$(cat /run/secrets/s3_access_key)" "$(cat /run/secrets/s3_secret_key)" >/dev/null +mc admin policy attach admin attachments-api --user "$(cat /run/secrets/s3_access_key)" >/dev/null +printf '%s\n' 'PASS: private bucket and restricted API account configured.' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..b9fb3b9 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::attachments"], + "Condition": {"StringLike": {"s3:prefix": ["original/*"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::attachments/original/*"] + } + ] +} diff --git a/internal/app/chat/attachments.go b/internal/app/chat/attachments.go new file mode 100644 index 0000000..f326d70 --- /dev/null +++ b/internal/app/chat/attachments.go @@ -0,0 +1,68 @@ +package chat + +import ( + "context" + "mime" + "net/http" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "github.com/labstack/echo/v4" +) + +func (h *Chat) WithAttachments(service *attachment.Service) *Chat { h.attachments = service; return h } +func (h *Chat) uploadAttachment(c echo.Context) error { + chat, ok, err := h.authorizedChat(c, domain.Chat{ID: c.Param("chat_id")}) + if err != nil || !ok { + return err + } + filename, present, err := oneQueryValue(c, "filename") + if err != nil || !present { + return h.invalid(c, "Требуется один filename") + } + if c.Request().ContentLength > attachment.MaxBytes { + return h.writeError(c, 413, "payload_too_large", "Файл превышает 10 MiB") + } + // Both socket reads and the application operation have a fixed deadline. + deadline := time.Now().Add(attachment.UploadTimeout) + _ = http.NewResponseController(c.Response().Writer).SetReadDeadline(deadline) + defer http.NewResponseController(c.Response().Writer).SetReadDeadline(time.Time{}) + c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, attachment.MaxBytes+1) + a, err := h.attachments.Upload(c.Request().Context(), chat.ID, authenticatedUser(c).ID, filename, c.Request().Header.Get("Content-Type"), c.Request().Body) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusCreated, a) +} +func (h *Chat) getAttachment(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + return c.JSON(http.StatusOK, a) +} +func (h *Chat) attachmentContent(c echo.Context) error { + a, err := h.attachments.Get(c.Request().Context(), c.Param("attachment_id"), authenticatedUser(c).ID) + if err != nil { + return h.useCaseError(c, err) + } + variant, present, err := oneQueryValue(c, "variant") + if err != nil { + return h.invalid(c, "Некорректный variant") + } + if present && variant != "original" { + return h.writeError(c, 404, "not_found", "Вариант не найден") + } + ctx, cancel := context.WithTimeout(c.Request().Context(), 45*time.Second) + defer cancel() + obj, err := h.attachments.Content(ctx, a) + if err != nil { + return h.useCaseError(c, err) + } + defer obj.Close() + c.Response().Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename})) + c.Response().Header().Set("X-Content-Type-Options", "nosniff") + c.Response().Header().Set("Cache-Control", "private, no-store") + return c.Stream(http.StatusOK, a.ContentType, obj) +} diff --git a/internal/app/chat/create_message.go b/internal/app/chat/create_message.go index 3fcdebf..3c0c27b 100644 --- a/internal/app/chat/create_message.go +++ b/internal/app/chat/create_message.go @@ -40,12 +40,19 @@ func (h *Chat) createMessage(c echo.Context) error { } func createMessageRequestToDomain(request dto.CreateMessageRequest) domain.Message { - return domain.Message{ChatID: request.ChatID, Text: request.Text} + return domain.Message{ChatID: request.ChatID, Text: request.Text, AttachmentIDs: request.AttachmentIDs} } func createMessageResponseFromDomain(message domain.Message) dto.CreateMessageResponse { return dto.CreateMessageResponse{ ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, - Text: message.Text, CreatedAt: dto.Timestamp(message.CreatedAt), + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), } } + +func nonnilIDs(ids []string) []string { + if ids == nil { + return []string{} + } + return ids +} diff --git a/internal/app/chat/errors.go b/internal/app/chat/errors.go index a8bd10b..c556830 100644 --- a/internal/app/chat/errors.go +++ b/internal/app/chat/errors.go @@ -33,6 +33,10 @@ func (h *Chat) useCaseError(c echo.Context, err error) error { } status := http.StatusInternalServerError switch applicationError.Code { + case "unsupported_media_type": + status = http.StatusUnsupportedMediaType + case "payload_too_large": + status = http.StatusRequestEntityTooLarge case domain.ErrorUnavailable: status = http.StatusServiceUnavailable case domain.ErrorInvalid: diff --git a/internal/app/chat/list_messages.go b/internal/app/chat/list_messages.go index 05cb861..1d7f252 100644 --- a/internal/app/chat/list_messages.go +++ b/internal/app/chat/list_messages.go @@ -63,7 +63,7 @@ func listMessagesResponseFromDomain(page domain.MessagePage, nextCursor *string) for _, message := range page.Items { items = append(items, dto.ListMessagesItem{ ID: message.ID, ChatID: message.ChatID, SenderID: message.SenderID, - Text: message.Text, CreatedAt: dto.Timestamp(message.CreatedAt), + Text: message.Text, AttachmentIDs: nonnilIDs(message.AttachmentIDs), CreatedAt: dto.Timestamp(message.CreatedAt), }) } return dto.ListMessagesResponse{Items: items, NextCursor: nextCursor} diff --git a/internal/app/chat/readiness.go b/internal/app/chat/readiness.go index d8ee064..950ca17 100644 --- a/internal/app/chat/readiness.go +++ b/internal/app/chat/readiness.go @@ -16,5 +16,10 @@ func (h *Chat) readiness(c echo.Context) error { if err := h.useCase.Ready(ctx); err != nil { return h.useCaseError(c, err) } + if h.attachments != nil { + if err := h.attachments.Ready(ctx); err != nil { + return h.useCaseError(c, err) + } + } return c.JSON(http.StatusOK, dto.ReadinessResponse{Status: "ok"}) } diff --git a/internal/app/chat/service.go b/internal/app/chat/service.go index 5fd2a28..0e4115c 100644 --- a/internal/app/chat/service.go +++ b/internal/app/chat/service.go @@ -1,6 +1,7 @@ package chat import ( + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" "log" "os" "sync/atomic" @@ -13,10 +14,11 @@ import ( // Chat is the HTTP handler and depends only on the application boundary. type Chat struct { - instanceID string - draining atomic.Bool - useCase UseCase - cursor cursorCodec + attachments *attachment.Service + instanceID string + draining atomic.Bool + useCase UseCase + cursor cursorCodec } func NewChat(useCase UseCase, cursorKey [32]byte) *Chat { @@ -81,6 +83,11 @@ func (h *Chat) Router() *echo.Echo { api.GET("/chats/:chat_id", h.getChat) api.POST("/chats/:chat_id/messages", h.createMessage) api.GET("/chats/:chat_id/messages", h.listMessages) + if h.attachments != nil { + api.POST("/chats/:chat_id/attachments", h.uploadAttachment) + api.GET("/attachments/:attachment_id", h.getAttachment) + api.GET("/attachments/:attachment_id/content", h.attachmentContent) + } return e } diff --git a/internal/attachment/image.go b/internal/attachment/image.go new file mode 100644 index 0000000..3e688cf --- /dev/null +++ b/internal/attachment/image.go @@ -0,0 +1,128 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "os" + "strings" + "unicode" + "unicode/utf8" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const MaxBytes int64 = 10 * 1024 * 1024 +const MaxPixels int64 = 40_000_000 + +func appError(code domain.ErrorCode, message string) error { + return domain.NewApplicationError(code, message) +} +func unavailable() error { + return appError(domain.ErrorUnavailable, "Хранилище вложений недоступно") +} +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return appError(domain.ErrorInvalid, "Некорректный filename") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return appError(domain.ErrorInvalid, "Недопустимые символы filename") + } + } + return nil +} + +type Original struct { + File *os.File + Size int64 + SHA256, ContentType string + Width, Height int +} + +func (o *Original) Close() { name := o.File.Name(); o.File.Close(); os.Remove(name) } + +// Spool bounds compressed bytes on disk before any decode. DecodeConfig checks the +// pixel budget before a full decode verifies CRCs/truncated scan data. +func Spool(ctx context.Context, body io.Reader, declared string) (*Original, error) { + media, _, err := mime.ParseMediaType(declared) + if err != nil || (media != "image/png" && media != "image/jpeg") { + return nil, appError("unsupported_media_type", "Допустимы PNG и JPEG") + } + f, err := os.CreateTemp("", "messenger-upload-*") + if err != nil { + return nil, unavailable() + } + o := &Original{File: f} + good := false + defer func() { + if !good { + o.Close() + } + }() + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(f, hash), io.LimitReader(&contextReader{ctx: ctx, r: body}, MaxBytes+1)) + if n > MaxBytes { + return nil, appError("payload_too_large", "Файл превышает 10 MiB") + } + if err != nil { + return nil, appError(domain.ErrorInvalid, "Загрузка прервана") + } + o.Size = n + o.SHA256 = hex.EncodeToString(hash.Sum(nil)) + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + cfg, format, err := image.DecodeConfig(f) + if err != nil { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + actual := "image/" + format + if actual != media { + return nil, appError("unsupported_media_type", "MIME не соответствует содержимому") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > MaxPixels/int64(cfg.Height) { + return nil, appError("payload_too_large", "Изображение превышает 40 миллионов пикселей") + } + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + decoded, format, err := image.Decode(f) + if err != nil || decoded == nil || "image/"+format != actual { + return nil, appError("unsupported_media_type", "Повреждённое изображение") + } + if decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height { + return nil, appError("unsupported_media_type", "Некорректные размеры изображения") + } + decoded = nil + if _, err = f.Seek(0, 0); err != nil { + return nil, unavailable() + } + o.ContentType = actual + o.Width = cfg.Width + o.Height = cfg.Height + good = true + return o, nil +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r *contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + n, err := r.r.Read(p) + if errors.Is(err, context.Canceled) { + return n, err + } + return n, err +} diff --git a/internal/attachment/image_test.go b/internal/attachment/image_test.go new file mode 100644 index 0000000..30700b7 --- /dev/null +++ b/internal/attachment/image_test.go @@ -0,0 +1,139 @@ +package attachment + +import ( + "bytes" + "context" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +func pngFixture(t *testing.T) []byte { + t.Helper() + m := image.NewRGBA(image.Rect(0, 0, 10, 8)) + m.Set(1, 1, color.RGBA{R: 255, A: 255}) + var b bytes.Buffer + if err := png.Encode(&b, m); err != nil { + t.Fatal(err) + } + return b.Bytes() +} +func code(t *testing.T, err error, want domain.ErrorCode) { + t.Helper() + app, ok := domain.AsApplicationError(err) + if !ok || app.Code != want { + t.Fatalf("want %s, got %v", want, err) + } +} +func TestSpoolOriginalAndFullValidation(t *testing.T) { + original := pngFixture(t) + good, err := Spool(context.Background(), bytes.NewReader(original), "image/png") + if err != nil { + t.Fatal(err) + } + defer good.Close() + got, err := io.ReadAll(good.File) + if err != nil || !bytes.Equal(got, original) { + t.Fatal("original changed") + } + if good.Width != 10 || good.Height != 8 || good.Size != int64(len(original)) || len(good.SHA256) != 64 { + t.Fatalf("wrong metadata %+v", good) + } + _, err = Spool(context.Background(), bytes.NewReader(original), "image/jpeg") + code(t, err, "unsupported_media_type") + _, err = Spool(context.Background(), bytes.NewReader(original[:len(original)-12]), "image/png") + code(t, err, "unsupported_media_type") + corrupted := bytes.Clone(original) + corrupted[len(corrupted)-18] ^= 0x80 + _, err = Spool(context.Background(), bytes.NewReader(corrupted), "image/png") + code(t, err, "unsupported_media_type") + var jpegBytes bytes.Buffer + if err = jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 12, 9)), nil); err != nil { + t.Fatal(err) + } + jpegOriginal, err := Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()), "image/jpeg") + if err != nil { + t.Fatal(err) + } + jpegOriginal.Close() + _, err = Spool(context.Background(), bytes.NewReader(jpegBytes.Bytes()[:jpegBytes.Len()/2]), "image/jpeg") + code(t, err, "unsupported_media_type") +} +func TestSpoolLimitsBeforeDecode(t *testing.T) { + // Oversized compressed input is stopped after exactly MaxBytes+1 reads. + source := &countingReader{remaining: MaxBytes + 100_000} + _, err := Spool(context.Background(), source, "image/png") + code(t, err, "payload_too_large") + if source.read != MaxBytes+1 { + t.Fatalf("read %d bytes", source.read) + } + bomb := bytes.Clone(pngFixture(t)) + binary.BigEndian.PutUint32(bomb[16:20], 40001) + binary.BigEndian.PutUint32(bomb[20:24], 1000) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + _, err = Spool(context.Background(), bytes.NewReader(bomb), "image/png") + code(t, err, "payload_too_large") +} + +type countingReader struct{ remaining, read int64 } + +func (c *countingReader) Read(p []byte) (int, error) { + if c.remaining == 0 { + return 0, io.EOF + } + n := min(int64(len(p)), c.remaining) + clear(p[:n]) + c.remaining -= n + c.read += n + return int(n), nil +} +func TestFilenameAndCancelledUpload(t *testing.T) { + for _, name := range []string{"", "../x.png", "x\\photo.jpg", "a\x00b", "a\nb", "\t"} { + code(t, ValidateFilename(name), domain.ErrorInvalid) + } + for _, name := range []string{"фото.png", "image.jpg", ".."} { + if err := ValidateFilename(name); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Spool(ctx, bytes.NewReader(pngFixture(t)), "image/png") + code(t, err, domain.ErrorInvalid) +} +func TestReconcileRejectsUnsafeGrace(t *testing.T) { + s := &Service{} + if _, err := s.Reconcile(context.Background(), true, 89*time.Second); err == nil { + t.Fatal("unsafe grace accepted") + } + if MinimumGrace <= UploadTimeout { + t.Fatal("grace does not cover in-flight PUT") + } +} + +func TestValid16BitWidePNG(t *testing.T) { + // Exercise the same RGBA64 + scanline allocation path as an extreme-aspect + // 40M image, at 1/1000 of that pixel count to keep the unit suite small. + img := image.NewNRGBA64(image.Rect(0, 0, 40000, 1)) + img.SetNRGBA64(1, 0, color.NRGBA64{R: 65535, G: 12345, B: 4, A: 32000}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, img); err != nil { + t.Fatal(err) + } + original, err := Spool(context.Background(), bytes.NewReader(encoded.Bytes()), "image/png") + if err != nil { + t.Fatal(err) + } + defer original.Close() + if original.Width != 40000 || original.Height != 1 { + t.Fatal("wide PNG dimensions changed") + } +} diff --git a/internal/attachment/json.go b/internal/attachment/json.go new file mode 100644 index 0000000..cb3958f --- /dev/null +++ b/internal/attachment/json.go @@ -0,0 +1,18 @@ +package attachment + +import ( + "encoding/json" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/dto" +) + +// MarshalJSON uses the API's fixed microsecond precision instead of time.Time's +// variable fractional precision, which Python 3.9 clients cannot always parse. +func (a Attachment) MarshalJSON() ([]byte, error) { + type fields Attachment + return json.Marshal(struct { + fields + CreatedAt dto.Timestamp `json:"created_at"` + UpdatedAt dto.Timestamp `json:"updated_at"` + }{fields: fields(a), CreatedAt: dto.Timestamp(a.CreatedAt), UpdatedAt: dto.Timestamp(a.UpdatedAt)}) +} diff --git a/internal/attachment/json_test.go b/internal/attachment/json_test.go new file mode 100644 index 0000000..4812d60 --- /dev/null +++ b/internal/attachment/json_test.go @@ -0,0 +1,37 @@ +package attachment + +import ( + "encoding/json" + "testing" + "time" +) + +func TestAttachmentTimestampsUseFixedMicroseconds(t *testing.T) { + value := Attachment{ID: "attachment-id", CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 123400000, time.UTC), UpdatedAt: time.Date(2026, 1, 2, 6, 4, 5, 0, time.FixedZone("MSK", 3*60*60)), Key: "original/private", Variants: []any{}} + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err = json.Unmarshal(encoded, &fields); err != nil { + t.Fatal(err) + } + if fields["created_at"] != "2026-01-02T03:04:05.123400Z" { + t.Fatalf("created_at=%v", fields["created_at"]) + } + if fields["updated_at"] != "2026-01-02T03:04:05.000000Z" { + t.Fatalf("updated_at=%v", fields["updated_at"]) + } + if fields["id"] != value.ID { + t.Fatal("serialization lost metadata") + } + if _, exposed := fields["Key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if _, exposed := fields["key"]; exposed { + t.Fatal("serialization exposed private object key") + } + if fields["error_code"] != nil { + t.Fatal("ready error_code must be null") + } +} diff --git a/internal/attachment/reconcile.go b/internal/attachment/reconcile.go new file mode 100644 index 0000000..402d7b3 --- /dev/null +++ b/internal/attachment/reconcile.go @@ -0,0 +1,222 @@ +package attachment + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/minio/minio-go/v7" +) + +type Summary struct { + Apply bool `json:"apply"` + GraceSeconds float64 `json:"grace_seconds"` + ObjectsScanned int `json:"objects_scanned"` + MetadataScanned int `json:"metadata_scanned"` + BytesScanned int64 `json:"bytes_scanned"` + ListRequestsEstimate int `json:"list_requests_estimate"` + StatRequests int `json:"stat_requests"` + DBQueries int `json:"db_queries"` + Orphans int `json:"orphans"` + Missing int `json:"missing"` + FreshSkipped int `json:"fresh_skipped"` + LockedSkipped int `json:"locked_skipped"` + ReferencedMissing int `json:"referenced_missing"` + Deleted int `json:"deleted"` + MarkedFailed int `json:"marked_failed"` + DurationMS int64 `json:"duration_ms"` + Actions []Action `json:"actions"` +} +type Action struct { + Kind string `json:"kind"` + Key string `json:"key"` + Applied bool `json:"applied"` +} + +// Reconcile scans a fixed cutoff and rechecks each candidate while holding the +// same transaction-scoped advisory lock as upload. Metadata, including message +// references, is never deleted. Missing originals become failed tombstones. +func (s *Service) Reconcile(ctx context.Context, apply bool, grace time.Duration) (result Summary, err error) { + started := time.Now() + result = Summary{Apply: apply, GraceSeconds: grace.Seconds(), Actions: []Action{}} + defer func() { result.DurationMS = time.Since(started).Milliseconds() }() + if grace < MinimumGrace { + return result, fmt.Errorf("grace must be at least %s (upload timeout %s)", MinimumGrace, UploadTimeout) + } + cutoff := started.Add(-grace) + for obj := range s.Store.Client.ListObjects(ctx, s.Store.Bucket, minio.ListObjectsOptions{Prefix: "original/", Recursive: true, MaxKeys: 1000}) { + if obj.Err != nil { + return result, fmt.Errorf("S3 listing failed") + } + result.ObjectsScanned++ + result.BytesScanned += obj.Size + if result.ObjectsScanned%1000 == 1 { + result.ListRequestsEstimate++ + } + if !obj.LastModified.Before(cutoff) { + result.FreshSkipped++ + continue + } + err = s.reconcileObject(ctx, obj.Key, cutoff, apply, &result) + if err != nil { + return result, err + } + } + // Bounded keyset pages avoid retaining the entire bucket or a DB cursor while + // making per-object transactions. A later run picks up concurrent new rows. + last := "00000000-0000-0000-0000-000000000000" + for { + result.DBQueries++ + rows, e := s.Pool.Query(ctx, `SELECT id::text,object_key,created_at,status FROM attachments WHERE id>$1 ORDER BY id LIMIT 100`, last) + if e != nil { + return result, fmt.Errorf("metadata scan failed") + } + type metadata struct { + id, key, status string + created time.Time + } + page := []metadata{} + for rows.Next() { + var m metadata + if e = rows.Scan(&m.id, &m.key, &m.created, &m.status); e != nil { + rows.Close() + return result, e + } + page = append(page, m) + } + e = rows.Err() + rows.Close() + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, m := range page { + last = m.id + result.MetadataScanned++ + if m.status != "ready" { + continue + } + if !m.created.Before(cutoff) { + result.FreshSkipped++ + continue + } + if e = s.reconcileMetadata(ctx, m.id, m.key, cutoff, apply, &result); e != nil { + return result, e + } + } + } + if ctx.Err() != nil { + return result, ctx.Err() + } + return result, nil +} +func (s *Service) locked(ctx context.Context, key string, r *Summary) (pgx.Tx, bool, error) { + tx, err := s.Pool.Begin(ctx) + if err != nil { + return nil, false, fmt.Errorf("DB unavailable") + } + var locked bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, key).Scan(&locked); err != nil { + tx.Rollback(context.Background()) + return nil, false, fmt.Errorf("lock failed") + } + if !locked { + r.LockedSkipped++ + tx.Rollback(context.Background()) + return nil, false, nil + } + return tx, true, nil +} +func (s *Service) reconcileObject(ctx context.Context, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var exists bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)`, key).Scan(&exists); err != nil { + return fmt.Errorf("metadata recheck failed") + } + if exists { + return nil + } + r.StatRequests++ + obj, err := s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if IsMissing(err) { + return nil + } + if err != nil { + return fmt.Errorf("object recheck failed") + } + if !obj.LastModified.Before(cutoff) { + r.FreshSkipped++ + return nil + } + r.Orphans++ + if apply { + if err = s.Store.Client.RemoveObject(ctx, s.Store.Bucket, key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("orphan deletion failed") + } + r.Deleted++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "orphan", Key: key, Applied: apply}) + } + return nil +} +func (s *Service) reconcileMetadata(ctx context.Context, id, key string, cutoff time.Time, apply bool, r *Summary) error { + tx, ok, err := s.locked(ctx, key, r) + if err != nil || !ok { + return err + } + defer tx.Rollback(context.Background()) + var created time.Time + var status string + r.DBQueries++ + err = tx.QueryRow(ctx, `SELECT created_at,status FROM attachments WHERE id=$1 FOR UPDATE`, id).Scan(&created, &status) + if err == pgx.ErrNoRows { + return nil + } + if err != nil { + return fmt.Errorf("metadata recheck failed") + } + if status != "ready" || !created.Before(cutoff) { + return nil + } + r.StatRequests++ + _, err = s.Store.Client.StatObject(ctx, s.Store.Bucket, key, minio.StatObjectOptions{}) + if err == nil { + return nil + } + if !IsMissing(err) { + return fmt.Errorf("object stat failed (no mutation)") + } + var referenced bool + r.DBQueries++ + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE $1::uuid=ANY(attachment_ids))`, id).Scan(&referenced); err != nil { + return fmt.Errorf("reference check failed") + } + r.Missing++ + if referenced { + r.ReferencedMissing++ + } + if apply { + r.DBQueries++ + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='original_missing' WHERE id=$1 AND status='ready'`, id); err != nil { + return fmt.Errorf("mark missing failed") + } + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit missing failed") + } + r.MarkedFailed++ + } + if len(r.Actions) < 100 { + r.Actions = append(r.Actions, Action{Kind: "missing", Key: key, Applied: apply}) + } + return nil +} diff --git a/internal/attachment/s3.go b/internal/attachment/s3.go new file mode 100644 index 0000000..2b1315c --- /dev/null +++ b/internal/attachment/s3.go @@ -0,0 +1,44 @@ +package attachment + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Store struct { + Client *minio.Client + Bucket string +} + +func NewStore(endpoint, bucket, access, secret string, secure bool) (*Store, error) { + transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 3 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, IdleConnTimeout: 30 * time.Second, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 5 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}} + c, err := minio.New(endpoint, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: secure, Region: "us-east-1", Transport: transport, MaxRetries: 1}) + if err != nil { + return nil, err + } + return &Store{Client: c, Bucket: bucket}, nil +} +func (s *Store) Ready(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + for item := range s.Client.ListObjects(ctx, s.Bucket, minio.ListObjectsOptions{Prefix: "original/", MaxKeys: 1, Recursive: true}) { + if item.Err != nil { + return unavailable() + } + return nil + } + if ctx.Err() != nil { + return unavailable() + } + return nil +} +func IsMissing(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchObject" +} diff --git a/internal/attachment/service.go b/internal/attachment/service.go new file mode 100644 index 0000000..3156afb --- /dev/null +++ b/internal/attachment/service.go @@ -0,0 +1,132 @@ +package attachment + +import ( + "context" + "errors" + "io" + "log" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" +) + +const UploadTimeout = 45 * time.Second +const MinimumGrace = 90 * time.Second + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Key string `json:"-"` +} + +type Service struct { + Pool *pgxpool.Pool + Store *Store + slots chan struct{} +} + +func New(pool *pgxpool.Pool, store *Store) *Service { + return &Service{Pool: pool, Store: store, slots: make(chan struct{}, 1)} +} +func (s *Service) Ready(ctx context.Context) error { return s.Store.Ready(ctx) } +func (s *Service) Upload(ctx context.Context, chatID, userID, name, contentType string, body io.Reader) (Attachment, error) { + if err := ValidateFilename(name); err != nil { + return Attachment{}, err + } + ctx, cancel := context.WithTimeout(ctx, UploadTimeout) + defer cancel() + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + case <-ctx.Done(): + return Attachment{}, unavailable() + } + original, err := Spool(ctx, body, contentType) + if err != nil { + return Attachment{}, err + } + defer original.Close() + a := Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: name, ContentType: original.ContentType, Size: original.Size, SHA256: original.SHA256, Status: "ready", Variants: []any{}} + a.Key = "original/" + a.ID + tx, err := s.Pool.Begin(ctx) + if err != nil { + return Attachment{}, unavailable() + } + defer tx.Rollback(context.Background()) + // Keep the object lock through PUT, metadata insert and COMMIT. A crash releases + // the lock; the grace protects a still-running client PUT at the S3 server. + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, a.Key); err != nil { + return Attachment{}, unavailable() + } + _, err = s.Store.Client.PutObject(ctx, s.Store.Bucket, a.Key, original.File, a.Size, minio.PutObjectOptions{ContentType: a.ContentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": a.SHA256}}) + if err != nil { + return Attachment{}, unavailable() + } + err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,uploader_id,filename,object_key,content_type,size_bytes,sha256,width,height) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING created_at`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.Key, a.ContentType, a.Size, a.SHA256, original.Width, original.Height).Scan(&a.CreatedAt) + if err != nil { + log.Print("attachment metadata failed; orphan retained for grace-period reconciliation") + return Attachment{}, unavailable() + } + // Never DELETE on an ambiguous COMMIT failure: the DB may have committed. A + // later reconciler checks both stores under the same lock before any cleanup. + if err = tx.Commit(ctx); err != nil { + return Attachment{}, unavailable() + } + a.UpdatedAt = a.CreatedAt + return a, nil +} +func (s *Service) Get(ctx context.Context, id, userID string) (Attachment, error) { + parsed, err := uuid.Parse(id) + if err != nil { + return Attachment{}, appError(domain.ErrorInvalid, "Некорректный attachment_id") + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + a := Attachment{Variants: []any{}} + err = s.Pool.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.uploader_id::text,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,a.error_code,a.created_at FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=$1 AND m.user_id=$2`, parsed.String(), userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.Key, &a.ContentType, &a.Size, &a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return Attachment{}, appError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return Attachment{}, unavailable() + } + a.UpdatedAt = a.CreatedAt + // Keep the mandatory lab4 ready-only metadata schema. Failed originals are + // represented by 409; internal status remains inspectable in reconciliation. + if a.Status != "ready" { + return Attachment{}, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return a, nil +} +func (s *Service) Content(ctx context.Context, a Attachment) (*minio.Object, error) { + if a.Status != "ready" { + return nil, appError(domain.ErrorConflict, "Вложение не готово") + } + obj, err := s.Store.Client.GetObject(ctx, s.Store.Bucket, a.Key, minio.GetObjectOptions{}) + if err != nil { + return nil, unavailable() + } + if _, err = obj.Stat(); err != nil { + obj.Close() + if IsMissing(err) { + return nil, appError(domain.ErrorConflict, "Оригинал вложения недоступен") + } + return nil, unavailable() + } + return obj, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 84b4dcb..ace6c24 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,11 @@ import ( ) type Config struct { + S3Endpoint string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseSSL bool Port string PostgresHost string PostgresPort uint16 @@ -96,5 +101,19 @@ func Load() (Config, error) { return c, fmt.Errorf("CURSOR_KEY must be 32 random bytes encoded as hex") } copy(c.CursorKey[:], decoded) + c.S3Endpoint = value("S3_ENDPOINT", "minio:9000") + c.S3Bucket = value("S3_BUCKET", "attachments") + c.S3UseSSL, err = strconv.ParseBool(value("S3_USE_SSL", "false")) + if err != nil { + return c, fmt.Errorf("invalid S3_USE_SSL") + } + c.S3AccessKey, err = secret("S3_ACCESS_KEY") + if err != nil { + return c, err + } + c.S3SecretKey, err = secret("S3_SECRET_KEY") + if err != nil { + return c, err + } return c, nil } diff --git a/internal/pkg/domain/message.go b/internal/pkg/domain/message.go index 41b93ac..3487996 100644 --- a/internal/pkg/domain/message.go +++ b/internal/pkg/domain/message.go @@ -7,6 +7,7 @@ type Message struct { ID string ChatID string SenderID string + AttachmentIDs []string Text string CreatedAt time.Time } diff --git a/internal/pkg/repository/postgres/lab4_test.go b/internal/pkg/repository/postgres/lab4_test.go new file mode 100644 index 0000000..53ab8d9 --- /dev/null +++ b/internal/pkg/repository/postgres/lab4_test.go @@ -0,0 +1,341 @@ +package postgres_test + +import ( + "bufio" + "bytes" + "context" + "encoding/xml" + "fmt" + "image" + "image/png" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/attachment" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/repository/postgres" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type fakeObject struct { + data []byte + modified time.Time +} +type fakeS3 struct { + mu sync.Mutex + objects map[string]fakeObject + failPUT bool +} + +func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + key := strings.TrimPrefix(r.URL.Path, "/attachments/") + if r.Method == "GET" && r.URL.Path == "/attachments/" || r.Method == "GET" && r.URL.Path == "/attachments" { + type item struct { + Key string + LastModified string + ETag string + Size int + } + type listing struct { + XMLName xml.Name `xml:"ListBucketResult"` + Xmlns string `xml:"xmlns,attr"` + Name string + Prefix string + MaxKeys int + IsTruncated bool + Contents []item + } + l := listing{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", Name: "attachments", Prefix: r.URL.Query().Get("prefix"), MaxKeys: 1000} + for k, v := range f.objects { + if strings.HasPrefix(k, l.Prefix) { + l.Contents = append(l.Contents, item{Key: k, LastModified: v.modified.UTC().Format("2006-01-02T15:04:05.000Z"), ETag: `"abc"`, Size: len(v.data)}) + } + } + w.Header().Set("Content-Type", "application/xml") + xml.NewEncoder(w).Encode(l) + return + } + if r.Method == "PUT" { + if f.failPUT { + w.WriteHeader(503) + fmt.Fprint(w, `ServiceUnavailable`) + return + } + var data []byte + var err error + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") { + reader := bufio.NewReader(r.Body) + for { + line, e := reader.ReadString('\n') + if e != nil { + err = e + break + } + size, e := strconv.ParseInt(strings.Split(strings.TrimSpace(line), ";")[0], 16, 64) + if e != nil { + err = e + break + } + if size == 0 { + break + } + chunk := make([]byte, size) + if _, e = io.ReadFull(reader, chunk); e != nil { + err = e + break + } + data = append(data, chunk...) + if _, e = reader.Discard(2); e != nil { + err = e + break + } + } + } else { + data, err = io.ReadAll(r.Body) + } + if err != nil { + w.WriteHeader(400) + return + } + f.objects[key] = fakeObject{data: data, modified: time.Now().UTC()} + w.Header().Set("ETag", `"900150983cd24fb0d6963f7d28e17f72"`) + w.WriteHeader(200) + return + } + obj, ok := f.objects[key] + if !ok { + w.WriteHeader(404) + if r.Method != "HEAD" { + fmt.Fprint(w, `NoSuchKey`) + } + return + } + if r.Method == "DELETE" { + delete(f.objects, key) + w.WriteHeader(204) + return + } + w.Header().Set("Last-Modified", obj.modified.UTC().Format(http.TimeFormat)) + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Content-Length", fmt.Sprint(len(obj.data))) + w.Header().Set("Content-Type", "image/png") + if r.Method == "GET" { + w.Write(obj.data) + } +} +func lab4Backend(t *testing.T, pool *pgxpool.Pool, repo *postgres.Storage, user domain.User, chat domain.Chat) { + ctx := context.Background() + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + service := attachment.New(pool, &attachment.Store{Client: client, Bucket: "attachments"}) + var imageBytes bytes.Buffer + if err = png.Encode(&imageBytes, image.NewRGBA(image.Rect(0, 0, 10, 8))); err != nil { + t.Fatal(err) + } + upload := func(name string) attachment.Attachment { + t.Helper() + a, e := service.Upload(ctx, chat.ID, user.ID, name, "image/png", bytes.NewReader(imageBytes.Bytes())) + if e != nil { + t.Fatal(e) + } + return a + } + first := upload("first.png") + second := upload("second.png") + requireCode := func(err error, want domain.ErrorCode) { + t.Helper() + e, ok := domain.AsApplicationError(err) + if !ok || e.Code != want { + t.Fatalf("expected %s, got %v", want, err) + } + } + outside, err := repo.CreateUser(ctx, "attachment_outsider", "Outside", "hash") + if err != nil { + t.Fatal(err) + } + _, err = service.Get(ctx, first.ID, outside.ID) + requireCode(err, domain.ErrorNotFound) + obj, err := service.Content(ctx, first) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(obj) + obj.Close() + if err != nil || !bytes.Equal(data, imageBytes.Bytes()) { + t.Fatalf("original mismatch: %v", err) + } + otherChat, err := repo.CreateChat(ctx, "Other", []string{user.ID}) + if err != nil { + t.Fatal(err) + } + _, err = repo.CreateMessageWithAttachments(ctx, otherChat.ID, user.ID, "", "cross-chat", []string{first.ID}) + requireCode(err, domain.ErrorNotFound) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "unknown", []string{uuid.NewString()}) + requireCode(err, domain.ErrorNotFound) + sent, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil { + t.Fatal(err) + } + replay, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || sent.ID != replay.ID { + t.Fatalf("replay failed: %v", err) + } + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{second.ID, first.ID}) + requireCode(err, domain.ErrorConflict) + textOnly, err := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", nil) + if err != nil { + t.Fatal(err) + } + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "text", "empty-equivalent", []string{}) + if err != nil || textOnly.ID != replay.ID { + t.Fatalf("absent/empty differs: %v", err) + } + + const callers = 12 + outcomes := make(chan domain.Message, callers) + failures := make(chan error, callers) + var concurrent sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < callers; i++ { + concurrent.Add(1) + go func() { + defer concurrent.Done() + <-start + m, e := repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "concurrent-attachments", []string{first.ID, second.ID}) + if e != nil { + failures <- e + } else { + outcomes <- m + } + }() + } + close(start) + concurrent.Wait() + close(outcomes) + close(failures) + for e := range failures { + t.Fatal(e) + } + identity := "" + for m := range outcomes { + if identity == "" { + identity = m.ID + } + if m.ID != identity { + t.Fatal("duplicate concurrent attachment message") + } + } + var effects int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id=$1 AND idempotency_key='concurrent-attachments'`, chat.ID).Scan(&effects); err != nil || effects != 1 { + t.Fatalf("concurrent effects=%d %v", effects, err) + } + t.Log("ACL, original bytes, cross-chat attachment rejection, ordered/concurrent idempotency and absent=[] passed") + + old := time.Now().Add(-3 * time.Minute) + orphan := "original/" + uuid.NewString() + fresh := "original/" + uuid.NewString() + locked := "original/" + uuid.NewString() + fake.mu.Lock() + delete(fake.objects, first.Key) + v := fake.objects[second.Key] + v.modified = old + fake.objects[second.Key] = v + fake.objects[orphan] = fakeObject{data: []byte("orphan"), modified: old} + fake.objects[fresh] = fakeObject{data: []byte("fresh"), modified: time.Now()} + fake.objects[locked] = fakeObject{data: []byte("locked"), modified: old} + fake.mu.Unlock() + if _, err = pool.Exec(ctx, `UPDATE attachments SET created_at=$1 WHERE id=ANY($2::uuid[])`, old, []string{first.ID, second.ID}); err != nil { + t.Fatal(err) + } + hold, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer hold.Rollback(ctx) + if _, err = hold.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, locked); err != nil { + t.Fatal(err) + } + dry, err := service.Reconcile(ctx, false, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if dry.Orphans != 1 || dry.Missing != 1 || dry.ReferencedMissing != 1 || dry.LockedSkipped != 1 || dry.FreshSkipped < 1 || dry.Deleted != 0 || dry.MarkedFailed != 0 { + t.Fatalf("bad dry-run %+v", dry) + } + var status string + if err = pool.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1`, first.ID).Scan(&status); err != nil || status != "ready" { + t.Fatalf("dry-run mutated metadata %s %v", status, err) + } + applied, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if applied.Deleted != 1 || applied.MarkedFailed != 1 { + t.Fatalf("bad apply %+v", applied) + } + fake.mu.Lock() + _, hasOrphan := fake.objects[orphan] + _, hasFresh := fake.objects[fresh] + _, hasLinked := fake.objects[second.Key] + _, hasLocked := fake.objects[locked] + fake.mu.Unlock() + if hasOrphan || !hasFresh || !hasLinked || !hasLocked { + t.Fatal("unsafe object cleanup") + } + _, err = service.Get(ctx, first.ID, user.ID) + requireCode(err, domain.ErrorConflict) + _, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "failed-attachment", []string{first.ID}) + requireCode(err, domain.ErrorConflict) + replay, err = repo.CreateMessageWithAttachments(ctx, chat.ID, user.ID, "", "ordered", []string{first.ID, second.ID}) + if err != nil || replay.ID != sent.ID { + t.Fatalf("historical replay changed after missing: %v", err) + } + repeated, err := service.Reconcile(ctx, true, 90*time.Second) + if err != nil { + t.Fatal(err) + } + if repeated.Deleted != 0 || repeated.MarkedFailed != 0 || repeated.Orphans != 0 || repeated.Missing != 0 { + t.Fatalf("not repeatable %+v", repeated) + } + t.Logf("dry-run=%+v apply=%+v repeat=%+v; fresh, locked and referenced objects retained", dry, applied, repeated) +} + +func TestS3WireFixture(t *testing.T) { + fake := &fakeS3{objects: map[string]fakeObject{}} + server := httptest.NewServer(fake) + defer server.Close() + client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{Creds: credentials.NewStaticV4("test-key", "test-secret", ""), Region: "us-east-1", MaxRetries: 1}) + if err != nil { + t.Fatal(err) + } + payload := []byte("exact object bytes") + _, err = client.PutObject(context.Background(), "attachments", "original/test", bytes.NewReader(payload), int64(len(payload)), minio.PutObjectOptions{ContentType: "image/png", DisableMultipart: true}) + if err != nil { + t.Fatal(err) + } + obj, err := client.GetObject(context.Background(), "attachments", "original/test", minio.GetObjectOptions{}) + if err != nil { + t.Fatal(err) + } + defer obj.Close() + got, err := io.ReadAll(obj) + if err != nil || !bytes.Equal(got, payload) { + t.Fatalf("wire fixture did not store original: %q %v", got, err) + } +} diff --git a/internal/pkg/repository/postgres/messages.go b/internal/pkg/repository/postgres/messages.go index ec99762..3453e78 100644 --- a/internal/pkg/repository/postgres/messages.go +++ b/internal/pkg/repository/postgres/messages.go @@ -3,41 +3,76 @@ package postgres import ( "context" "errors" - + "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" "github.com/google/uuid" "github.com/jackc/pgx/v5" - - "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" + "slices" ) func (s *Storage) CreateMessage(ctx context.Context, chatID, senderID, text, key string) (domain.Message, error) { + return s.CreateMessageWithAttachments(ctx, chatID, senderID, text, key, nil) +} + +func (s *Storage) CreateMessageWithAttachments(ctx context.Context, chatID, senderID, text, key string, ids []string) (domain.Message, error) { ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() - message := domain.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text} + if ids == nil { + ids = []string{} + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return domain.Message{}, dbError(err) + } + defer tx.Rollback(context.Background()) + // Serialize this idempotency scope before validating attachments. A replay remains a + // replay even when reconciliation has since marked an original as missing. + if key != "" { + _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, senderID+":"+chatID+":"+key) + if err != nil { + return domain.Message{}, dbError(err) + } + old := domain.Message{ChatID: chatID, SenderID: senderID} + err = tx.QueryRow(ctx, `SELECT id::text,text,attachment_ids::text[],created_at FROM messages WHERE sender_id=$1 AND chat_id=$2 AND idempotency_key=$3`, senderID, chatID, key).Scan(&old.ID, &old.Text, &old.AttachmentIDs, &old.CreatedAt) + if err == nil { + if old.Text != text || !slices.Equal(old.AttachmentIDs, ids) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") + } + return old, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, dbError(err) + } + } + // Lock each metadata row against reconciler updates until the link is committed. + for _, id := range ids { + var status string + err = tx.QueryRow(ctx, `SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE`, id, chatID).Scan(&status) + if errors.Is(err, pgx.ErrNoRows) { + return domain.Message{}, domain.NewApplicationError(domain.ErrorNotFound, "Объект не найден") + } + if err != nil { + return domain.Message{}, dbError(err) + } + if status != "ready" { + return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Вложение не готово") + } + } + m := domain.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text, AttachmentIDs: ids} var storedKey *string if key != "" { storedKey = &key } - // One row contains both the side effect and its durable replay result. - err := s.pool.QueryRow(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,idempotency_key) - VALUES($1,$2,$3,$4,$5) ON CONFLICT (sender_id,chat_id,idempotency_key) DO NOTHING RETURNING created_at`, - message.ID, chatID, senderID, text, storedKey).Scan(&message.CreatedAt) - if errors.Is(err, pgx.ErrNoRows) { - // A new READ COMMITTED statement sees the winner after the unique-index wait. - err = s.pool.QueryRow(ctx, `SELECT id::text,text,created_at FROM messages - WHERE sender_id=$1 AND chat_id=$2 AND idempotency_key=$3`, senderID, chatID, key). - Scan(&message.ID, &message.Text, &message.CreatedAt) - if err == nil && message.Text != text { - return domain.Message{}, domain.NewApplicationError(domain.ErrorConflict, "Idempotency-Key уже использован с другим телом") - } + err = tx.QueryRow(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,idempotency_key,attachment_ids) VALUES($1,$2,$3,$4,$5,$6) RETURNING created_at`, m.ID, chatID, senderID, text, storedKey, ids).Scan(&m.CreatedAt) + if err != nil { + return domain.Message{}, dbError(err) } - return message, dbError(err) + return m, dbError(tx.Commit(ctx)) } func (s *Storage) ListMessages(ctx context.Context, chatID string, after *domain.Position, limit int) ([]domain.Message, error) { ctx, cancel := context.WithTimeout(ctx, s.timeout) defer cancel() - query := `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.created_at FROM messages m WHERE m.chat_id=$1` + query := `SELECT m.id::text,m.chat_id::text,m.sender_id::text,m.text,m.attachment_ids::text[],m.created_at FROM messages m WHERE m.chat_id=$1` args := []any{chatID, limit} if after != nil { query += ` AND (m.created_at,m.id)>($3,$4)` @@ -49,13 +84,13 @@ func (s *Storage) ListMessages(ctx context.Context, chatID string, after *domain return nil, dbError(err) } defer rows.Close() - messages := make([]domain.Message, 0) + result := []domain.Message{} for rows.Next() { var m domain.Message - if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.AttachmentIDs, &m.CreatedAt); err != nil { return nil, dbError(err) } - messages = append(messages, m) + result = append(result, m) } - return messages, dbError(rows.Err()) + return result, dbError(rows.Err()) } diff --git a/internal/pkg/repository/postgres/storage_test.go b/internal/pkg/repository/postgres/storage_test.go index 78ad5ab..838f1ad 100644 --- a/internal/pkg/repository/postgres/storage_test.go +++ b/internal/pkg/repository/postgres/storage_test.go @@ -142,6 +142,13 @@ func TestIntegration(t *testing.T) { if err != nil { t.Fatal(err) } + t.Run("lab4_attachments_and_reconciliation", func(t *testing.T) { + isolated, e := repo.CreateChat(ctx, "Lab4", []string{user.ID}) + if e != nil { + t.Fatal(e) + } + lab4Backend(t, pool, repo, user, isolated) + }) _, err = pool.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT md5(i::text)::uuid,$1,$2,'synthetic message '||i, timestamptz '2026-01-01 00:00:00+00' + (i / 10) * interval '1 millisecond' diff --git a/internal/usecase/chat/service.go b/internal/usecase/chat/service.go index da6460f..0ca9425 100644 --- a/internal/usecase/chat/service.go +++ b/internal/usecase/chat/service.go @@ -3,7 +3,9 @@ package chat import ( "context" "errors" + "strings" "time" + "unicode/utf8" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/domain" "gitflic.ru/akselbant/mtusi2026lab1-fork/internal/pkg/password" @@ -105,10 +107,32 @@ func (u *UseCase) CreateMessage(ctx context.Context, callerID string, message do if _, err := u.chatForUser(ctx, chatID, callerID); err != nil { return domain.Message{}, err } - if !validRequiredText(message.Text, 4000) { + if !utf8.ValidString(message.Text) || utf8.RuneCountInString(message.Text) > 4000 || (strings.TrimSpace(message.Text) == "" && len(message.AttachmentIDs) == 0) { return domain.Message{}, invalid("text должен содержать от 1 до 4000 символов") } - created, err := u.repository.CreateMessage(ctx, chatID, callerID, message.Text, message.IdempotencyKey) + if len(message.AttachmentIDs) > 10 { + return domain.Message{}, invalid("Не более 10 вложений") + } + seen := map[string]bool{} + for i, id := range message.AttachmentIDs { + normalized, ok := normalizeUUID(id) + if !ok || seen[normalized] { + return domain.Message{}, invalid("Некорректные или повторные attachment_ids") + } + seen[normalized] = true + message.AttachmentIDs[i] = normalized + } + var created domain.Message + var err error + if repository, ok := u.repository.(interface { + CreateMessageWithAttachments(context.Context, string, string, string, string, []string) (domain.Message, error) + }); ok { + created, err = repository.CreateMessageWithAttachments(ctx, chatID, callerID, message.Text, message.IdempotencyKey, message.AttachmentIDs) + } else if len(message.AttachmentIDs) > 0 { + return domain.Message{}, notFound() + } else { + created, err = u.repository.CreateMessage(ctx, chatID, callerID, message.Text, message.IdempotencyKey) + } if errors.Is(err, ErrChatNotFound) || errors.Is(err, ErrNotMember) { return domain.Message{}, notFound() } diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..a8b812a --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + uploader_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready','failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((status='ready' AND error_code IS NULL) OR (status='failed' AND error_code IS NOT NULL)) +); + +-- +goose Down +DROP TABLE IF EXISTS attachments; diff --git a/migrations/004_message_attachments.sql b/migrations/004_message_attachments.sql new file mode 100644 index 0000000..1fd41a9 --- /dev/null +++ b/migrations/004_message_attachments.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE messages + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD COLUMN IF NOT EXISTS attachment_ids uuid[] NOT NULL DEFAULT '{}'; + +-- +goose Down +-- Attachment-only messages cannot exist in lab3; retain an explicit placeholder when downgrading. +UPDATE messages SET text='[attachment]' WHERE text=''; +ALTER TABLE messages + DROP COLUMN IF EXISTS attachment_ids, + DROP CONSTRAINT IF EXISTS messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000); diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index ae4321a..3a18367 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -4,12 +4,17 @@ cd "$(dirname "$0")/.." umask 077 mkdir -p secrets chmod 700 secrets -for name in postgres_password cursor_key; do +for name in postgres_password cursor_key s3_root_secret_key s3_secret_key; do if [ ! -s "secrets/$name" ]; then openssl rand -hex 32 > "secrets/$name" fi done +for name in s3_root_access_key s3_access_key; do + if [ ! -s "secrets/$name" ]; then + openssl rand -hex 8 > "secrets/$name" + fi +done # Compose mounts files directly on Linux; the unprivileged API must read them. # The host directory remains private (0700). -chmod 444 secrets/postgres_password secrets/cursor_key +chmod 444 secrets/postgres_password secrets/cursor_key secrets/s3_root_access_key secrets/s3_root_secret_key secrets/s3_access_key secrets/s3_secret_key printf '%s\n' 'Local secrets are ready.' diff --git a/scripts/measure_uploads.py b/scripts/measure_uploads.py new file mode 100644 index 0000000..7a0de35 --- /dev/null +++ b/scripts/measure_uploads.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Ten simultaneous proxy uploads, checksums, latency and cgroup memory samples.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import math +from pathlib import Path +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT/'tests/own')) +from helpers import Client, check, containers, docker, png, save, upload + + +def memory(ids): + result = {} + for container in ids: + raw = docker('exec',container,'cat','/sys/fs/cgroup/memory.current').stdout + result[container] = int(raw) + return result + + +def percentile(values, p): + return sorted(values)[max(0, math.ceil(len(values)*p)-1)] + + +def kernel_peak(ids): + return {container:int(docker('exec',container,'cat','/sys/fs/cgroup/memory.peak').stdout) + for container in ids} + + +def main(): + c = Client(timeout=55) + alice,bob,_,chat = c.bootstrap() + upload(c,alice,chat,png()) + ids = containers('api') + results = {'concurrency':10, 'warmup_uploads':1, 'memory_method':'cgroup v2 memory.current sampled every >=100ms; values include file cache', 'runs':[]} + for width,height in ((512,512),(1024,768)): + content = png(width,height,noisy=True) + sha = hashlib.sha256(content).hexdigest() + baseline = memory(ids) + baseline_kernel_peak = kernel_peak(ids) + samples = [] + barrier = threading.Barrier(11) + def worker(number): + local = Client(timeout=55) + barrier.wait() + return upload(local,alice,chat,content,filename='concurrent-'+str(number)+'.png') + started = time.monotonic() + with ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(worker,n) for n in range(10)] + barrier.wait() + while not all(f.done() for f in futures): + samples.append({'seconds':round(time.monotonic()-started,3), 'bytes':memory(ids)}) + time.sleep(.1) + responses = [f.result() for f in futures] + elapsed = time.monotonic()-started + samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)}) + for response in responses: + att = response['json'] + check(att['sha256']==sha,'concurrent upload checksum mismatch') + downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) + check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed') + latency = [r['elapsed']*1000 for r in responses] + peak = {container:max(sample['bytes'][container] for sample in samples) for container in ids} + run = {'width':width, 'height':height, 'size_bytes':len(content), 'sha256':sha, + 'requests':10, 'successful':len(responses), 'errors':0, 'duration_seconds':round(elapsed,3), + 'successful_rps':round(len(responses)/elapsed,3), + 'latency_ms':{str(p):round(percentile(latency,p),3) for p in (.5,.95,.99)}, + 'baseline_bytes':baseline, 'sampled_peak_bytes':peak, + 'kernel_lifetime_peak_before_bytes':baseline_kernel_peak, + 'kernel_lifetime_peak_after_bytes':kernel_peak(ids), + 'sampled_peak_sum_bytes':max(sum(sample['bytes'].values()) for sample in samples), + 'samples':samples} + check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit') + results['runs'].append(run) + print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True) + save('lab4-upload-memory.json',results) + print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.') + + +if __name__=='__main__': + main() diff --git a/scripts/swarm.py b/scripts/swarm.py index 8d5f444..a40d801 100644 --- a/scripts/swarm.py +++ b/scripts/swarm.py @@ -14,7 +14,11 @@ STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4') MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4') TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4') +S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z') +MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') PORT = os.environ.get('PORT', '8080') +SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key', + 's3_root_secret_key', 's3_access_key', 's3_secret_key') def docker(*args, capture=False, **kwargs): @@ -65,7 +69,7 @@ def wait_api(): def job(kind, image, command, integration=False): name = STACK+'_'+kind # Fixed service name is a deployment lock: concurrent runners fail to create it. - args = ['service', 'create', '--detach', '--name', name, '--mode', 'replicated-job', + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', '--constraint', 'node.labels.mtusi_lab4_storage==true', '--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m', @@ -73,15 +77,23 @@ def job(kind, image, command, integration=False): '--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password'] if integration: for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger', - 'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key']: + 'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key', + 'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments', + 'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']: args += ['--env', value] args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key'] + for secret in ('s3_access_key', 's3_secret_key'): + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] else: args += ['--env', 'GOOSE_DRIVER=postgres', '--env', 'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5'] docker(*args, image, *command) + wait_job(name, kind, 300 if integration else 90) + + +def wait_job(name, kind, timeout=90): try: - deadline = time.monotonic()+(300 if integration else 90) + deadline = time.monotonic()+timeout while time.monotonic() < deadline: ids = docker('service', 'ps', '-q', name, capture=True).split() states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids] @@ -97,7 +109,31 @@ def job(kind, image, command, integration=False): docker('service', 'rm', name) +def bootstrap_s3(): + name = STACK+'_s3-bootstrap' + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job', + '--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data', + '--constraint', 'node.labels.mtusi_lab4_storage==true', + '--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh', + '--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh', + '--config', 'source='+STACK+'_s3_policy,target=/policy.json'] + for secret in SECRETS[2:]: + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] + docker(*args, MC, '/bootstrap.sh') + wait_job(name, 'S3 bootstrap') + + +def reconcile(): + active = tasks('api') + if not active: + raise RuntimeError('no API task available for reconciliation') + container = active[0]['Status']['ContainerStatus']['ContainerID'] + docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:]) + + def build(): + docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.') + docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.') docker('build', '-t', API, '.') docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.') @@ -113,7 +149,7 @@ def up(): raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution') docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID']) subprocess.run(['./scripts/bootstrap.sh'], check=True) - for secret in ('postgres_password', 'cursor_key'): + for secret in SECRETS: known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split() if STACK+'_'+secret not in known: docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret) @@ -131,6 +167,8 @@ def up(): docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env) wait_service('postgres') wait_service('redis') + wait_service('minio') + bootstrap_s3() if fresh: job('migrations', MIGRATIONS, ['up']) docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2')) @@ -143,7 +181,7 @@ def up(): def restart(): # Force replaces every task while retaining the same Swarm secrets and volumes. - services = ('postgres', 'redis', 'api', 'lb') + services = ('postgres', 'redis', 'minio', 'api', 'lb') previous = {name: {t['ID'] for t in tasks(name)} for name in services} for service in services: docker('service', 'update', '--detach', '--force', STACK+'_'+service) @@ -155,6 +193,7 @@ def restart(): wait_service('lb') wait_service('postgres') wait_service('redis') + wait_service('minio') wait_api() @@ -171,7 +210,7 @@ def stop_one(): def update(): previous = {t['ID'] for t in tasks('api')} - docker('service', 'update', '--detach', '--force', '--image', API, STACK+'_api') + docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api') deadline = time.monotonic()+180 while time.monotonic() < deadline: state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State') @@ -187,7 +226,7 @@ def update(): def status(): result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}} - for name in ('lb','api','postgres','redis'): + for name in ('lb','api','postgres','redis','minio'): service = inspect('service', STACK+'_'+name) spec = service['Spec'] template = spec['TaskTemplate'] @@ -207,6 +246,8 @@ def main(): if action == 'status': status() elif action == 'up': up() elif action == 'build': build() + elif action == 'bootstrap-s3': bootstrap_s3() + elif action == 'reconcile': reconcile() elif action == 'wait': wait_api() elif action == 'restart': restart() elif action == 'stop-one': stop_one() diff --git a/stack.yaml b/stack.yaml index dc0d0c2..da59986 100644 --- a/stack.yaml +++ b/stack.yaml @@ -43,6 +43,25 @@ services: resources: limits: {cpus: '0.5', memory: 128M} update_config: {parallelism: 1, order: stop-first} + minio: + image: ${S3_IMAGE:-mtusi-minio:RELEASE.2025-09-07T16-13-09Z} + secrets: [s3_root_access_key, s3_root_secret_key] + volumes: [s3_data:/data] + networks: [data] + healthcheck: + test: [CMD, wget, -q, -T, '2', -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live'] + interval: 2s + timeout: 3s + start_period: 15s + retries: 15 + stop_grace_period: 20s + deploy: + replicas: 1 + placement: + constraints: [node.labels.mtusi_lab4_storage == true] + resources: + limits: {cpus: '1.0', memory: 512M} + update_config: {parallelism: 1, order: stop-first} api: image: ${API_IMAGE:-mtusi-messenger:lab4} environment: @@ -56,7 +75,14 @@ services: REDIS_ADDR: redis:6379 SESSION_TTL_SECONDS: '3600' QUERY_TIMEOUT_MS: '1000' - secrets: [postgres_password, cursor_key] + S3_ENDPOINT: minio:9000 + S3_BUCKET: attachments + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + S3_USE_SSL: 'false' + GOMEMLIMIT: 1152MiB + GOGC: '50' + secrets: [postgres_password, cursor_key, s3_access_key, s3_secret_key] networks: [backend, data] stop_grace_period: 15s healthcheck: @@ -69,7 +95,7 @@ services: replicas: ${API_REPLICAS:-2} endpoint_mode: dnsrr resources: - limits: {cpus: '1.0', memory: 256M} + limits: {cpus: '1.0', memory: 1536M} restart_policy: {condition: any, delay: 1s} update_config: parallelism: 1 @@ -107,11 +133,16 @@ networks: volumes: postgres_data: redis_data: + s3_data: configs: haproxy: file: ./infra/haproxy.cfg redis: file: ./infra/redis.conf + s3_bootstrap: + file: ./infra/s3-bootstrap.sh + s3_policy: + file: ./infra/s3-policy.json secrets: postgres_password: external: true @@ -119,3 +150,15 @@ secrets: cursor_key: external: true name: ${STACK_NAME:-mtusi-lab4}_cursor_key + s3_root_access_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_root_access_key + s3_root_secret_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_root_secret_key + s3_access_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_access_key + s3_secret_key: + external: true + name: ${STACK_NAME:-mtusi-lab4}_s3_secret_key diff --git a/tests/own/README.md b/tests/own/README.md index 5a7390e..b446401 100644 --- a/tests/own/README.md +++ b/tests/own/README.md @@ -1,14 +1,19 @@ -# Проверки базы из lab3 +# Собственные проверки lab4 -Эти проверки перенесены вместе с реализацией lab3. Они не подтверждают критерии оценки lab4. -Публичные HTTP-проверки используют текущий контракт lab4 и требуют реализации вложений. +`make test-lab4` проверяет реальные PNG/JPEG, лимиты, MIME, повреждения, права, приватность bucket и отказы. +Тест останавливает именно API-реплику, которая приняла файл, и проверяет скачивание через другую. +`make measure-uploads` запускает десять клиентов на двух размерах PNG и сохраняет замеры cgroup memory. +`make test-reconcile` внедряет ошибки S3/БД и проверяет grace, блокировки, dry-run, повторное исправление и стоимость обхода. +Эти сценарии используют реальный MinIO. Запускайте их последовательно на учебном Swarm-стенде. + +Результаты находятся в `evidence/lab4-*`. Подробная методика приведена в [REPORT.md](../../REPORT.md). Публичные tests/*.py и обязательный контракт сохранены. - `make test-own`: Go tests с race detector. Без INTEGRATION_TESTS=1 PostgreSQL-сценарий пропускается. - `make test-integration`: отдельный Swarm job, Go 1.26.4 и goose. Временная схема, миграции с нуля/повторно/legacy/reset/up, регистрационная гонка, rollback, 10 000 сообщений с одинаковыми timestamps, EXPLAIN и 16 конкурентных запросов одного ключа. - `make test-grade4`: `swarm_grade4.py`, ровно две API tasks. Проверяет общий ключ через LB, одинаковый результат, 409, границы ключа, область пользователя/чата, права и logout. По очереди останавливает обе исходные tasks с восстановлением, затем выполняет rolling update под нагрузкой. Проверяет историю, отсутствие дублей и повтор исходного ключа после воздействия. Сохраняет `evidence/swarm-grade4.json`. -- `make test-persistence`: перезапуск всех API, PostgreSQL, Redis и LB; volumes/secrets сохраняются. +- `make test-persistence`: перезапуск всех API, PostgreSQL, Redis, MinIO и LB; volumes/secrets сохраняются. - `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`: публичный сценарий отказа; `make wait` дожидается восстановления двух реплик. - `make measure-load`: 3s прогрева + 30s, concurrency 4, метрики и Docker stats в `evidence/swarm-*`. diff --git a/tests/own/fixtures/sample.jpg b/tests/own/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5dc1440705fab428cefe24e70611240bff9e8408 GIT binary patch literal 4140 zcmbtXYfKzf6rR}!yF5}HmR*n(Htf>kVrl}k;EF_x(nS)%F3<|UEKn*4CIzKRVzF3! z&`L5wwH0ha3&lk=@>;FGL|Y1^t*JcZ(LYseD=7(Rs}!qv?w!Zp*`4K4Ct;HvcFsNL zJKy)6Gs_dp)6`)~tyHO0N;Q0_)oLG2fK~%npufLw023G#!~`;o&ag2=7aXQ%n9#`3 zu<&rB(WnbCMVTU^HbxjDXn0KHquHRnydiP;XTQ!AcSswY!nSnv`q5_mOG)&1*Pf&N0?XB7E9ocoq{?@TKMvk9PuWCzk z*kg9RY%8;}=)MWQ=UT&iukV1{Sz|G&>6ZrQ+T*sRv+$%t|8SzYyLk}ZX6+{(xo6>r zu|miOb~>Z51V@lYOt$xwClz1JhtQcQMzkwNYj)?=`l$*t+=iG{ZG|(HC@U0P8UwOI zhoP*p>l*JsR=3(?QM#z2&Sr@5xLQbL$)CMfol&5-%yQ+p9)$1cgcD&xDv4%i^F{Hz zFKhz@CK(b;gdj4QL_-h~lSUUNO%hCqi0+u6MEDpOli{^tVwGYtM__Wbes~2;Tn_>! zMJr%ZZ@Z91@?`?i8pIT`tE{~nwf}vgJx%Q7T6rSHgy0EejI!c+Lg4Dj z6B3ivc+$ud%mWkDVv;9POjghiVyS=$I-O@a%SnR3WSStXt2rJ_C)7g3a3HQD{S|OB zlAyp9lZ8o-G!6Q5HV<$z;$zI~c_??^e8jf!kM$^L5(G1l?C=R#;aGHtVDZ9vc~)aM zQGvh=>?l%tai?P$Nh$#S4}qxTkkyFl_mYSxD^dL?opni4i?rPBzEx<{xM< zC2S!FYi6pAgfU3XQYaNyghUXJ91%bI8M`nFUVzTV=*jUOlY{q1R1Nl6lJ^2@a4J0S zdDoWbz35Ob=1EusYUB{Yg8)IfW~;=QV54@&B*W&_7*G#fn4FeiBJTwl6Kop!*xx-c z5$eI1fTK$sid3UVbG%>z4InV#l!SxJD9LSn5SYB_&J(Yg$nu0A&1;^c#4P4#AX0(A z6R(67+=b`KeL0kXE7lHbfOMo1mCWNx^KyM_E5B{5k&5%)X zT)_GN`ia;cIqit4NO*!gmexA72@+2tof z439MfWkqObJv_N3cr5gFY;bWFZyNYWapT2PNFE`9Z93|a_UnlTf6Zig1 zwD=8Igv!4=xbMkc^-T;Dp38Va!Nc)(hccS%M$*!8Cd#*I7A|-vJamV&76YIsI%MysfmrucwnN zU!uwlt$CMvoK1S*T5BHr(&o!krD*#xoWwz5;GDER(|Fk@y%QaLz=X=I7XXUq_}ggZ zg4jhkwmrrU$2aiD`)5jGCtMybok!W$wts%GX7XM&B;$G@1{7ZFF!Yqpo$2S3Nq}UQ GC;tPsw=*69 literal 0 HcmV?d00001 diff --git a/tests/own/helpers.py b/tests/own/helpers.py new file mode 100644 index 0000000..ca4dcfb --- /dev/null +++ b/tests/own/helpers.py @@ -0,0 +1,116 @@ +"""Local lab4 experiment helpers. Never print credentials or session tokens.""" +import json +import os +from pathlib import Path +import random +import struct +import subprocess +import sys +import time +import uuid +import zlib + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + +STACK = os.environ.get('STACK_NAME', 'mtusi-lab4') +MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z') + + +def docker(*args, data=None, check_result=True): + return subprocess.run(['docker', *args], input=data, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=check_result) + + +def containers(service): + ids = docker('ps', '--filter', 'label=com.docker.swarm.service.name='+STACK+'_'+service, + '--format', '{{.ID}}').stdout.decode().split() + check(bool(ids), 'no container for '+service) + return ids + + +def sql(statement): + result = docker('exec', '-i', containers('postgres')[0], 'psql', '-U', 'messenger', + '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', data=statement.encode()) + return result.stdout.decode().strip() + + +def reconcile(apply=False, grace='90s'): + args = ['exec', containers('api')[0], '/usr/local/bin/reconcile', '--grace='+grace] + if apply: + args.append('--apply') + return json.loads(docker(*args).stdout) + + +def upload(c, actor, chat, content, filename='sample.png', mime='image/png', expected=201): + from urllib.parse import quote + return c.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+quote(filename, safe=''), + expected, actor=actor, raw=content, headers={'Content-Type': mime}, + schema='Attachment' if expected == 201 else None) + + +def png(width=320, height=200, noisy=False): + def chunk(kind, body): + return struct.pack('>I', len(body))+kind+body+struct.pack('>I', zlib.crc32(kind+body)) + rng = random.Random(419) + lines = b''.join(b'\0'+(rng.randbytes(width*3) if noisy else b'\x18\x60\xc0'*width) + for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + +chunk(b'IDAT', zlib.compress(lines))+chunk(b'IEND', b'')) + + +def oversized_dimensions(): + body = struct.pack('>IIBBBBB', 10000, 4001, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n'+struct.pack('>I', len(body))+b'IHDR'+body+struct.pack('>I', zlib.crc32(b'IHDR'+body)) + + +def save(name, result): + (ROOT / 'evidence').mkdir(exist_ok=True) + (ROOT / 'evidence' / name).write_text(json.dumps(result, indent=2, ensure_ascii=False)+'\n') + + +def wait_ready(timeout=90): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + try: + Client(timeout=2).call('GET', '/health/ready', schema='Health') + return + except AssertionError: + time.sleep(.5) + raise AssertionError('readiness did not recover') + + +class Probe: + """Short-lived mc service; root secrets remain inside Docker secrets.""" + def __enter__(self): + self.name = STACK+'_probe_'+uuid.uuid4().hex[:8] + args = ['service', 'create', '--detach', '--no-resolve-image', '--name', self.name, '--network', STACK+'_data', + '--restart-condition', 'none', '--limit-memory', '512m', '--limit-cpu', '0.5', + '--entrypoint', '/bin/sh'] + for secret in ('s3_root_access_key', 's3_root_secret_key', 's3_access_key', 's3_secret_key'): + args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret] + docker(*args, MC, '-c', 'sleep 3600') + deadline = time.monotonic()+45 + while time.monotonic() < deadline: + ids = docker('ps', '--filter', 'label=com.docker.swarm.service.name='+self.name, '-q').stdout.decode().split() + if ids: + self.container = ids[0] + return self + time.sleep(.3) + docker('service', 'rm', self.name, check_result=False) + raise AssertionError('probe did not start') + + def mc(self, *args, data=None, admin=False, expected=0): + access = 's3_root_access_key' if admin else 's3_access_key' + secret = 's3_root_secret_key' if admin else 's3_secret_key' + script = ('set -eu; export MC_CONFIG_DIR=/tmp/mc; ' + 'mc alias set local http://minio:9000 "$(cat /run/secrets/'+access+')" ' + '"$(cat /run/secrets/'+secret+')" >/dev/null; exec mc "$@"') + result = docker('exec', '-i', self.container, '/bin/sh', '-c', script, 'sh', *args, + data=data, check_result=False) + check((result.returncode == 0) == (expected == 0), 'unexpected mc outcome: '+args[0]) + return result + + def __exit__(self, *args): + docker('service', 'rm', self.name, check_result=False) diff --git a/tests/own/lab4.py b/tests/own/lab4.py new file mode 100644 index 0000000..8622723 --- /dev/null +++ b/tests/own/lab4.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Real S3/API negative cases and failures; only synthetic lab data.""" +import hashlib +import http.client +import json +import os +from pathlib import Path +import subprocess +import time +import uuid +from urllib.parse import urlsplit + +from helpers import Client, Probe, ROOT, STACK, check, containers, docker, png, oversized_dimensions, save, sql, upload, wait_ready + + +def main(): + c = Client(timeout=55) + alice, bob, eve, chat = c.bootstrap() + data = png() + result = {'negative_cases': [], 'failures': {}, 'checksums': {}} + for name, body, mime, filename, status in ( + ('oversized_bytes', b'x'*(10*1024*1024+1), 'image/png', 'large.png', 413), + ('damaged_png', data[:50], 'image/png', 'broken.png', 415), + ('corrupt_crc', data[:-16]+b'\xff'*16, 'image/png', 'crc.png', 415), + ('mime_spoof', data, 'image/jpeg', 'fake.jpg', 415), + ('not_image', b'plain text', 'image/png', 'fake.png', 415), + ('unsupported_mime', data, 'application/octet-stream', 'file.png', 415), + ('pixel_budget', oversized_dimensions(), 'image/png', 'large-dimensions.png', 413), + ('unsafe_slash', data, 'image/png', '../file.png', 400), + ('unsafe_backslash', data, 'image/png', 'x\\y.png', 400), + ('unsafe_control', data, 'image/png', 'x\x00.png', 400), + ('empty_filename', data, 'image/png', '', 400), + ): + upload(c, alice, chat, body, filename, mime, status) + result['negative_cases'].append({'case': name, 'status': status}) + for content, filename, mime in ((data, 'изображение.png', 'image/png'), + ((ROOT/'tests/own/fixtures/sample.jpg').read_bytes(), 'photo.jpg', 'image/jpeg')): + att = upload(c, alice, chat, content, filename, mime)['json'] + downloaded = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob) + check(downloaded['bytes'] == content, 'original bytes changed') + check(att['sha256'] == hashlib.sha256(content).hexdigest(), 'incorrect SHA256') + check(downloaded['headers'].get_content_type() == mime, 'incorrect content MIME') + result['checksums'][mime] = att['sha256'] + att = upload(c, alice, chat, data)['json'] + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/'+str(uuid.uuid4())+suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/'+att['id']+suffix, 401) + upload(c, eve, chat, data, expected=404) + c.call('GET', '/api/v1/attachments/'+att['id']+'/content?variant=thumbnail', 404, actor=bob) + other = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title':'other chat', 'member_ids':[alice['id'], bob['id']]}, schema='Chat')['json'] + path = '/api/v1/chats/'+other['id']+'/messages' + for ids, status in (([att['id']], 404), ([str(uuid.uuid4())], 404), ([att['id'],att['id']], 400)): + c.call('POST', path, status, actor=alice, body={'text':'', 'attachment_ids':ids}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + result['negative_cases'] += [{'case': x, 'status': s} for x,s in ( + ('foreign_attachment',404), ('foreign_chat_link',404), ('unknown_attachment',404), + ('duplicate_ids',400), ('missing_authentication',401), ('unknown_variant',404))] + + # The proxy must stop oversized chunked bodies even without Content-Length. + address = urlsplit(c.base) + conn = http.client.HTTPConnection(address.hostname, address.port, timeout=55) + conn.request('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png', + body=iter([b'x'*65536]*161), encode_chunked=True, + headers={'Authorization':'Bearer '+alice['token'], 'Content-Type':'image/png'}) + response = conn.getresponse() + check(response.status == 413, 'chunked upload bypassed byte limit') + response.read(); conn.close() + result['negative_cases'].append({'case':'chunked_oversized', 'status':413}) + + # Disconnect halfway through the declared body. No object/metadata can be published. + before = int(sql('SELECT count(*) FROM attachments;')) + conn = http.client.HTTPConnection(address.hostname, address.port, timeout=5) + conn.putrequest('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=aborted.png') + conn.putheader('Authorization', 'Bearer '+alice['token']) + conn.putheader('Content-Type', 'image/png') + conn.putheader('Content-Length', str(len(data)+10000)) + conn.endheaders(); conn.send(data[:40]); conn.close() + time.sleep(1) + check(int(sql('SELECT count(*) FROM attachments;')) == before, 'aborted upload published metadata') + leftovers = [docker('exec', container, 'find', '/tmp', '-maxdepth', '1', '-name', 'messenger-upload-*').stdout.decode() + for container in containers('api')] + check(not any(leftovers), 'aborted upload left spool files') + result['failures']['client_disconnect'] = {'metadata_added':0, 'spool_files_remaining':0} + + with Probe() as probe: + key = sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"';") + raw = docker('exec', containers('minio')[0], 'wget', '-S', '-O', '/dev/null', + 'http://127.0.0.1:9000/attachments/'+key, check_result=False) + check(b'403' in raw.stderr, 'unauthenticated direct S3 GET was not forbidden') + probe.mc('stat', 'local/attachments/'+key) + probe.mc('mb', 'local/forbidden-'+uuid.uuid4().hex[:8], expected=1) + probe.mc('pipe', 'local/attachments/outside-prefix', data=b'x', expected=1) + result['bucket_security'] = {'anonymous_get_status':403, 'create_bucket_denied':True, + 'write_outside_prefix_denied':True, 'console_published':False} + service = json.loads(docker('service', 'inspect', STACK+'_minio').stdout)[0] + check(not service['Spec'].get('EndpointSpec', {}).get('Ports'), 'MinIO publishes an external port') + for _ in range(2): + subprocess.run(['python3', str(ROOT/'scripts/swarm.py'), 'bootstrap-s3'], check=True, + stdout=subprocess.DEVNULL) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data, + 'repeated bucket bootstrap changed data') + result['bucket_security']['bootstrap_repeats'] = 2 + + # Stop the replica that accepted THIS upload, then read through another one. + accepted = upload(c,alice,chat,data) + source = accepted['headers']['X-Instance-Id'] + env = dict(os.environ, TARGET_INSTANCE_ID=source) + started = time.monotonic() + subprocess.run(['python3',str(ROOT/'scripts/swarm.py'),'stop-one'],env=env,check=True, + stdout=subprocess.DEVNULL) + deadline = time.monotonic()+10 + while True: + try: + downloaded = Client(timeout=2).call('GET','/api/v1/attachments/'+accepted['json']['id']+'/content',actor=bob) + check(downloaded['headers']['X-Instance-Id']!=source,'stopped upload replica still serves bytes') + check(downloaded['bytes']==data,'cross-replica original changed') + break + except AssertionError: + check(time.monotonic()0 + + # Keep one old linked object and remove a second original behind PostgreSQL. + live = upload(c,alice,chat,data)['json'] + c.message(alice,chat,'keep live original',attachments=[live['id']]) + missing = upload(c,alice,chat,data)['json'] + message = c.message(alice,chat,'keep missing reference',attachments=[missing['id']])['json'] + missing_key = sql("SELECT object_key FROM attachments WHERE id='"+missing['id']+"';") + probe.mc('rm','local/attachments/'+missing_key,admin=True) + current = reconcile(apply=True) + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='ready','fresh metadata changed before grace') + result['fresh_metadata_preserved'] = current['fresh_skipped']>0 + + # Measure a real scan at two bucket sizes, crossing an S3 page boundary. + cost_prefix = 'original/cost-'+experiment+'/' + try: + for group,count,total in (('small',100,100),('extra',1000,1100)): + script = ('mkdir -p /tmp/cost/"$1"; i=1; while [ "$i" -le "$2" ]; do ' + 'printf x > /tmp/cost/"$1"/"$i"; i=$((i+1)); done') + docker('exec',probe.container,'/bin/sh','-c',script,'sh',group,str(count)) + probe.mc('mirror','--quiet','/tmp/cost/'+group,'local/attachments/'+cost_prefix+group,admin=True) + summary = reconcile() + check(summary['objects_scanned']>=total,'scan skipped S3 objects') + result['cost'].append({'phase':'fresh', 'synthetic_objects':total, **summary}) + print('Fresh scan cost:',json.dumps({k:summary[k] for k in ('objects_scanned','metadata_scanned','list_requests_estimate','stat_requests','db_queries','duration_ms')}),flush=True) + print('Waiting92s for actual S3 timestamps: measuring aged-object locking, DB and HEAD cost next.',flush=True) + time.sleep(92) + for total in (1100,100): + summary = reconcile() + check(summary['orphans']>=total,'aged scan did not inspect all orphan fixtures') + result['cost'].append({'phase':'aged', 'synthetic_objects':total, **summary}) + print('Aged scan cost:',json.dumps({k:summary[k] for k in ('objects_scanned','metadata_scanned','list_requests_estimate','stat_requests','db_queries','duration_ms')}),flush=True) + if total==1100: + probe.mc('rm','--recursive','--force','local/attachments/'+cost_prefix+'extra/',admin=True) + finally: + probe.mc('rm','--recursive','--force','local/attachments/'+cost_prefix,admin=True) + + # S3/DB unavailability must stop reconciliation without destructive guesses. + for service in ('minio','postgres'): + try: + docker('service','scale','--detach',STACK+'_'+service+'=0') + deadline=time.monotonic()+30 + while time.monotonic()0: + print('Waiting for actual S3 object age to exceed90s grace:',round(remaining,1),'seconds',flush=True) + time.sleep(remaining) + sql("UPDATE attachments SET created_at=clock_timestamp()-interval '10 minutes' WHERE id IN ('"+live['id']+"','"+missing['id']+"');") + + # A second process holds the upload lock: even an old orphan is protected. + lock_script = "BEGIN; SELECT pg_advisory_xact_lock(hashtextextended('"+orphan+"',0));\n" + lock = subprocess.Popen(['docker','exec','-i',containers('postgres')[0],'psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + lock.stdin.write(lock_script.encode()); lock.stdin.flush() + try: + deadline = time.monotonic()+10 + while sql("SELECT pg_try_advisory_xact_lock(hashtextextended('"+orphan+"',0));") != 'f': + check(time.monotonic()=1 and orphan in objects(probe),'upload lock did not protect orphan') + finally: + lock.stdin.write(b'ROLLBACK;\n'); lock.stdin.close() + lock.wait(timeout=15) + result['locked_candidate_preserved'] = True + # The apply above may mark the aged missing row; restore only its internal state + # to compare the following dry-run with an unchanged, well-defined fixture. + sql("UPDATE attachments SET status='ready',error_code=NULL WHERE id='"+missing['id']+"';") + dry = reconcile() + check(dry['orphans']>=1 and dry['missing']>=1,'dry-run did not detect injected inconsistencies') + check(orphan in objects(probe),'dry-run deleted an object') + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='ready','dry-run modified metadata') + applied = reconcile(apply=True) + check(orphan not in objects(probe),'apply did not delete old orphan') + check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"';")=='failed','missing original not marked failed') + check(sql("SELECT count(*) FROM messages WHERE id='"+message['id']+"' AND '"+missing['id']+"'::uuid=ANY(attachment_ids);")=='1','repair deleted linked metadata or reference') + c.call('GET','/api/v1/attachments/'+missing['id'],409,actor=alice) + check(c.call('GET','/api/v1/attachments/'+live['id']+'/content',actor=bob)['bytes']==data,'repair deleted a live linked original') + repeated = reconcile(apply=True) + check(repeated['deleted']==0 and repeated['marked_failed']==0,'repeat apply was not idempotent') + result.update({'dry_run':dry, 'apply':applied, 'repeat_apply':repeated, + 'live_linked_preserved':True, 'missing_linked_reference_preserved':True}) + save('lab4-reconciliation.json',result) + print('PASS: real two-store faults, grace, lock, dry-run/apply/repeat, linked data preserved and paginated scan cost.') + + +if __name__=='__main__': + main()