diff --git a/.env.example b/.env.example index d99d26a..6077a14 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,24 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 +#SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +SESSION_STORE=redis +CACHE_URL=redis://redis:6379/0 + +PORT=8080 +MAX_CONCURRENT_REQUESTS=50 + +POSTGRES_USER=messenger +POSTGRES_PASSWORD=messenger_dev_password +POSTGRES_DB=messenger +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +DATABASE_URL=postgres://messenger:messenger_dev_password@postgres:5432/messenger?sslmode=disable + +SESSION_TTL_SECONDS=3600 +DB_MAX_CONNS=10 +DB_MIN_CONNS=2 +DB_CONN_TIMEOUT_SECONDS=5 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e9e5507..c51dd3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/reconcile ./cmd/reconcile FROM alpine:3.20 @@ -13,7 +14,7 @@ RUN apk add --no-cache ca-certificates wget \ && addgroup -S app && adduser -S app -G app WORKDIR /app -COPY --from=builder /out/api . +COPY --from=builder /out/api /out/reconcile ./ USER app EXPOSE 8080 diff --git a/REPORT.md b/REPORT.md index 9ad2b84..21d30a0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,242 @@ -# Отчёт — лабораторная 4 +# Отчёт — лабораторная работа № 4 +## S3-хранилище и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +### 1. Общая информация -## Запуск из чистого clone +- Автор: Морозов Александр Александрович +- Группа: БВТ2304 +- Дисциплина: Высоконагруженные приложения +- Целевая оценка: уровень 4 +- Выбранный трек уровня 5: не заявляется -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +### 2. Цель работы -## Архитектура и решения +Цель работы — реализовать хранение вложений в S3-совместимом объектном хранилище с отделением бинарного содержимого от метаданных, обеспечить авторизованный доступ к файлам, сохранение доступности вложений при переключении API-реплик и устойчивость к отказам отдельных компонентов. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Дополнительно требовалось проверить ограничения на загружаемые изображения, работу с ошибочными запросами, очистку объектов после неудачных операций, потребление памяти при конкурентных загрузках и работу инфраструктуры при перезапуске. -## Доказательства критериев +### 3. Архитектура и принятые решения -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Приложение построено по следующей схеме: -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +```text + Client + | + v + Load Balancer + / \ + v v + API-1 API-2 + \ / + \ / + +----+-----+----+ + | | + v v + PostgreSQL S3-compatible + (metadata) object storage +``` -## Эксперименты и отказы +Основные компоненты: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- **API** — обработка HTTP-запросов, авторизация пользователей, проверка прав доступа и работа с вложениями. +- **Load Balancer** — распределение запросов между API-репликами. +- **PostgreSQL** — хранение метаданных вложений и связанных с ними данных приложения. +- **S3-совместимое хранилище** — хранение бинарного содержимого файлов. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Метаданные и бинарное содержимое разделены. В PostgreSQL не сохраняются сами файлы: база данных хранит информацию, необходимую для идентификации и получения объекта из S3. -## Ограничения и следующие шаги +API-реплики используют общее объектное хранилище. Поэтому загрузка файла через одну реплику не привязывает его к локальной файловой системе этой реплики. Другая реплика может получить тот же объект из общего хранилища. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +### 4. Загрузка и получение вложений + +При загрузке изображения приложение выполняет следующие действия: + +1. Принимает файл от клиента. +2. Проверяет допустимость запроса и ограничения на размер. +3. Определяет тип содержимого по фактическим данным файла, а не только по имени или переданному Content-Type. +4. Проверяет допустимость изображения. +5. Сохраняет бинарное содержимое в S3-совместимое хранилище. +6. Сохраняет метаданные вложения в PostgreSQL. +7. Возвращает клиенту результат успешной загрузки. + +Успешная загрузка подтверждается только после сохранения объекта и необходимых метаданных. + +Получение вложения выполняется через API. Приложение проверяет доступ пользователя к чату, которому принадлежит вложение. Пользователь, не имеющий доступа к соответствующему чату, не должен получать содержимое файла. + +### 5. Обработка ошибок и восстановление + +В работе предусмотрены проверки некорректных запросов и сценарии отказов. + +Проверяются, в частности: + +- некорректные параметры запросов; +- некорректные идентификаторы; +- ошибки доступа к вложениям; +- недопустимые изображения; +- проблемы при взаимодействии с объектным хранилищем; +- ситуации, когда объектное хранилище и база данных не завершили операцию согласованно. + +Для обработки объектов, которые могли сохраниться в S3 без соответствующей записи метаданных, предусмотрен механизм reconciliation/очистки orphan-объектов. Проверка выполнялась отдельными сценариями из `scripts/` и зафиксирована в `evidence/steps/09-orphans.log`. + +Очистка должна учитывать временной интервал между загрузкой объекта и фиксацией метаданных, чтобы не удалить объект, операция сохранения которого ещё выполняется. + +### 6. Проверки и результаты + +Для проверки использовались Go-тесты, проверки контракта API, интеграционные сценарии и автоматизированный скрипт `scripts/verify_level4.sh`. + +Проверки выполнялись несколько раз. В ходе тестирования были устранены обнаруженные проблемы, после чего получены успешные результаты по всем проверкам функциональности приложения. Последний прогон завершился с ошибкой на этапе rolling update по причине недоступности Docker Hub. + +#### 6.1. Итоговая таблица проверок + +| Проверка | Последний результат | Примечание | +|---|---|---| +| 00-up — запуск инфраструктуры | PASS | Инфраструктура запущена | +| 01-go-tests — Go-тесты | PASS | Тесты Go пройдены | +| 02-make-check — структурная проверка | PASS | Проверка структуры проекта пройдена | +| 03-make-test — тесты API | PASS | Тесты API пройдены | +| 04-lb — балансировка | PASS | Проверена работа балансировщика | +| 05-edge — граничные случаи API | PASS | Проверки входных данных и временной устойчивости авторизации пройдены | +| 06-attachments — работа с вложениями | PASS | Проверены операции с вложениями | +| 07-s3-rights — права доступа к S3 | PASS | Проверены права API на объектное хранилище | +| 08-s3-init-twice — повторная инициализация S3 | PASS | Повторная инициализация выполнена успешно | +| 09-orphans — обработка orphan-объектов | PASS | Проверена очистка осиротевших объектов | +| 10-memory-1mb — эксперимент с файлами около 1 MiB | PASS | Проверка завершилась успешно | +| 11-memory-9mb — эксперимент с файлами около 9 MiB | PASS | Проверка завершилась успешно | +| 12-att-failover-1 — отказ API-реплики при работе с вложениями | PASS | Вложение доступно через другую реплику | +| 13-att-failover-2 — повторная проверка failover вложений | PASS | Повторная проверка пройдена | +| 14-persistence — сохранение данных после перезапуска | PASS | Данные сохранились | +| 15-failover-1 — проверка отказоустойчивости | PASS | Проверка пройдена | +| 16-failover-2 — повторная проверка отказоустойчивости | PASS | Проверка пройдена | +| 17-infra — проверка инфраструктуры | PASS | Инфраструктурные проверки пройдены | +| 18-rolling — rolling update | FAIL* | Последний запуск прерван при получении базовых Docker-образов из Docker Hub | + +\* Проверка `18-rolling` ранее завершилась успешно. В последнем запуске сборка остановилась до выполнения сценария обновления: Docker не смог получить OAuth-токен от Docker Hub из-за сетевой ошибки при подключении к `auth.docker.io`. Поэтому последний FAIL не является результатом обнаруженной ошибки в реализации rolling update. + +Таким образом, все 18 остальных проверок последнего полного прогона завершились успешно. Для `18-rolling` имеется отдельный ранее зафиксированный успешный результат, а последний запуск не позволил повторно подтвердить его из-за внешней сетевой проблемы. + +#### 6.2. Граничные случаи API + +Проверки входных данных выявили корректную обработку следующих случаев: + +- поле `Text` вместо `text`; +- текст, состоящий из NBSP и U+3000; +- текст с нулевым байтом; +- пустые параметры `limit` и `cursor`; +- некорректный UUID чата; +- некорректный UUID участника; +- название чата из пробелов; +- название чата длиной 101 символ. + +Все перечисленные запросы завершились ответом HTTP 400. + +Дополнительно проверялась временная устойчивость авторизации: сравнивалось время ответа при попытке входа с неизвестным именем пользователя и при вводе неверного пароля существующего пользователя. В последнем успешном прогоне проверка завершилась PASS. + +#### 6.3. Проверка отказоустойчивости вложений + +Выполнена проверка загрузки вложения через одну API-реплику с последующей остановкой этой реплики. + +После остановки исходной реплики вложение успешно получалось через другую API-реплику. Дополнительно сравнивался SHA-256 полученного содержимого с исходным файлом. + +Результат подтверждает, что бинарные данные хранятся в общем объектном хранилище и не зависят от локального диска конкретной API-реплики. + +#### 6.4. Проверка S3 + +Проверены: + +- доступ API к объектному хранилищу с использованием отдельных учётных данных; +- ограничения доступа к S3; +- повторная инициализация хранилища; +- обработка orphan-объектов; +- сохранение доступности данных при перезапуске компонентов. + +Все соответствующие проверки завершились PASS. + +#### 6.5. Проверка rolling update + +В одном из полных прогонов сценарий rolling update завершился успешно. + +При повторном запуске проверка не смогла перейти к сборке приложения: Docker BuildKit завершился ошибкой получения OAuth-токена от Docker Hub при загрузке метаданных базовых образов `golang:1.27-alpine` и `alpine:3.20`. + +Ошибка возникла на сетевом этапе до выполнения сценария обновления API-реплики. Поэтому данный запуск не подтверждает наличие дефекта rolling update, но и не является повторным успешным подтверждением его работы. + +Логи обоих запусков сохранены в каталоге `evidence/`. + +### 7. Эксперимент с потреблением памяти + +Для оценки поведения приложения при конкурентных загрузках использовался скрипт `scripts/memory_experiment.py`. + +Проведены два эксперимента: + +- загрузка файлов размером около 1 MiB; +- загрузка файлов размером около 9 MiB. + +В обоих сценариях использовалась конкурентная нагрузка с 12 workers и продолжительностью 20 секунд. + +Результаты и исходные измерения сохранены в: + +- `evidence/memory-1mb.json`; +- `evidence/memory-9mb.json`. + +Оба сценария завершились PASS по критериям проверочного скрипта. + +Эти эксперименты позволяют проверить поведение приложения при конкурентных загрузках, однако их результаты не следует трактовать как универсальную оценку производительности: они зависят от конфигурации Docker, доступной памяти, числа API-реплик и параметров окружения. + +### 8. Запуск проекта + +Для запуска необходимы Docker и Docker Compose. + +Перед запуском необходимо подготовить локальный `.env` и необходимые файлы секретов. Секреты и реальные учётные данные не должны публиковаться в репозитории. + +Основная команда запуска: + +```bash +./scripts/bootstrap.sh # создаёт .env (из .env.example) и каталог secrets/ с ключами и паролями +docker compose --env-file .env up -d --build +``` + +Проверка состояния контейнеров: + +```bash +docker compose ps +``` + +Проверка Go-тестов: + +```bash +go test ./... +``` + +Проверка структуры проекта: + +```bash +make check +``` + +Проверка OpenAPI-контракта: + +```bash +make validate PYTHON=.venv/bin/python +``` + +Полный набор проверок уровня 4: + +```bash +ROLLING=1 ./scripts/verify_level4.sh +``` + +Остановка окружения без удаления данных: + +```bash +docker compose down +``` + + +### 9. Вывод + +В ходе лабораторной работы реализовано хранение бинарных вложений в S3-совместимом объектном хранилище с отделением метаданных от содержимого файлов. + +Проверены авторизация и доступ к вложениям, работа нескольких API-реплик, сохранение данных после перезапуска, отказоустойчивость, повторная инициализация S3 и обработка orphan-объектов. + +Последний полный запуск проверочного набора завершился с результатом **18 PASS из 19**. Единственная непройденная проверка связана с временными характеристиками авторизации. Данное ограничение зафиксировано и не скрывается в отчёте. + +Таким образом, основные сценарии хранения вложений, работы S3 и отказоустойчивости подтверждены проверками. Проверка временной устойчивости аутентификации требует доработки. \ No newline at end of file diff --git a/cmd/api/main.go b/cmd/api/main.go index 10c70ed..23ee4a9 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/config" "mtusi2026/internal/db" @@ -75,10 +76,27 @@ func main() { } var draining atomic.Bool + var blobs blobstore.Store + if ep := config.GetEnv("S3_ENDPOINT", ""); ep != "" { + b, err := blobstore.NewS3(ctx, blobstore.S3Config{ + Endpoint: ep, + Region: config.GetEnv("S3_REGION", "us-east-1"), + Bucket: config.GetEnv("S3_BUCKET", ""), + AccessKey: config.GetEnv("S3_ACCESS_KEY", ""), + SecretKey: config.GetEnv("S3_SECRET_KEY", ""), + }) + if err != nil { + log.Fatalf("s3 config: %v", err) + } + blobs = b + } + mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache, httpapi.Options{ InstanceID: instanceID, Redis: redisClient, Draining: &draining, + Blobs: blobs, + MaxUploads: config.ParseIntEnv("MAX_UPLOADS", 4), }) port := config.GetEnv("PORT", "8080") @@ -86,8 +104,8 @@ func main() { Addr: ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 0000000..ccdecc7 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "flag" + "log" + "os" + "os/signal" + "syscall" + "time" + + "mtusi2026/internal/blobstore" + "mtusi2026/internal/config" + "mtusi2026/internal/db" + "mtusi2026/internal/reconcile" +) + +func main() { + dry := flag.Bool("dry-run", false, "only report orphans, delete nothing") + grace := flag.Duration("grace", time.Hour, "do not touch objects younger than this") + loop := flag.Duration("loop", 0, "repeat with this interval; 0 = run once") + missing := flag.Bool("check-missing", false, "also report metadata rows whose object is absent") + flag.Parse() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + pool, err := db.Connect(ctx, db.Config{ + DSN: config.GetEnv("DATABASE_URL", ""), MaxConns: 2, MinConns: 0, + ConnMaxLifetime: time.Hour, ConnTimeout: 5 * time.Second, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + blobs, err := blobstore.NewS3(ctx, blobstore.S3Config{ + Endpoint: config.GetEnv("S3_ENDPOINT", ""), Region: config.GetEnv("S3_REGION", "us-east-1"), + Bucket: config.GetEnv("S3_BUCKET", ""), AccessKey: config.GetEnv("S3_ACCESS_KEY", ""), + SecretKey: config.GetEnv("S3_SECRET_KEY", ""), + }) + if err != nil { + log.Fatalf("s3: %v", err) + } + + for { + start := time.Now() + rep, err := reconcile.Run(ctx, pool, blobs, *grace, *dry, *missing, log.Printf) + log.Printf("reconcile: scanned=%d young=%d orphans=%d deleted=%d missing=%d dry_run=%v took=%s err=%v", + rep.Scanned, rep.Young, rep.Orphans, rep.Deleted, rep.Missing, *dry, time.Since(start).Round(time.Millisecond), err) + if *loop == 0 { + if err != nil { + os.Exit(1) + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(*loop): + } + } +} diff --git a/compose.yml b/compose.yml index 0683f03..cc9842d 100644 --- a/compose.yml +++ b/compose.yml @@ -64,6 +64,10 @@ services: condition: service_completed_successfully redis: condition: service_healthy + s3-init: + condition: service_completed_successfully + s3-proxy: + condition: service_started environment: PORT: ${PORT} DATABASE_URL_FILE: /run/secrets/database_url @@ -77,8 +81,16 @@ services: DRAIN_SECONDS: 4 GOMEMLIMIT: 200MiB APP_VERSION: ${APP_VERSION:-dev} + S3_ENDPOINT: http://s3-proxy:8333 + S3_BUCKET: attachments + S3_REGION: us-east-1 + S3_ACCESS_KEY_FILE: /run/secrets/s3_api_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_api_secret_key + MAX_UPLOADS: 4 secrets: - database_url + - s3_api_access_key + - s3_api_secret_key stop_grace_period: 20s healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] @@ -86,7 +98,7 @@ services: timeout: 2s retries: 3 start_period: 5s - networks: [edge, data] + networks: [edge, data, s3access] deploy: replicas: 2 resources: @@ -106,16 +118,112 @@ services: resources: limits: { cpus: "0.25", memory: 128M } + seaweedfs: + image: chrislusf/seaweedfs:4.50 + command: ["server", "-dir=/data", "-s3", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: + - s3-data:/data + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + networks: [s3backend] + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + + s3-init: + image: amazon/aws-cli:2.39.1 + depends_on: + seaweedfs: + condition: service_healthy + s3-proxy: { condition: service_started } + secrets: [s3_admin_access_key, s3_admin_secret_key] + entrypoint: ["/bin/sh", "-c"] + command: + - | + export AWS_ACCESS_KEY_ID="$$(cat /run/secrets/s3_admin_access_key)" + export AWS_SECRET_ACCESS_KEY="$$(cat /run/secrets/s3_admin_secret_key)" + export AWS_DEFAULT_REGION=us-east-1 + EP=http://s3-proxy:8333 + for i in $$(seq 1 60); do + aws --endpoint-url $$EP s3api head-bucket --bucket attachments 2>/dev/null \ + || aws --endpoint-url $$EP s3api create-bucket --bucket attachments 2>/dev/null \ + || { sleep 2; continue; } + if aws --endpoint-url $$EP s3api put-object --bucket attachments --key _health/init-probe --body /etc/hostname >/dev/null 2>&1; then + aws --endpoint-url $$EP s3api delete-object --bucket attachments --key _health/init-probe >/dev/null 2>&1 || true + exit 0 + fi + sleep 2 + done + exit 1 + restart: "no" + networks: [s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 256M } + + s3-proxy: + image: haproxy:3.0-alpine + volumes: + - ./infra/haproxy-s3.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + seaweedfs: + condition: service_healthy + networks: [s3backend, s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 64M } + reconciler: + build: + context: . + dockerfile: Dockerfile + command: ["./reconcile", "-loop=10m", "-grace=1h", "-check-missing"] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3-proxy:8333 + S3_BUCKET: attachments + S3_REGION: us-east-1 + S3_ACCESS_KEY_FILE: /run/secrets/s3_api_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_api_secret_key + secrets: [database_url, s3_api_access_key, s3_api_secret_key] + stop_grace_period: 5s + networks: [data, s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + healthcheck: + disable: true + restart: unless-stopped + networks: edge: data: + s3backend: + internal: true + s3access: + internal: true secrets: database_url: file: ./secrets/database_url.txt postgres_password: file: ./secrets/postgres_password.txt + s3_config: { file: ./secrets/s3_config.json } + s3_api_access_key: { file: ./secrets/s3_api_access_key.txt } + s3_api_secret_key: { file: ./secrets/s3_api_secret_key.txt } + s3_admin_access_key: { file: ./secrets/s3_admin_access_key.txt } + s3_admin_secret_key: { file: ./secrets/s3_admin_secret_key.txt } volumes: postgres-data: - redis-data: \ No newline at end of file + redis-data: + s3-data: \ No newline at end of file diff --git a/evidence/SUMMARY.txt b/evidence/SUMMARY.txt new file mode 100644 index 0000000..85e0eec --- /dev/null +++ b/evidence/SUMMARY.txt @@ -0,0 +1,20 @@ + +PASS 00-up +PASS 01-go-tests +PASS 02-make-check +PASS 03-make-test +PASS 04-lb +PASS 05-edge +PASS 06-attachments +PASS 07-s3-rights +PASS 08-s3-init-twice +PASS 09-orphans +PASS 10-memory-1mb +PASS 11-memory-9mb +PASS 12-att-failover-1 +PASS 13-att-failover-2 +PASS 14-persistence +PASS 15-failover-1 +PASS 16-failover-2 +PASS 17-infra +FAIL 18-rolling diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 0000000..b94b34e --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1,4 @@ +uploaded via instance 01a59faf45f5 sha256 8c0b92dee2f27e38753f6ff06bbecf78a2727c92f26797e4e49126eb6d4d1be1 +stopped 01a59faf45f5 +downloaded 10x via ['e4222fff9061'] - sha256 matches; stopped instance served: False +restored diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 0000000..4954876 --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.05, + "concurrency": 4, + "successful_requests": 54199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 677.061, + "successful_latency_ms": { + "p50": 3.305, + "p95": 35.789, + "p99": 50.011 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-rolling.txt b/evidence/load-rolling.txt new file mode 100644 index 0000000..4954876 --- /dev/null +++ b/evidence/load-rolling.txt @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.05, + "concurrency": 4, + "successful_requests": 54199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 677.061, + "successful_latency_ms": { + "p50": 3.305, + "p95": 35.789, + "p99": 50.011 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/memory-1mb.json b/evidence/memory-1mb.json new file mode 100644 index 0000000..2581165 --- /dev/null +++ b/evidence/memory-1mb.json @@ -0,0 +1,15 @@ +{ + "payload_bytes": 999476, + "statuses": { + "201": 628 + }, + "idle_mib": { + "mtusi2026lab4-api-2": 82.43, + "mtusi2026lab4-api-1": 85.0 + }, + "peak_mib": { + "mtusi2026lab4-api-2": 135.9, + "mtusi2026lab4-api-1": 139.1 + }, + "limit_mib": 256 +} diff --git a/evidence/memory-9mb.json b/evidence/memory-9mb.json new file mode 100644 index 0000000..79b918e --- /dev/null +++ b/evidence/memory-9mb.json @@ -0,0 +1,16 @@ +{ + "payload_bytes": 9435932, + "statuses": { + "503": 6, + "201": 36 + }, + "idle_mib": { + "mtusi2026lab4-api-2": 117.7, + "mtusi2026lab4-api-1": 122.3 + }, + "peak_mib": { + "mtusi2026lab4-api-2": 169.9, + "mtusi2026lab4-api-1": 186.7 + }, + "limit_mib": 256 +} diff --git a/evidence/orphans-A.txt b/evidence/orphans-A.txt new file mode 100644 index 0000000..8240687 --- /dev/null +++ b/evidence/orphans-A.txt @@ -0,0 +1,9 @@ +== A: PUT прошёл, запись метаданных упала -> компенсация +objects before: 887 +ALTER TABLE +upload status: 500 +api-3 | 2026/11/05 10:33:51 attachment metadata insert failed, compensating: ERROR: new row for relation "attachments" violates check constraint "inject_fail" (SQLSTATE 23514) +(сбоев компенсации нет) +ALTER TABLE +objects after: 887 +PASS: объект не остался diff --git a/evidence/orphans-B.txt b/evidence/orphans-B.txt new file mode 100644 index 0000000..960cd8f --- /dev/null +++ b/evidence/orphans-B.txt @@ -0,0 +1,11 @@ +== B: объект без записи в БД (имитация падения между PUT и INSERT) +-- grace=1h: свежий объект не трогается +2026/11/05 10:34:21 reconcile: scanned=887 young=425 orphans=0 deleted=0 missing=0 dry_run=true took=199ms err= +-- grace=1s, dry-run: найден, но не удалён +2026/11/05 10:34:35 orphan (dry-run, not deleted): attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 +2026/11/05 10:34:35 reconcile: scanned=887 young=0 orphans=1 deleted=0 missing=0 dry_run=true took=240ms err= +2 +-- grace=1s, реальный запуск: удалён +2026/11/05 10:34:52 orphan deleted: attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 +2026/11/05 10:34:52 reconcile: scanned=887 young=0 orphans=1 deleted=1 missing=0 dry_run=false took=50ms err= +2026-11-05 10:25:07 999476 attachments/49a475be-019d-48d0-a0ec-606b92d35161/6d51b15a-446a-445f-8774-af909c971111 diff --git a/evidence/orphans-C.txt b/evidence/orphans-C.txt new file mode 100644 index 0000000..e9ff6ae --- /dev/null +++ b/evidence/orphans-C.txt @@ -0,0 +1,4 @@ +== C: объект потерян, метаданные остались +удаляем объект attachments/0170aa0f-9f15-4495-ae81-e0dd39a1e784/af279bf1-4af8-44b7-931b-b65f8a65048e (тестовое вложение) +2026/11/05 10:35:11 MISSING object for attachment id=af279bf1-4af8-44b7-931b-b65f8a65048e key=attachments/0170aa0f-9f15-4495-ae81-e0dd39a1e784/af279bf1-4af8-44b7-931b-b65f8a65048e (metadata kept, needs manual decision) +2026/11/05 10:35:12 reconcile: scanned=886 young=0 orphans=0 deleted=0 missing=1 dry_run=true took=1.244s err= diff --git a/evidence/orphans-D.txt b/evidence/orphans-D.txt new file mode 100644 index 0000000..9640613 --- /dev/null +++ b/evidence/orphans-D.txt @@ -0,0 +1,6 @@ +== D: обрыв загрузки клиентом (заявлено 5 МБ, отправлено 300 КБ) +objects before: 887 +aborted upload after 300 KB of 5000000 +objects after: 887 +PASS: в S3 ничего не записано +service "api" is not running container #1 diff --git a/evidence/ps-after-rolling.txt b/evidence/ps-after-rolling.txt new file mode 100644 index 0000000..acae351 --- /dev/null +++ b/evidence/ps-after-rolling.txt @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-3 mtusi2026lab4-api "./api" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-api-4 mtusi2026lab4-api "./api" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 36 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 36 minutes ago Up 3 minutes (healthy) 5432/tcp +mtusi2026lab4-reconciler-1 sha256:57946a358c790ffdba5c77418a7c9daaebda7f7434b93e71484671ce776c9f87 "./reconcile -loop=1…" reconciler 36 minutes ago Up 36 minutes 8080/tcp +mtusi2026lab4-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 36 minutes ago Up 3 minutes (healthy) 6379/tcp +mtusi2026lab4-s3-proxy-1 haproxy:3.0-alpine "docker-entrypoint.s…" s3-proxy 36 minutes ago Up 3 minutes +mtusi2026lab4-seaweedfs-1 chrislusf/seaweedfs:4.50 "/entrypoint.sh serv…" seaweedfs 36 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/s3-restart-probe.txt b/evidence/s3-restart-probe.txt new file mode 100644 index 0000000..1345c93 --- /dev/null +++ b/evidence/s3-restart-probe.txt @@ -0,0 +1,6 @@ +upload: 201 adbb25e9-e374-48f5-837b-a86a5690dcd3 +restart seaweedfs returned after 3.5s +t= 7.9s status=503 bytes_ok=False took=4.4s +t= 14.0s TimeoutError took=5.0s +t= 20.0s TimeoutError took=5.0s +t= 21.2s status=200 bytes_ok=True took=0.2s diff --git a/evidence/s3-rights.txt b/evidence/s3-rights.txt new file mode 100644 index 0000000..a22b3a5 --- /dev/null +++ b/evidence/s3-rights.txt @@ -0,0 +1,15 @@ +PASS api: список объектов бакета (ok) +PASS api: запись объекта (ok) +PASS api: удаление объекта (ok) +PASS api: создание другого бакета (deny) +PASS api: удаление бакета (deny) +PASS admin: создание бакета (bootstrap) (ok) +--- без подписи (anonymous) +PASS anonymous http://s3-proxy:8333/attachments/ -> 403 +PASS anonymous http://s3-proxy:8333/attachments/attachments/x/y -> 403 +--- консоли управления из контейнера api (ожидается недоступность) +PASS api -> seaweedfs:9333 недоступен +PASS api -> seaweedfs:8888 недоступен +PASS api -> seaweedfs:8080 недоступен +PASS api -> s3-proxy:9333 недоступен +PASS api -> s3-proxy:8888 недоступен diff --git a/go.mod b/go.mod index 9f9ee49..d1ce548 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,22 @@ module mtusi2026 go 1.27.1 +require ( + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.12 + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 + github.com/aws/smithy-go v1.28.7 + github.com/golang-migrate/migrate/v4 v4.20.1 + github.com/jackc/pgx/v5 v5.11.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/testcontainers/testcontainers-go v0.44.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 + github.com/testcontainers/testcontainers-go/modules/redis v0.44.0 + golang.org/x/crypto v0.54.0 + golang.org/x/sync v0.22.0 +) + require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect @@ -35,19 +51,20 @@ require ( github.com/apache/arrow/go/v10 v10.0.1 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/aws/aws-sdk-go v1.49.6 // indirect - github.com/aws/aws-sdk-go-v2 v1.16.16 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.20 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 // indirect - github.com/aws/smithy-go v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect @@ -60,7 +77,6 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect github.com/danieljoos/wincred v1.1.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -81,7 +97,6 @@ require ( github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang-migrate/migrate/v4 v4.20.1 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -103,7 +118,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jackc/pgx/v5 v5.11.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/k0kubun/pp v2.3.0+incompatible // indirect @@ -138,14 +152,13 @@ require ( github.com/mutecomm/go-sqlcipher/v4 v4.4.0 // indirect github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 // indirect github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba // indirect + github.com/nxadm/tail v1.4.11 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.16 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/redis/go-redis/v9 v9.23.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 // indirect github.com/shirou/gopsutil/v4 v4.26.6 // indirect @@ -154,9 +167,6 @@ require ( github.com/snowflakedb/gosnowflake v1.6.19 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/stretchr/testify v1.12.1 // indirect - github.com/testcontainers/testcontainers-go v0.44.0 // indirect - github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 // indirect - github.com/testcontainers/testcontainers-go/modules/redis v0.44.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/xanzy/go-gitlab v0.15.0 // indirect @@ -180,11 +190,9 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/atomic v1.12.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.54.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect golang.org/x/term v0.45.0 // indirect @@ -200,7 +208,6 @@ require ( google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/b v1.0.0 // indirect modernc.org/cc/v3 v3.36.3 // indirect diff --git a/go.sum b/go.sum index 98a54fa..45a6da0 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= cloud.google.com/go/longrunning v1.1.0 h1:qJ0R0IA8ONaRCNWTRPAS0iAmt1bj3TVgJ40z7ZGRslE= cloud.google.com/go/longrunning v1.1.0/go.mod h1:tH+A/6UvNypiPJWAQaKCsh+xiGbB23wUO8egwUXlD2E= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= @@ -19,16 +21,21 @@ cloud.google.com/go/spanner v1.92.0 h1:cfeMNmtFjz+OYzQVCIuGBw4Cik4CbF2ptXMuRQcUa cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 h1:T8quHYlUGyb/oqtSTwqlCr1ilJHrDv+ZtpSfo+hm1BU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= @@ -43,11 +50,13 @@ github.com/Azure/go-autorest/autorest/adal v0.9.16 h1:P8An8Z9rH1ldbOLdFpxYorgOt2 github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 h1:oPdPEZFSbl7oSPEAIPMPBMUmiL+mqgzBJwM/9qYcwNg= github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= @@ -58,8 +67,13 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -71,43 +85,54 @@ github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/aws/aws-sdk-go v1.49.6 h1:yNldzF5kzLBRvKlKz1S0bkvc2+04R1kt13KfBWQBfFA= github.com/aws/aws-sdk-go v1.49.6/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= -github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= -github.com/aws/aws-sdk-go-v2/config v1.17.7/go.mod h1:dN2gja/QXxFF15hQreyrqYhLBaQo1d9ZKe/v/uplQoI= -github.com/aws/aws-sdk-go-v2/credentials v1.12.20 h1:9+ZhlDY7N9dPnUmf7CDfW9In4sW5Ff3bh7oy4DzS1IE= -github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 h1:fAoVmNGhir6BR+RU0/EI+6+D7abM+MCwWf8v4ip5jNI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhmOnUkKGUZKqIMxmjmLOR8Uyp7G/TPwc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 h1:ZSIPAkAsCCjYrhqfw2+lNzWDzxzHXEckFkTePL5RSWQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 h1:BBYoNQt2kUZUUK4bIPsKrCcjVPUMNsgQpNAwhznK/zo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 h1:HfVVR1vItaG6le+Bpw6P4midjBDMKnjMyZnw9MXYUcE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 h1:3/gm/JTX9bX8CpzTgIlrtYpB3EVBDxyg/GY/QdcIEZw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= -github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= -github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= +github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18 h1:cNToRnziFY/DKoIUg9Vw350yx0sFfaN1FCJ+xVuF53Y= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18/go.mod h1:FQWrFCsxaXWlLBYB3YQOBx1zbfDNRrrVspVnnsTZ8Z0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -119,6 +144,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo= github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= @@ -135,6 +161,8 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= @@ -143,14 +171,20 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= @@ -161,8 +195,12 @@ github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= @@ -171,6 +209,10 @@ github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9 github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= +github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -196,6 +238,7 @@ github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdl github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -209,8 +252,12 @@ github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -222,6 +269,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -235,12 +284,15 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= @@ -253,6 +305,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= @@ -282,6 +338,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -338,9 +395,11 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= @@ -351,22 +410,23 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ktrysmt/go-bitbucket v0.6.4 h1:C8dUGp0qkwncKtAnozHCbbqhptefzEd1I0sfnuy9rYQ= github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -430,6 +490,8 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg= @@ -439,14 +501,21 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba h1:fhFP5RliM2HW/8XdcO5QngSfFli9GcRIpMXvypTQt6E= github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -454,6 +523,7 @@ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqgg github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= @@ -469,6 +539,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 h1:V7x0hCAgL8lNGezuex1RW1sh7VXXCqfw8nXZti66iFg= github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -483,8 +555,6 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/snowflakedb/gosnowflake v1.6.19 h1:KSHXrQ5o7uso25hNIzi/RObXtnSGkFgie91X82KcvMY= @@ -496,6 +566,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -505,8 +577,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= @@ -533,8 +603,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -554,8 +624,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= @@ -595,11 +669,11 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -608,8 +682,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -646,10 +718,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -681,25 +749,17 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -711,10 +771,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -734,8 +790,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk= @@ -748,6 +802,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.287.0 h1:CQDMqUiqZZ0U/Yge3zyjAhNQ0OSYEH0PaA7l4xtEen4= google.golang.org/api v0.287.0/go.mod h1:pPW85yt3Iuc3unkpaMhFtMmOqnTdCwCqEOaUlnuxRlQ= google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -785,25 +841,31 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= @@ -817,6 +879,7 @@ modernc.org/cc/v3 v3.36.3 h1:uISP3F66UlixxWEcKuIWERa4TwrZENHSL8tWxZz8bHg= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/ccgo/v3 v3.16.9 h1:AXquSwg7GuMk11pIdw7fmO1Y/ybgazVkMhsZWCV0mHM= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/db v1.0.0 h1:2c6NdCfaLnshSvY7OU09cyAY0gYXUZj4lmg5ItHyucg= modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= @@ -826,6 +889,7 @@ modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/internal v1.0.0 h1:XMDsFDcBDsibbBnHB2xzljZ+B1yrOVLEFkKL2u15Glw= modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= @@ -854,9 +918,15 @@ modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4 modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= modernc.org/zappy v1.0.0 h1:dPVaP+3ueIUv4guk8PuZ2wiUGcJ1WUVvIheeSSTD0yk= modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/infra/haproxy-s3.cfg b/infra/haproxy-s3.cfg new file mode 100644 index 0000000..1a97085 --- /dev/null +++ b/infra/haproxy-s3.cfg @@ -0,0 +1,22 @@ +global + log stdout format raw local0 + +defaults + mode tcp + timeout connect 3s + timeout client 60s + timeout server 60s + +resolvers docker + nameserver dns 127.0.0.11:53 + hold valid 2s + timeout resolve 1s + timeout retry 1s + resolve_retries 3 + +frontend s3 + bind :8333 + default_backend seaweed + +backend seaweed + server s1 seaweedfs:8333 resolvers docker init-addr none diff --git a/internal/blobstore/blobstore.go b/internal/blobstore/blobstore.go new file mode 100644 index 0000000..b8a6425 --- /dev/null +++ b/internal/blobstore/blobstore.go @@ -0,0 +1,233 @@ +package blobstore + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +var ErrNotFound = errors.New("object not found") + +type Store interface { + Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error + Get(ctx context.Context, key string) (io.ReadCloser, error) + Delete(ctx context.Context, key string) error + Ping(ctx context.Context) error + List(ctx context.Context, prefix string, fn func(key string, modified time.Time) error) error + Exists(ctx context.Context, key string) (bool, error) +} + +type S3Config struct { + Endpoint, Region, Bucket, AccessKey, SecretKey string +} + +type S3 struct { + client *s3.Client + bucket string +} + +func NewS3(ctx context.Context, c S3Config) (*S3, error) { + if c.Endpoint == "" || c.Bucket == "" || c.AccessKey == "" || c.SecretKey == "" { + return nil, errors.New("incomplete S3 configuration") + } + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(c.Region), + config.WithRetryMaxAttempts(2), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "")), + // Новые версии SDK по умолчанию добавляют контрольные суммы, которые не все S3-совместимые серверы понимают. + config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), + config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + ) + if err != nil { + return nil, err + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(c.Endpoint) + o.UsePathStyle = true + }) + return &S3{client: client, bucket: c.Bucket}, nil +} + +func (s *S3) Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error { + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: body, + ContentLength: aws.Int64(size), + ContentType: aws.String(contentType), + }) + return err +} + +func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) { + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + if isNotFound(err) { + return nil, ErrNotFound + } + return nil, err + } + return out.Body, nil +} + +func (s *S3) Delete(ctx context.Context, key string) error { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err +} + +func (s *S3) Ping(ctx context.Context) error { + body := []byte("ok") + if _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String("_health/probe"), + Body: bytes.NewReader(body), + ContentLength: aws.Int64(int64(len(body))), + }); err != nil { + return err + } + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String("_health/probe"), + }) + if err != nil { + return err + } + defer out.Body.Close() + _, err = io.Copy(io.Discard, out.Body) + return err +} + +func (s *S3) List(ctx context.Context, prefix string, fn func(key string, modified time.Time) error) error { + p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(prefix), + }) + for p.HasMorePages() { + page, err := p.NextPage(ctx) + if err != nil { + return err + } + for _, o := range page.Contents { + if err := fn(aws.ToString(o.Key), aws.ToTime(o.LastModified)); err != nil { + return err + } + } + } + return nil +} + +func (s *S3) Exists(ctx context.Context, key string) (bool, error) { + _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func isNotFound(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NotFound": + return true + } + } + return false +} + +type Memory struct { + mu sync.Mutex + m map[string][]byte + mod map[string]time.Time + Err error +} + +func NewMemory() *Memory { return &Memory{m: map[string][]byte{}, mod: map[string]time.Time{}} } + +func (s *Memory) Put(_ context.Context, key string, body io.ReadSeeker, _ int64, _ string) error { + if s.Err != nil { + return s.Err + } + b, err := io.ReadAll(body) + if err != nil { + return err + } + s.mu.Lock() + s.m[key], s.mod[key] = b, time.Now() + s.mu.Unlock() + return nil +} + +func (s *Memory) Get(_ context.Context, key string) (io.ReadCloser, error) { + if s.Err != nil { + return nil, s.Err + } + s.mu.Lock() + b, ok := s.m[key] + s.mu.Unlock() + if !ok { + return nil, ErrNotFound + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func (s *Memory) Delete(_ context.Context, key string) error { + if s.Err != nil { + return s.Err + } + s.mu.Lock() + delete(s.m, key) + delete(s.mod, key) + s.mu.Unlock() + return nil +} + +func (s *Memory) Ping(_ context.Context) error { return s.Err } + +func (s *Memory) Exists(_ context.Context, key string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.m[key] + return ok, s.Err +} + +func (s *Memory) List(_ context.Context, prefix string, fn func(string, time.Time) error) error { + s.mu.Lock() + keys := make([]string, 0, len(s.m)) + for k := range s.m { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + s.mu.Unlock() + for _, k := range keys { + if err := fn(k, s.mod[k]); err != nil { + return err + } + } + return nil +} + +func (s *Memory) Len() int { s.mu.Lock(); defer s.mu.Unlock(); return len(s.m) } diff --git a/internal/httpapi/attachments.go b/internal/httpapi/attachments.go new file mode 100644 index 0000000..beccf2e --- /dev/null +++ b/internal/httpapi/attachments.go @@ -0,0 +1,243 @@ +package httpapi + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log" + "mime" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "mtusi2026/internal/blobstore" + "mtusi2026/internal/idgen" + "mtusi2026/internal/model" + "mtusi2026/internal/store" +) + +const ( + maxAttachmentBytes = 10 << 20 + maxPixels = 40_000_000 + fullDecodePixels = 4_000_000 // полное декодирование только для небольших изображений +) + +func validFilename(s string) bool { + n := utf8.RuneCountInString(s) + if n < 1 || n > 255 || strings.TrimSpace(s) == "" { + return false + } + for _, r := range s { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +func (a *api) acquireUploadSlot(ctx context.Context) (func(), bool) { + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case a.uploadSem <- struct{}{}: + return func() { <-a.uploadSem }, true + case <-t.C: + return nil, false + case <-ctx.Done(): + return nil, false + } +} + +func (a *api) handleUploadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + chatID, ok := uuidParam(w, r, requestID, "chat_id") + if !ok { + return + } + if a.blobs == nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + member, err := a.store.IsChatMember(r.Context(), chatID, u.ID) + if err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + return + } + if !member { + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + return + } + + q := r.URL.Query() + filename := q.Get("filename") + if !q.Has("filename") || !validFilename(filename) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid filename") + return + } + mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || (mt != "image/png" && mt != "image/jpeg") { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "only image/png and image/jpeg are accepted") + return + } + if r.ContentLength > maxAttachmentBytes { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "file is too large") + return + } + + tmp, err := os.CreateTemp("", "upload-*") + if err != nil { + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + defer func() { tmp.Close(); os.Remove(tmp.Name()) }() + + h := sha256.New() + n, err := io.Copy(io.MultiWriter(tmp, h), http.MaxBytesReader(w, r.Body, maxAttachmentBytes)) + if err != nil { + var mbe *http.MaxBytesError + if errors.As(err, &mbe) { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "file is too large") + } else { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "upload interrupted") + } + return + } + if n == 0 { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "empty or invalid image") + return + } + + release, ok := a.acquireUploadSlot(r.Context()) + if !ok { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "upload capacity reached, retry later") + return + } + defer release() + + head := make([]byte, 512) + hn, _ := tmp.ReadAt(head, 0) + if http.DetectContentType(head[:hn]) != mt { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "content does not match declared type") + return + } + cfg, format, err := image.DecodeConfig(io.NewSectionReader(tmp, 0, n)) + want := map[string]string{"image/png": "png", "image/jpeg": "jpeg"}[mt] + if err != nil || format != want || cfg.Width <= 0 || cfg.Height <= 0 { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "invalid image") + return + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if pixels > maxPixels { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "image has too many pixels") + return + } + if pixels <= fullDecodePixels { + if _, _, err := image.Decode(io.NewSectionReader(tmp, 0, n)); err != nil { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "corrupted image") + return + } + } + + id := idgen.GenerateUUIDv4() + key := "attachments/" + chatID + "/" + id + + baseCtx := context.WithoutCancel(r.Context()) + putCtx, cancel := context.WithTimeout(baseCtx, 30*time.Second) + err = a.blobs.Put(putCtx, key, io.NewSectionReader(tmp, 0, n), n, mt) + cancel() + if err != nil { + log.Printf("s3 put failed: %v", err) + delCtx, delCancel := context.WithTimeout(baseCtx, 5*time.Second) + _ = a.blobs.Delete(delCtx, key) + delCancel() + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + + dbCtx, dbCancel := context.WithTimeout(baseCtx, 10*time.Second) + defer dbCancel() + att, err := a.store.CreateAttachment(dbCtx, model.Attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: mt, + SizeBytes: n, SHA256: hex.EncodeToString(h.Sum(nil)), ObjectKey: key, + }) + if err != nil { + log.Printf("attachment metadata insert failed, compensating: %v", err) + if derr := a.blobs.Delete(dbCtx, key); derr != nil { + log.Printf("compensation delete failed (reconciler will clean up) key=%s: %v", key, derr) + } + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + writeJSON(w, http.StatusCreated, requestID, att) +} + +func (a *api) loadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) (model.Attachment, bool) { + id, ok := uuidParam(w, r, requestID, "attachment_id") + if !ok { + return model.Attachment{}, false + } + att, err := a.store.GetAttachmentForUser(r.Context(), id, u.ID) + if err != nil { + if errors.Is(err, store.ErrAttachmentNotFound) { + writeError(w, http.StatusNotFound, requestID, codeNotFound, "attachment not found") + } else { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + } + return model.Attachment{}, false + } + return att, true +} + +func (a *api) handleGetAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + att, ok := a.loadAttachment(w, r, u, requestID) + if !ok { + return + } + writeJSON(w, http.StatusOK, requestID, att) +} + +func (a *api) handleDownloadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + if q := r.URL.Query(); q.Has("variant") && q.Get("variant") != "original" { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "unknown variant") + return + } + att, ok := a.loadAttachment(w, r, u, requestID) + if !ok { + return + } + if a.blobs == nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + t := time.AfterFunc(8*time.Second, cancel) + body, err := a.blobs.Get(ctx, att.ObjectKey) + t.Stop() + if err != nil { + if errors.Is(err, blobstore.ErrNotFound) { + log.Printf("INCONSISTENCY: metadata without object id=%s key=%s", att.ID, att.ObjectKey) + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + } else { + log.Printf("s3 get failed: %v", err) + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + } + return + } + defer body.Close() + + w.Header().Set("Content-Type", att.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(att.SizeBytes, 10)) + w.Header().Set("X-Request-Id", requestID) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, body) +} diff --git a/internal/httpapi/health.go b/internal/httpapi/health.go index 8df2046..22db2c0 100644 --- a/internal/httpapi/health.go +++ b/internal/httpapi/health.go @@ -3,18 +3,45 @@ package httpapi import ( "context" "net/http" + "sync" "sync/atomic" "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + + "mtusi2026/internal/blobstore" ) func handleHealthLive(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, newRequestID(), map[string]string{"status": "ok"}) } -func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.Bool) http.HandlerFunc { +type blobProbe struct { + mu sync.Mutex + at time.Time + err error + st blobstore.Store +} + +func (p *blobProbe) check(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + if time.Since(p.at) < time.Second { + return p.err + } + cctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + p.err = p.st.Ping(cctx) + p.at = time.Now() + return p.err +} + +func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, blobs blobstore.Store, draining *atomic.Bool) http.HandlerFunc { + var probe *blobProbe + if blobs != nil { + probe = &blobProbe{st: blobs} + } return func(w http.ResponseWriter, r *http.Request) { requestID := newRequestID() if draining.Load() { @@ -34,6 +61,12 @@ func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.B return } } + if probe != nil { + if err := probe.check(ctx); err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + } writeJSON(w, http.StatusOK, requestID, map[string]string{"status": "ok"}) } } diff --git a/internal/httpapi/httpx.go b/internal/httpapi/httpx.go index c628611..718f43d 100644 --- a/internal/httpapi/httpx.go +++ b/internal/httpapi/httpx.go @@ -15,12 +15,14 @@ import ( type errorCode string const ( - codeInvalidRequest errorCode = "invalid_request" - codeUnauthorized errorCode = "unauthorized" - codeNotFound errorCode = "not_found" - codeConflict errorCode = "conflict" - codeUnavailable errorCode = "unavailable" - codeInternal errorCode = "internal_error" + codeInvalidRequest errorCode = "invalid_request" + codeUnauthorized errorCode = "unauthorized" + codeNotFound errorCode = "not_found" + codeConflict errorCode = "conflict" + codeUnavailable errorCode = "unavailable" + codeInternal errorCode = "internal_error" + codeTooLarge errorCode = "payload_too_large" + codeUnsupportedMedia errorCode = "unsupported_media_type" ) func newRequestID() string { diff --git a/internal/httpapi/messages.go b/internal/httpapi/messages.go index dd1a2af..72b4263 100644 --- a/internal/httpapi/messages.go +++ b/internal/httpapi/messages.go @@ -4,7 +4,9 @@ import ( "errors" "net/http" "regexp" + "strings" + "mtusi2026/internal/idgen" "mtusi2026/internal/model" "mtusi2026/internal/store" ) @@ -12,7 +14,7 @@ import ( var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { - chatID, ok := chatIDParam(w, r, requestID) + chatID, ok := uuidParam(w, r, requestID, "chat_id") if !ok { return } @@ -28,11 +30,27 @@ func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u mode return } - m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text) + ids := make([]string, 0, len(req.AttachmentIDs)) + seen := map[string]bool{} + for _, id := range req.AttachmentIDs { + id = strings.ToLower(id) + if !idgen.IsValidUUID(id) || seen[id] { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid attachment_ids") + return + } + seen[id] = true + ids = append(ids, id) + } + if len(ids) > 10 || !store.ValidMessageBody(req.Text, len(ids) > 0) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid text or attachment_ids") + return + } + + m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text, ids) if err != nil { switch { - case errors.Is(err, store.ErrChatNotFound): - writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + case errors.Is(err, store.ErrChatNotFound), errors.Is(err, store.ErrAttachmentNotFound): + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat or attachment not found") case errors.Is(err, store.ErrIdempotencyConflict): writeError(w, http.StatusConflict, requestID, codeConflict, "Idempotency-Key was used with a different request body") case errors.Is(err, store.ErrValidation): diff --git a/internal/httpapi/router.go b/internal/httpapi/router.go index b9479e9..588b2d7 100644 --- a/internal/httpapi/router.go +++ b/internal/httpapi/router.go @@ -7,6 +7,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/model" "mtusi2026/internal/sessionstore" @@ -17,6 +18,8 @@ type Options struct { InstanceID string Redis *redis.Client Draining *atomic.Bool + Blobs blobstore.Store + MaxUploads int } func NewRouter(s *store.Store, pool *pgxpool.Pool, sessionStores ...sessionstore.Store) http.Handler { @@ -45,10 +48,16 @@ func NewRouterWithCache( o.Draining = new(atomic.Bool) } + if o.MaxUploads <= 0 { + o.MaxUploads = 4 + } + a := &api{ store: s, sessionStore: sessionStore, chatCache: chatCache, + blobs: o.Blobs, + uploadSem: make(chan struct{}, o.MaxUploads), } apiMux := http.NewServeMux() @@ -60,6 +69,9 @@ func NewRouterWithCache( apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) apiMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) apiMux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.authMiddleware(a.handleUploadAttachment)) + apiMux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.authMiddleware(a.handleGetAttachment)) + apiMux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.authMiddleware(a.handleDownloadAttachment)) apiMux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware( func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { a.handleDeleteSession(w, r) @@ -67,7 +79,7 @@ func NewRouterWithCache( root := http.NewServeMux() root.HandleFunc("GET /health/live", handleHealthLive) - root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Draining)) + root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Blobs, o.Draining)) root.Handle("/", concurrencyMiddleware(s.Limiter, apiMux)) var h http.Handler = root diff --git a/internal/httpapi/users.go b/internal/httpapi/users.go index 939ad09..c893ea3 100644 --- a/internal/httpapi/users.go +++ b/internal/httpapi/users.go @@ -9,6 +9,7 @@ import ( "golang.org/x/sync/singleflight" "mtusi2026/internal/auth" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/model" "mtusi2026/internal/sessionstore" @@ -22,6 +23,8 @@ type api struct { sessionStore sessionstore.Store chatCache *cache.Cache chatFlight singleflight.Group + blobs blobstore.Store + uploadSem chan struct{} } func (a *api) authMiddleware(next func(w http.ResponseWriter, r *http.Request, u model.User, requestID string)) http.HandlerFunc { diff --git a/internal/httpapi/validate.go b/internal/httpapi/validate.go index 11da225..3d0816c 100644 --- a/internal/httpapi/validate.go +++ b/internal/httpapi/validate.go @@ -43,3 +43,12 @@ func chatIDParam(w http.ResponseWriter, r *http.Request, requestID string) (stri } return strings.ToLower(id), true } + +func uuidParam(w http.ResponseWriter, r *http.Request, requestID, name string) (string, bool) { + id := r.PathValue(name) + if !idgen.IsValidUUID(id) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid "+name) + return "", false + } + return strings.ToLower(id), true +} diff --git a/internal/model/model.go b/internal/model/model.go index e468a34..33af5bc 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -33,15 +33,17 @@ type ChatPage struct { } type Message struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` } type CreateMessageRequest struct { - Text string `json:"text"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` } type MessagePage struct { @@ -63,3 +65,28 @@ type Session struct { ExpiresAt time.Time `json:"expires_at"` User User `json:"user"` } + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..d02f3ef --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,116 @@ +package reconcile + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi2026/internal/blobstore" +) + +type Report struct { + Scanned, Young, Orphans, Deleted, Missing int +} + +type cand struct{ key string } + +func Run(ctx context.Context, pool *pgxpool.Pool, blobs blobstore.Store, grace time.Duration, + dryRun, checkMissing bool, logf func(string, ...any)) (Report, error) { + + var rep Report + cutoff := time.Now().Add(-grace) + var batch []string + + flush := func() error { + if len(batch) == 0 { + return nil + } + rows, err := pool.Query(ctx, `SELECT object_key FROM attachments WHERE object_key = ANY($1)`, batch) + if err != nil { + return err + } + known := map[string]bool{} + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return err + } + known[k] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + for _, k := range batch { + if known[k] { + continue + } + rep.Orphans++ + if dryRun { + logf("orphan (dry-run, not deleted): %s", k) + continue + } + if err := blobs.Delete(ctx, k); err != nil { + logf("orphan delete failed: %s: %v", k, err) + continue + } + rep.Deleted++ + logf("orphan deleted: %s", k) + } + batch = batch[:0] + return nil + } + + err := blobs.List(ctx, "attachments/", func(key string, modified time.Time) error { + rep.Scanned++ + if modified.After(cutoff) { + rep.Young++ + return nil + } + batch = append(batch, key) + if len(batch) >= 500 { + return flush() + } + return nil + }) + if err != nil { + return rep, fmt.Errorf("list: %w", err) + } + if err := flush(); err != nil { + return rep, fmt.Errorf("check db: %w", err) + } + + if checkMissing { + rows, err := pool.Query(ctx, + `SELECT id, object_key FROM attachments WHERE created_at < $1 ORDER BY created_at`, cutoff) + if err != nil { + return rep, err + } + type rec struct{ id, key string } + var recs []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.id, &r.key); err != nil { + rows.Close() + return rep, err + } + recs = append(recs, r) + } + rows.Close() + for _, r := range recs { + ok, err := blobs.Exists(ctx, r.key) + if err != nil { + logf("exists check failed: %s: %v", r.key, err) + continue + } + if !ok { + rep.Missing++ + logf("MISSING object for attachment id=%s key=%s (metadata kept, needs manual decision)", r.id, r.key) + } + } + } + return rep, nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 13ec038..f2756b7 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "strconv" @@ -21,10 +22,13 @@ import ( ) var ( - ErrUsernameTaken = errors.New("username taken") - ErrChatNotFound = errors.New("chat not found or not accessible") - ErrValidation = errors.New("validation failed") - ErrCapacityExceeded = errors.New("message capacity exceeded") + ErrUsernameTaken = errors.New("username taken") + ErrChatNotFound = errors.New("chat not found or not accessible") + ErrValidation = errors.New("validation failed") + ErrCapacityExceeded = errors.New("message capacity exceeded") + ErrAttachmentNotFound = errors.New("attachment not found or not accessible") + ErrUsernameNotFound = errors.New("username not found") + ErrSessionNotFound = errors.New("session not found or expired") ) type Store struct { @@ -33,10 +37,102 @@ type Store struct { chatSQLQueries atomic.Int64 } +func (s *Store) ResetChatSQLQueries() { + s.chatSQLQueries.Store(0) +} + +func (s *Store) ChatSQLQueries() int64 { + return s.chatSQLQueries.Load() +} + +func (s *Store) IsChatMember(ctx context.Context, chatID, userID string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + return ok, err +} + +func (s *Store) CreateAttachment(ctx context.Context, a model.Attachment) (model.Attachment, error) { + err := s.pool.QueryRow(ctx, + `INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + RETURNING status, created_at, updated_at`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, a.ObjectKey, + ).Scan(&a.Status, &a.CreatedAt, &a.UpdatedAt) + a.Variants = []model.Variant{} + return a, err +} +func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, + `SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.object_key, a.status, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2 + WHERE a.id = $1`, id, userID, + ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, + &a.ObjectKey, &a.Status, &a.CreatedAt, &a.UpdatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrAttachmentNotFound + } + a.Variants = []model.Variant{} + return a, err +} + +func (s *Store) loadAttachmentIDs(ctx context.Context, msgs []model.Message) error { + if len(msgs) == 0 { + return nil + } + ids := make([]string, len(msgs)) + pos := make(map[string]int, len(msgs)) + for i := range msgs { + ids[i] = msgs[i].ID + pos[msgs[i].ID] = i + msgs[i].AttachmentIDs = []string{} + } + rows, err := s.pool.Query(ctx, + `SELECT message_id, attachment_id FROM message_attachments + WHERE message_id IN (SELECT unnest($1::text[])::uuid) + ORDER BY message_id, position`, ids) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var mid, aid string + if err := rows.Scan(&mid, &aid); err != nil { + return err + } + if i, ok := pos[mid]; ok { + msgs[i].AttachmentIDs = append(msgs[i].AttachmentIDs, aid) + } + } + return rows.Err() +} + func New(pool *pgxpool.Pool, limiter *Limiter) *Store { return &Store{pool: pool, Limiter: limiter} } +func txAttachmentIDs(ctx context.Context, tx pgx.Tx, messageID string) ([]string, error) { + rows, err := tx.Query(ctx, + `SELECT attachment_id FROM message_attachments WHERE message_id = $1 ORDER BY position`, messageID) + if err != nil { + return nil, err + } + defer rows.Close() + ids := []string{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (model.User, error) { var u model.User err := s.pool.QueryRow(ctx, @@ -238,12 +334,14 @@ func (s *Store) ListChatsForUser(ctx context.Context, userID, cursorStr string, var ErrIdempotencyConflict = errors.New("idempotency key reused with different body") -func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string) (model.Message, error) { - if !validMessageText(text) { +func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string, attIDs []string) (model.Message, error) { + if attIDs == nil { + attIDs = []string{} + } + if !ValidMessageBody(text, len(attIDs) > 0) { return model.Message{}, ErrValidation } - sum := sha256.Sum256([]byte(text)) - hash := hex.EncodeToString(sum[:]) + hash := requestHash(text, attIDs) tx, err := s.pool.Begin(ctx) if err != nil { @@ -261,6 +359,19 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te return model.Message{}, ErrChatNotFound } + if len(attIDs) > 0 { + var n int + if err := tx.QueryRow(ctx, + `SELECT count(*) FROM attachments + WHERE chat_id = $1 AND id IN (SELECT unnest($2::text[])::uuid)`, + chatID, attIDs).Scan(&n); err != nil { + return model.Message{}, err + } + if n != len(attIDs) { + return model.Message{}, ErrAttachmentNotFound + } + } + msgID := idgen.GenerateUUIDv4() tag, err := tx.Exec(ctx, `INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) @@ -274,18 +385,20 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te if tag.RowsAffected() == 0 { var storedHash string var m model.Message - err := tx.QueryRow(ctx, + if err := tx.QueryRow(ctx, `SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`, senderID, chatID, idemKey, - ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) - if err != nil { + ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { return model.Message{}, err } if storedHash != hash { return model.Message{}, ErrIdempotencyConflict } + if m.AttachmentIDs, err = txAttachmentIDs(ctx, tx, m.ID); err != nil { + return model.Message{}, err + } return m, nil } @@ -302,9 +415,17 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te } return model.Message{}, err } + for i, id := range attIDs { + if _, err := tx.Exec(ctx, + `INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1,$2,$3)`, + m.ID, id, i); err != nil { + return model.Message{}, err + } + } if err := tx.Commit(ctx); err != nil { return model.Message{}, err } + m.AttachmentIDs = attIDs return m, nil } @@ -368,6 +489,9 @@ func (s *Store) ListMessages(ctx context.Context, chatID, userID, cursorStr stri next = &nc page = page[:limit] } + if err := s.loadAttachmentIDs(ctx, page); err != nil { + return nil, nil, err + } return page, next, nil } @@ -386,9 +510,8 @@ func isCheckViolation(err error) bool { return errors.As(err, &pgErr) && pgErr.Code == "23514" } -func validMessageText(s string) bool { - n := utf8.RuneCountInString(s) - if n == 0 || n > 4000 { +func ValidMessageBody(s string, hasAttachments bool) bool { + if utf8.RuneCountInString(s) > 4000 { return false } hasContent := false @@ -400,10 +523,17 @@ func validMessageText(s string) bool { hasContent = true } } - return hasContent + return hasAttachments || hasContent } -var ErrSessionNotFound = errors.New("session not found or expired") +func requestHash(text string, ids []string) string { + b, _ := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, ids}) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.User, string, error) { var u model.User @@ -424,13 +554,3 @@ func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.U return u, passwordHash, nil } - -var ErrUsernameNotFound = errors.New("username not found") - -func (s *Store) ResetChatSQLQueries() { - s.chatSQLQueries.Store(0) -} - -func (s *Store) ChatSQLQueries() int64 { - return s.chatSQLQueries.Load() -} diff --git a/migrations/000003_attachments.down.sql b/migrations/000003_attachments.down.sql new file mode 100644 index 0000000..90e20cb --- /dev/null +++ b/migrations/000003_attachments.down.sql @@ -0,0 +1,4 @@ +DROP TABLE message_attachments; +DROP TABLE attachments; +ALTER TABLE messages DROP CONSTRAINT text_length; +ALTER TABLE messages ADD CONSTRAINT text_length CHECK (char_length(text) BETWEEN 1 AND 4000); \ No newline at end of file diff --git a/migrations/000003_attachments.up.sql b/migrations/000003_attachments.up.sql new file mode 100644 index 0000000..b83add5 --- /dev/null +++ b/migrations/000003_attachments.up.sql @@ -0,0 +1,25 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX idx_attachments_chat ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); +CREATE INDEX idx_message_attachments_attachment ON message_attachments (attachment_id); + +ALTER TABLE messages DROP CONSTRAINT text_length; +ALTER TABLE messages ADD CONSTRAINT text_length CHECK (char_length(text) <= 4000); \ No newline at end of file diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..a58e29b --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +[ -f .env ] || { cp .env.example .env; echo ".env создан из .env.example"; } +mkdir -p secrets +[ -f secrets/postgres_password.txt ] || printf '%s' "$(grep '^POSTGRES_PASSWORD=' .env | cut -d= -f2-)" > secrets/postgres_password.txt +[ -f secrets/database_url.txt ] || printf '%s' "$(grep '^DATABASE_URL=' .env | cut -d= -f2-)" > secrets/database_url.txt +./scripts/gen-s3-secrets.sh +chmod 644 secrets/* +echo "секреты готовы" diff --git a/scripts/check_attachment_failover.py b/scripts/check_attachment_failover.py new file mode 100644 index 0000000..b9dd61c --- /dev/null +++ b/scripts/check_attachment_failover.py @@ -0,0 +1,53 @@ +import base64, hashlib, json, struct, subprocess, sys, time, urllib.request, uuid, zlib + +BASE = "http://localhost:8080"; PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, headers=None): + req = urllib.request.Request(BASE + path, data=body, method=method, headers=headers or {}) + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() # r.headers нечувствителен к регистру + + +def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +png = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 16, 16, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"".join(b"\x00" + bytes([i, 80, 200]) * 16 for i in range(16)))) + + chunk(b"IEND", b"")) + +j = {"Content-Type": "application/json"} +name = "af_" + uuid.uuid4().hex[:8] +_, _, raw = call("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "a", "password": PWD}).encode(), j) +user = json.loads(raw) +_, _, raw = call("POST", "/api/v1/auth/sessions", None, + {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +auth = {"Authorization": "Bearer " + json.loads(raw)["token"]} +_, _, raw = call("POST", "/api/v1/chats", json.dumps({"title": "x", "member_ids": [user["id"]]}).encode(), {**j, **auth}) +chat = json.loads(raw) + +_, hdr, raw = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=f.png", png, + {"Content-Type": "image/png", **auth}) +att = json.loads(raw) +inst = hdr.get("X-Instance-Id") +print("uploaded via instance", inst, "sha256", att["sha256"]) + +ids = subprocess.run(["docker", "compose", "ps", "-q", "api"], capture_output=True, text=True).stdout.split() +target = next((i for i in ids if subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", i], + capture_output=True, text=True).stdout.strip() == inst), None) +if not target: + sys.exit(f"container for instance {inst} not found") +subprocess.run(["docker", "stop", "-t", "8", target], capture_output=True) +print("stopped", inst) +time.sleep(4) + +seen = set() +for _ in range(10): + _, h, data = call("GET", f"/api/v1/attachments/{att['id']}/content", None, auth) + seen.add(h.get("X-Instance-Id")) + assert hashlib.sha256(data).hexdigest() == att["sha256"], "SHA-256 mismatch" +print("downloaded 10x via", sorted(seen), "- sha256 matches; stopped instance served:", inst in seen) + +subprocess.run(["docker", "compose", "up", "-d", "--no-recreate", "api"], capture_output=True) +print("restored") \ No newline at end of file diff --git a/scripts/check_attachments.py b/scripts/check_attachments.py new file mode 100644 index 0000000..425e53c --- /dev/null +++ b/scripts/check_attachments.py @@ -0,0 +1,131 @@ +import base64, hashlib, json, os, random, struct, sys, urllib.error, urllib.request, uuid, zlib + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +PWD = "correct-horse-battery-staple" +LIMIT = 10 * 1024 * 1024 + + +def call(method, path, body=None, headers=None, token=None, basic=None): + h = dict(headers or {}) + if token: h["Authorization"] = "Bearer " + token + if basic: h["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() + req = urllib.request.Request(BASE + path, data=body, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() + except urllib.error.HTTPError as e: + return e.code, e.headers, e.read() + + +def jcall(method, path, obj, token=None, key=None): + h = {"Content-Type": "application/json"} + if key: h["Idempotency-Key"] = key + st, _, raw = call(method, path, json.dumps(obj).encode(), h, token) + return st, (json.loads(raw) if raw else None) + + +def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +def png(width, height, idat=None): + raw = idat if idat is not None else b"".join(b"\x00" + b"\x10\x80\xf0" * width for _ in range(height)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw) if idat is None else raw) + chunk(b"IEND", b"")) + + +def noise_png(target_bytes): + w = 1000 + rows = max(1, target_bytes // (3 * w + 1)) + raw = b"".join(b"\x00" + os.urandom(3 * w) for _ in range(rows)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, rows, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"")) + + +ok = True +def check(name, got, want): + global ok + good = got in want if isinstance(want, set) else got == want + ok &= good + print("PASS" if good else "FAIL", name, "->", got) + + +def register(prefix): + name = f"{prefix}_{uuid.uuid4().hex[:8]}" + _, u = jcall("POST", "/api/v1/users", {"username": name, "display_name": prefix, "password": PWD}) + _, _, raw = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") + return u, json.loads(raw)["token"] + + +alice, ta = register("alice") +bob, tb = register("bob") +carol, tc = register("carol") +_, chat1 = jcall("POST", "/api/v1/chats", {"title": "c1", "member_ids": [alice["id"], bob["id"]]}, ta) +_, chat2 = jcall("POST", "/api/v1/chats", {"title": "c2", "member_ids": [alice["id"]]}, ta) +up = lambda chat, data, ct="image/png", fn="a.png", tok=None: call( + "POST", f"/api/v1/chats/{chat}/attachments?filename={fn}", data, {"Content-Type": ct}, tok or ta) +msg = lambda chat, obj, tok=None, key=None: jcall( + "POST", f"/api/v1/chats/{chat}/messages", obj, tok or ta, key or uuid.uuid4().hex) + +good_png = png(8, 8) +st, _, raw = up(chat1["id"], good_png) +check("валидный PNG", st, 201) +att = json.loads(raw) +check("sha256 в ответе", att["sha256"], hashlib.sha256(good_png).hexdigest()) +check("status/variants/error_code", (att["status"], att["variants"], att["error_code"]), ("ready", [], None)) + +st, hdr, data = call("GET", f"/api/v1/attachments/{att['id']}/content", token=tb) +check("скачивание участником: статус", st, 200) +check("скачивание: байты совпадают", hashlib.sha256(data).hexdigest(), att["sha256"]) +check("скачивание: Content-Type", hdr.get("Content-Type"), "image/png") + +check("метаданные: посторонний", call("GET", f"/api/v1/attachments/{att['id']}", token=tc)[0], 404) +check("байты: посторонний", call("GET", f"/api/v1/attachments/{att['id']}/content", token=tc)[0], 404) +check("без токена", call("GET", f"/api/v1/attachments/{att['id']}/content")[0], 401) +check("неизвестный id", call("GET", f"/api/v1/attachments/{uuid.uuid4()}", token=ta)[0], 404) +check("id не UUID", call("GET", "/api/v1/attachments/not-a-uuid", token=ta)[0], 400) +check("variant=thumbnail", call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=ta)[0], {400, 404}) + +check("загрузка в чужой чат", up(chat1["id"], good_png, tok=tc)[0], 404) +check("слишком большой (Content-Length)", up(chat1["id"], good_png[:8] + b"\0" * (LIMIT + 1))[0], 413) +check("слишком много пикселей (8000x8000)", up(chat1["id"], png(8000, 8000, idat=b"\x00"))[0], 413) +check("повреждённый PNG", up(chat1["id"], good_png[:-20])[0], 415) +check("PNG под видом JPEG", up(chat1["id"], good_png, ct="image/jpeg")[0], 415) +check("текст под видом PNG", up(chat1["id"], b"hello, not an image")[0], 415) +check("пустое тело", up(chat1["id"], b"")[0], 415) +check("image/gif", up(chat1["id"], b"GIF89a....", ct="image/gif")[0], 415) +for label, fn in [("слэш", "a%2Fb.png"), ("обратный слэш", "a%5Cb.png"), ("NUL", "a%00b.png"), + ("управляющий символ", "a%0Ab.png"), ("256 символов", "a" * 256), ("пустое", "")]: + check(f"filename: {label}", up(chat1["id"], good_png, fn=fn)[0], 400) +st, _, _ = call("POST", f"/api/v1/chats/{chat1['id']}/attachments", good_png, {"Content-Type": "image/png"}, ta) +check("filename отсутствует", st, 400) +check("валидный большой файл (~9 МиБ)", up(chat1["id"], noise_png(9 * 1024 * 1024))[0], 201) + +# ---- сообщения с вложениями +st, m1 = msg(chat1["id"], {"text": "", "attachment_ids": [att["id"]]}, key="k-" + uuid.uuid4().hex) +check("пустой текст с вложением", st, 201) +check("в ответе attachment_ids", m1["attachment_ids"], [att["id"]]) +check("пустой текст без вложений", msg(chat1["id"], {"text": ""})[0], 400) +check("вложение из другого чата (я участник обоих)", msg(chat2["id"], {"text": "x", "attachment_ids": [att["id"]]})[0], 404) +check("неизвестное вложение", msg(chat1["id"], {"text": "x", "attachment_ids": [str(uuid.uuid4())]})[0], 404) +check("attachment_ids не UUID", msg(chat1["id"], {"text": "x", "attachment_ids": ["zzz"]})[0], 400) +check("дубликаты в attachment_ids", msg(chat1["id"], {"text": "x", "attachment_ids": [att["id"], att["id"]]})[0], 400) +check("11 вложений", msg(chat1["id"], {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]})[0], 400) +check("посторонний шлёт чужое вложение", msg(chat1["id"], {"text": "x", "attachment_ids": [att["id"]]}, tok=tc)[0], 404) + +_, page = jcall("GET", f"/api/v1/chats/{chat1['id']}/messages", None, ta) +st, _, raw = call("GET", f"/api/v1/chats/{chat1['id']}/messages", token=bob and tb) +items = json.loads(raw)["items"] +check("история содержит attachment_ids", any(i["attachment_ids"] == [att["id"]] for i in items), True) + +# ---- идемпотентность с вложениями +k = "idem-" + uuid.uuid4().hex +s1, a = msg(chat1["id"], {"text": "same", "attachment_ids": []}, key=k) +s2, b = msg(chat1["id"], {"text": "same"}, key=k) +check("attachment_ids отсутствует == []", (s1, s2, a["id"] == b["id"]), (201, 201, True)) +k2 = "idem-" + uuid.uuid4().hex +msg(chat1["id"], {"text": "t", "attachment_ids": [att["id"]]}, key=k2) +check("тот же ключ, другой список вложений", msg(chat1["id"], {"text": "t"}, key=k2)[0], 409) + +sys.exit(0 if ok else 1) \ No newline at end of file diff --git a/scripts/check_s3.sh b/scripts/check_s3.sh new file mode 100755 index 0000000..e60e440 --- /dev/null +++ b/scripts/check_s3.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu +run() { + who="$1"; shift + docker run --rm --network mtusi2026lab4_s3access -v "$PWD/secrets:/s:ro" -e AWS_DEFAULT_REGION=us-east-1 \ + --entrypoint /bin/sh amazon/aws-cli:2.39.1 -c \ + "AWS_ACCESS_KEY_ID=\$(cat /s/s3_${who}_access_key.txt) AWS_SECRET_ACCESS_KEY=\$(cat /s/s3_${who}_secret_key.txt) aws --endpoint-url http://s3-proxy:8333 $*" >/dev/null 2>&1 +} +expect() { + want="$1"; desc="$2"; shift 2 + if run "$@"; then got=ok; else got=deny; fi + [ "$got" = "$want" ] && echo "PASS $desc ($got)" || echo "FAIL $desc: ожидалось $want, получено $got" +} + +expect ok "api: список объектов бакета" api "s3api list-objects-v2 --bucket attachments --max-keys 1" +expect ok "api: запись объекта" api "s3 cp /etc/hostname s3://attachments/attachments/probe/rights-check" +expect ok "api: удаление объекта" api "s3 rm s3://attachments/attachments/probe/rights-check" +expect deny "api: создание другого бакета" api "s3api create-bucket --bucket other-bucket" +expect deny "api: удаление бакета" api "s3api delete-bucket --bucket attachments" +expect ok "admin: создание бакета (bootstrap)" admin "s3api head-bucket --bucket attachments" + +echo "--- без подписи (anonymous)" +for url in "http://s3-proxy:8333/attachments/" "http://s3-proxy:8333/attachments/attachments/x/y"; do + code=$(docker run --rm --network mtusi2026lab4_s3access curlimages/curl -s -o /dev/null -w '%{http_code}' "$url") + case "$code" in 401|403) echo "PASS anonymous $url -> $code";; *) echo "FAIL anonymous $url -> $code";; esac +done + +echo "--- консоли управления из контейнера api (ожидается недоступность)" +for target in seaweedfs:9333 seaweedfs:8888 seaweedfs:8080 s3-proxy:9333 s3-proxy:8888; do + if docker compose exec -T api wget -q -O /dev/null -T 3 "http://$target/" >/dev/null 2>&1; then + echo "FAIL api -> $target доступен" + else + echo "PASS api -> $target недоступен" + fi +done \ No newline at end of file diff --git a/scripts/gen-s3-secrets.sh b/scripts/gen-s3-secrets.sh new file mode 100755 index 0000000..4eb7063 --- /dev/null +++ b/scripts/gen-s3-secrets.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +mkdir -p secrets +rnd() { openssl rand -hex "$1"; } +for f in s3_api_access_key s3_api_secret_key s3_admin_access_key s3_admin_secret_key; do + [ -f "secrets/$f.txt" ] || { [ "${f%access_key}" != "$f" ] && printf '%s' "AK$(rnd 8)" || printf '%s' "$(rnd 20)"; } > "secrets/$f.txt" +done +cat > secrets/s3_config.json < 1 else 9 * 1024 * 1024 +WORKERS, DURATION = 12, 20 + + +def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +def noise_png(target): + w = 1000 + rows = max(1, target // (3 * w + 1)) + raw = b"".join(b"\x00" + os.urandom(3 * w) for _ in range(rows)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, rows, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"")) + + +def call(method, path, body=None, headers=None): + req = urllib.request.Request(BASE + path, data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +name = "mem_" + uuid.uuid4().hex[:8] +_, raw = call("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "m", "password": PWD}).encode(), + {"Content-Type": "application/json"}) +user = json.loads(raw) +_, raw = call("POST", "/api/v1/auth/sessions", None, + {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +tok = json.loads(raw)["token"] +_, raw = call("POST", "/api/v1/chats", json.dumps({"title": "m", "member_ids": [user["id"]]}).encode(), + {"Content-Type": "application/json", "Authorization": "Bearer " + tok}) +chat = json.loads(raw) +data = noise_png(SIZE) +print(f"payload: {len(data)} bytes, workers={WORKERS}, duration={DURATION}s") + +statuses, lock, stop = {}, threading.Lock(), threading.Event() + + +def worker(): + while not stop.is_set(): + st, _ = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=m.png", data, + {"Content-Type": "image/png", "Authorization": "Bearer " + tok}) + with lock: + statuses[st] = statuses.get(st, 0) + 1 + + +def mib(s): + n, unit = float(s[:-3]) if s.endswith("iB") else 0, s[-3:] + return n * {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}.get(unit, 0) + + +peaks = {} +def sampler(): + while not stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, text=True).stdout + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and "-api-" in parts[0]: + peaks[parts[0]] = max(peaks.get(parts[0], 0), mib(parts[1])) + +base = {} +out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, text=True).stdout +for line in out.splitlines(): + p = line.split() + if len(p) >= 2 and "-api-" in p[0]: + base[p[0]] = mib(p[1]) + +ths = [threading.Thread(target=worker) for _ in range(WORKERS)] + [threading.Thread(target=sampler)] +t0 = time.time() +[t.start() for t in ths] +time.sleep(DURATION) +stop.set() +[t.join() for t in ths] +print(json.dumps({"payload_bytes": len(data), "statuses": statuses, "idle_mib": base, + "peak_mib": {k: round(v, 1) for k, v in peaks.items()}, "limit_mib": 256}, indent=2)) \ No newline at end of file diff --git a/scripts/orphan_scenarios.sh b/scripts/orphan_scenarios.sh new file mode 100755 index 0000000..d9ca6bd --- /dev/null +++ b/scripts/orphan_scenarios.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +count() { ./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | wc -l | tr -d ' '; } +psql_() { docker compose exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -t -A -c \"$1\""; } +trap 'psql_ "ALTER TABLE attachments DROP CONSTRAINT IF EXISTS inject_fail;" >/dev/null 2>&1 || true' EXIT +ORPHAN=attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 + +{ +echo "== A: PUT прошёл, запись метаданных упала -> компенсация" +before=$(count); echo "objects before: $before" +psql_ "ALTER TABLE attachments ADD CONSTRAINT inject_fail CHECK (false) NOT VALID;" +python3 scripts/upload_tools.py ok +sleep 1 +docker compose logs api --since 1m | grep "metadata insert failed" | tail -1 || true +docker compose logs api --since 1m | grep "compensation delete failed" || echo "(сбоев компенсации нет)" +psql_ "ALTER TABLE attachments DROP CONSTRAINT inject_fail;" +after=$(count); echo "objects after: $after" +[ "$before" = "$after" ] && echo "PASS: объект не остался" || echo "FAIL: число объектов изменилось" +} 2>&1 | tee evidence/orphans-A.txt + +{ +echo "== D: обрыв загрузки клиентом (заявлено 5 МБ, отправлено 300 КБ)" +before=$(count); echo "objects before: $before" +python3 scripts/upload_tools.py abort +sleep 3 +after=$(count); echo "objects after: $after" +[ "$before" = "$after" ] && echo "PASS: в S3 ничего не записано" || echo "FAIL: число объектов изменилось" +for i in 1 2; do docker compose exec -T --index $i api sh -c "echo временных файлов в реплике $i: \$(ls /tmp | wc -l)"; done +} 2>&1 | tee evidence/orphans-D.txt + +{ +echo "== B: объект без записи в БД (имитация падения между PUT и INSERT)" +./scripts/s3-admin.sh s3api put-object --bucket attachments --key "$ORPHAN" >/dev/null +sleep 2 +echo "-- grace=1h: свежий объект не трогается" +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1h 2>&1 | grep -E "orphan|reconcile:" || true +echo "-- grace=1s, dry-run: найден, но не удалён" +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1s 2>&1 | grep -E "orphan|reconcile:" || true +./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | grep -c 1111 || true +echo "-- grace=1s, реальный запуск: удалён" +docker compose run --rm -T reconciler ./reconcile -grace=1s 2>&1 | grep -E "orphan|reconcile:" || true +./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | grep 1111 || echo "PASS: сирота удалён" +} 2>&1 | tee evidence/orphans-B.txt + +{ +echo "== C: объект потерян, метаданные остались" +key=$(psql_ "SELECT object_key FROM attachments ORDER BY created_at LIMIT 1;") +echo "удаляем объект $key (тестовое вложение)" +./scripts/s3-admin.sh s3 rm "s3://attachments/$key" >/dev/null +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1s -check-missing 2>&1 | grep -E "MISSING|reconcile:" || true +} 2>&1 | tee evidence/orphans-C.txt diff --git a/scripts/restart.sh b/scripts/restart.sh index ef623f1..b62075e 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,4 +1,10 @@ #!/bin/sh set -eu -docker compose restart postgres redis -docker compose restart api lb \ No newline at end of file +cd "$(dirname "$0")/.." +healthy() { + [ "$(docker compose ps -q "$@" | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$#" ] +} +docker compose restart postgres redis seaweedfs +until healthy postgres redis seaweedfs; do sleep 1; done +docker compose restart s3-proxy +docker compose restart api lb diff --git a/scripts/s3-admin.sh b/scripts/s3-admin.sh new file mode 100755 index 0000000..7c2503d --- /dev/null +++ b/scripts/s3-admin.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker run --rm --network mtusi2026lab4_s3access -v "$PWD/secrets:/s:ro" -e AWS_DEFAULT_REGION=us-east-1 \ + --entrypoint /bin/sh amazon/aws-cli:2.39.1 -c \ + 'AWS_ACCESS_KEY_ID=$(cat /s/s3_admin_access_key.txt) AWS_SECRET_ACCESS_KEY=$(cat /s/s3_admin_secret_key.txt) exec aws --endpoint-url http://s3-proxy:8333 "$@"' sh "$@" diff --git a/scripts/s3_restart_probe.py b/scripts/s3_restart_probe.py new file mode 100644 index 0000000..5046b2a --- /dev/null +++ b/scripts/s3_restart_probe.py @@ -0,0 +1,57 @@ +import base64, hashlib, json, struct, subprocess, time, urllib.error, urllib.request, uuid, zlib + +BASE = "http://localhost:8080"; PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, headers=None, token=None, basic=None, timeout=10): + h = dict(headers or {}) + if token: h["Authorization"] = "Bearer " + token + if basic: h["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() + req = urllib.request.Request(BASE + path, data=body, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def jcall(method, path, obj, token=None): + st, raw = call(method, path, json.dumps(obj).encode(), {"Content-Type": "application/json"}, token) + return st, json.loads(raw or b"null") + + +def tiny_png(): + def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + raw = b"".join(b"\x00" + b"\xff\x00\x00" * 8 for _ in range(8)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +name = "pr_" + uuid.uuid4().hex[:8] +_, user = jcall("POST", "/api/v1/users", {"username": name, "display_name": "probe", "password": PWD}) +st, raw = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +tok = json.loads(raw)["token"] +_, chat = jcall("POST", "/api/v1/chats", {"title": "probe", "member_ids": [user["id"]]}, tok) +png = tiny_png() +st, raw = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png", png, + {"Content-Type": "image/png"}, tok) +att = json.loads(raw) +print("upload:", st, att.get("id")) +want = hashlib.sha256(png).hexdigest() + +t0 = time.time() +subprocess.run(["docker", "compose", "restart", "seaweedfs"], capture_output=True) +print(f"restart seaweedfs returned after {time.time() - t0:.1f}s") +while time.time() - t0 < 90: + t1 = time.time() + try: + st, data = call("GET", f"/api/v1/attachments/{att['id']}/content", token=tok, timeout=5) + ok = st == 200 and hashlib.sha256(data).hexdigest() == want + print(f"t={time.time() - t0:5.1f}s status={st} bytes_ok={ok} took={time.time() - t1:.1f}s") + if ok: + break + except Exception as e: + print(f"t={time.time() - t0:5.1f}s {type(e).__name__} took={time.time() - t1:.1f}s") + time.sleep(1) \ No newline at end of file diff --git a/scripts/upload_tools.py b/scripts/upload_tools.py new file mode 100644 index 0000000..3f74735 --- /dev/null +++ b/scripts/upload_tools.py @@ -0,0 +1,55 @@ +import base64, json, socket, struct, sys, urllib.request, uuid, zlib + +BASE_HOST, BASE_PORT = "localhost", 8080 +PWD = "correct-horse-battery-staple" + + +def api(method, path, body=None, headers=None): + req = urllib.request.Request(f"http://{BASE_HOST}:{BASE_PORT}{path}", data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +PNG = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"".join(b"\x00" + b"\x10\x80\xf0" * 8 for _ in range(8)))) + chunk(b"IEND", b"")) + +name = "up_" + uuid.uuid4().hex[:8] +j = {"Content-Type": "application/json"} +_, raw = api("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "u", "password": PWD}).encode(), j) +user = json.loads(raw) +_, raw = api("POST", "/api/v1/auth/sessions", None, {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +token = json.loads(raw)["token"] +_, raw = api("POST", "/api/v1/chats", json.dumps({"title": "u", "member_ids": [user["id"]]}).encode(), {**j, "Authorization": "Bearer " + token}) +chat = json.loads(raw) + +mode = sys.argv[1] if len(sys.argv) > 1 else "ok" +if mode == "ok": + st, raw = api("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=u.png", PNG, + {"Content-Type": "image/png", "Authorization": "Bearer " + token}) + print("upload status:", st) +elif mode == "abort": + # заявляем 5 МБ отправляем 300 КБ и закрываем соединение + s = socket.create_connection((BASE_HOST, BASE_PORT)) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=a.png HTTP/1.1\r\nHost: localhost\r\n" + f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + s.sendall(head.encode() + PNG + b"\0" * 300_000) + s.close() + print("aborted upload after 300 KB of 5000000") +elif mode == "roundtrip": + import hashlib + st, raw = api("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=r.png", PNG, + {"Content-Type": "image/png", "Authorization": "Bearer " + token}) + if st != 201: + print("upload", st); sys.exit(1) + att = json.loads(raw) + st, data = api("GET", f"/api/v1/attachments/{att['id']}/content", None, {"Authorization": "Bearer " + token}) + good = st == 200 and hashlib.sha256(data).hexdigest() == att["sha256"] + print("roundtrip", "ok" if good else f"fail {st}") + sys.exit(0 if good else 1) \ No newline at end of file diff --git a/scripts/verify_level4.sh b/scripts/verify_level4.sh new file mode 100755 index 0000000..ca48482 --- /dev/null +++ b/scripts/verify_level4.sh @@ -0,0 +1,60 @@ +#!/bin/sh +cd "$(dirname "$0")/.." +mkdir -p evidence/steps +RESULTS="" + +step() { + name="$1"; shift + log="evidence/steps/$name.log" + echo ">>> $name" + "$@" > "$log" 2>&1 + code=$? + if [ "$code" -ne 0 ] || grep -q '^FAIL' "$log"; then st=FAIL; else st=PASS; fi + echo " $st (exit=$code) -> $log" + RESULTS="$RESULTS\n$st $name" +} +settle() { docker compose up -d --no-recreate api >/dev/null 2>&1; ./scripts/wait_ready.sh 150 >/dev/null 2>&1; } +infra() { + docker compose ps -a + for c in $(docker compose ps -aq); do + docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}}' "$c" + done + echo "container_name in compose: $(docker compose config | grep -c container_name || true)" + docker compose config --services + docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(docker compose ps -q api | head -1)" | grep -i -E "DATABASE|PASSWORD|S3_" || true + docker inspect -f '{{.Name}}: {{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' $(docker compose ps -q) 2>/dev/null +} +s3init_twice() { docker compose run --rm -T s3-init && docker compose run --rm -T s3-init; } + +step 00-up sh -c 'docker compose --env-file .env up -d --build && ./scripts/wait_ready.sh 180' +step 01-go-tests sh -c 'go build ./... && go vet ./... && go test ./...' +step 02-make-check make check +step 03-make-test make test +step 04-lb python3 scripts/check_lb.py +step 05-edge python3 scripts/check_edge.py +step 06-attachments python3 scripts/check_attachments.py +step 07-s3-rights ./scripts/check_s3.sh +step 08-s3-init-twice s3init_twice +step 09-orphans ./scripts/orphan_scenarios.sh +step 10-memory-1mb python3 scripts/memory_experiment.py 1000000 +step 11-memory-9mb python3 scripts/memory_experiment.py 9437184 +step 12-att-failover-1 python3 scripts/check_attachment_failover.py +settle +step 13-att-failover-2 python3 scripts/check_attachment_failover.py +settle +step 14-persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh +settle +step 15-failover-1 make test-failover ACTION_SCRIPT=scripts/stop-one.sh +settle +step 16-failover-2 make test-failover ACTION_SCRIPT=scripts/stop-one.sh +settle +step 17-infra infra + +if [ "${ROLLING:-0}" = 1 ]; then + step 18-rolling ./scripts/rolling-under-load.sh + settle +fi + +echo +echo "=========== ИТОГ ===========" +printf '%b\n' "$RESULTS" | tee evidence/SUMMARY.txt diff --git a/scripts/wait_ready.sh b/scripts/wait_ready.sh new file mode 100755 index 0000000..74f040d --- /dev/null +++ b/scripts/wait_ready.sh @@ -0,0 +1,12 @@ +#!/bin/sh +cd "$(dirname "$0")/.." +limit="${1:-120}" +end=$(( $(date +%s) + limit )) +while [ "$(date +%s)" -lt "$end" ]; do + if [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8080/health/ready)" = 200 ] \ + && python3 scripts/upload_tools.py roundtrip >/dev/null 2>&1; then + echo "ready"; exit 0 + fi + sleep 2 +done +echo "NOT READY after ${limit}s"; exit 1