From 57a253e0b4367795a1feeeba81b2cc4fa2108503 Mon Sep 17 00:00:00 2001 From: rey256 Date: Tue, 17 Nov 2026 17:32:06 +0300 Subject: [PATCH] Prepare verified laboratory 5 implementation and evidence --- .dockerignore | 28 +- .github/workflows/application.yml | 26 + .gitignore | 32 +- Dockerfile | 14 + REPORT.md | 76 +- SUBMISSION-MANIFEST.json | 163 + SUBMISSION.md | 5 + backend/.golangci.yaml | 36 + backend/cmd/app/main.go | 195 + backend/cmd/worker/main.go | 134 + backend/codegen.yaml | 15 + backend/db/migrate.go | 25 + backend/db/migrations/00001_core.sql | 45 + backend/db/migrations/00002_idempotency.sql | 12 + backend/db/migrations/00003_attachments.sql | 49 + backend/db/migrations/00005_replay_audit.sql | 4 + backend/db/sqlc/queries/attachments.sql | 15 + backend/db/sqlc/queries/core.sql | 42 + backend/db/sqlc/queries/idempotency.sql | 12 + backend/db/sqlc/queries/pipeline.sql | 50 + backend/db/sqlc/sqlc.yaml | 16 + backend/generator-overlay.yaml | 15 + backend/go.mod | 331 ++ backend/go.sum | 1302 +++++ backend/integration/object_metadata_test.go | 67 + backend/internal/adapter/handler/api.gen.go | 4431 +++++++++++++++++ backend/internal/adapter/handler/api.go | 111 + .../internal/adapter/handler/attachments.go | 96 + backend/internal/adapter/handler/auth.go | 47 + backend/internal/adapter/handler/errors.go | 85 + backend/internal/adapter/handler/gate.go | 78 + backend/internal/adapter/handler/gate_test.go | 102 + backend/internal/adapter/handler/handler.go | 102 + backend/internal/adapter/handler/json.go | 65 + .../adapter/handler/message_json_test.go | 17 + .../internal/adapter/handler/middleware.go | 161 + backend/internal/adapter/handler/stats.go | 47 + backend/internal/adapter/handler/utf8_test.go | 21 + backend/internal/adapter/imagefile/image.go | 72 + .../internal/adapter/imagefile/image_test.go | 81 + .../internal/adapter/imagefile/stress_test.go | 135 + .../internal/adapter/imagefile/variants.go | 129 + .../imagefile/variants_integration_test.go | 93 + .../adapter/imagefile/variants_test.go | 12 + backend/internal/adapter/password/argon.go | 36 + backend/internal/adapter/queue/streams.go | 133 + .../repository/postgres/attachments.go | 57 + .../adapter/repository/postgres/pipeline.go | 171 + .../postgres/pipeline_integration_test.go | 331 ++ .../postgres/queries/attachments.sql.go | 193 + .../repository/postgres/queries/core.sql.go | 358 ++ .../adapter/repository/postgres/queries/db.go | 32 + .../postgres/queries/idempotency.sql.go | 81 + .../repository/postgres/queries/models.go | 97 + .../postgres/queries/pipeline.sql.go | 393 ++ .../adapter/repository/postgres/repository.go | 243 + backend/internal/adapter/storage/bootstrap.go | 86 + .../adapter/storage/readiness_test.go | 21 + backend/internal/adapter/storage/s3.go | 81 + .../application/usecase/attachments.go | 79 + backend/internal/application/usecase/auth.go | 86 + .../internal/application/usecase/messenger.go | 102 + .../internal/application/usecase/pipeline.go | 138 + .../usecase/pipeline_retry_test.go | 135 + .../internal/application/usecase/reconcile.go | 166 + .../usecase/reconcile_pipeline_test.go | 31 + .../application/usecase/reconcile_test.go | 184 + backend/internal/config/config.go | 86 + backend/internal/config/config_test.go | 24 + backend/internal/domain/model/attachment.go | 35 + backend/internal/domain/model/chat.go | 52 + backend/internal/domain/model/errors.go | 33 + backend/internal/domain/model/job.go | 24 + backend/internal/domain/model/message.go | 82 + backend/internal/domain/model/model_test.go | 126 + backend/internal/domain/model/page.go | 84 + .../domain/model/postgres_text_test.go | 13 + backend/internal/domain/model/text.go | 24 + backend/internal/domain/model/user.go | 34 + backend/internal/domain/port/auth.go | 18 + backend/internal/domain/port/objects.go | 20 + backend/internal/domain/port/pipeline.go | 28 + .../internal/domain/repository/repository.go | 64 + backend/pkg/logger/logger.go | 18 + backend/pkg/server/server.go | 29 + compose.yaml | 122 + config/haproxy.cfg | 27 + evidence/both-replicas.json | 19 + evidence/both-replicas.txt | 2 + evidence/final-audit-build.txt | 149 + evidence/final-checks.txt | 59 + evidence/final-runtime.txt | 37 + .../fresh-db-messenger_stage_70b6ccce.json | 1 + evidence/golangci-lint.txt | 6 + evidence/green-replay-version2.txt | 6 + evidence/openapi-validate.txt | 2 + evidence/public-acceptance.txt | 10 + evidence/red-completed-cleanup.txt | 6 + evidence/red-fit.txt | 6 + evidence/red-reconciler.txt | 19 + evidence/red-replay-audit.txt | 31 + evidence/red-replay-version2.txt | 8 + evidence/red-transform-infrastructure.txt | 6 + evidence/replay-version2-command.md | 22 + evidence/runtime-commit-publish-gap.txt | 2 + .../runtime-pipeline-faults-after_commit.json | 14 + ...runtime-pipeline-faults-after_objects.json | 14 + ...ine-faults-after_original-restart-gap.json | 15 + evidence/runtime-pipeline-faults.json | 14 + evidence/runtime-reconciler-red.json | 10 + evidence/validate-dependencies.txt | 62 + evidence/versioned-fault-matrix.txt | 29 + scripts/bootstrap.ts | 35 + scripts/compose.sh | 4 + scripts/restart.sh | 4 + scripts/stop-one.sh | 12 + scripts/up.sh | 10 + student-tests/both-replicas.ts | 12 + student-tests/pipeline-faults.ts | 34 + student-tests/reconciler-red.ts | 20 + student-tests/run-db-matrix.sh | 11 + student-tests/wait.ts | 14 + 122 files changed, 13404 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 SUBMISSION-MANIFEST.json create mode 100644 SUBMISSION.md create mode 100644 backend/.golangci.yaml create mode 100644 backend/cmd/app/main.go create mode 100644 backend/cmd/worker/main.go create mode 100644 backend/codegen.yaml create mode 100644 backend/db/migrate.go create mode 100644 backend/db/migrations/00001_core.sql create mode 100644 backend/db/migrations/00002_idempotency.sql create mode 100644 backend/db/migrations/00003_attachments.sql create mode 100644 backend/db/migrations/00005_replay_audit.sql create mode 100644 backend/db/sqlc/queries/attachments.sql create mode 100644 backend/db/sqlc/queries/core.sql create mode 100644 backend/db/sqlc/queries/idempotency.sql create mode 100644 backend/db/sqlc/queries/pipeline.sql create mode 100644 backend/db/sqlc/sqlc.yaml create mode 100644 backend/generator-overlay.yaml create mode 100644 backend/go.mod create mode 100644 backend/go.sum create mode 100644 backend/integration/object_metadata_test.go create mode 100644 backend/internal/adapter/handler/api.gen.go create mode 100644 backend/internal/adapter/handler/api.go create mode 100644 backend/internal/adapter/handler/attachments.go create mode 100644 backend/internal/adapter/handler/auth.go create mode 100644 backend/internal/adapter/handler/errors.go create mode 100644 backend/internal/adapter/handler/gate.go create mode 100644 backend/internal/adapter/handler/gate_test.go create mode 100644 backend/internal/adapter/handler/handler.go create mode 100644 backend/internal/adapter/handler/json.go create mode 100644 backend/internal/adapter/handler/message_json_test.go create mode 100644 backend/internal/adapter/handler/middleware.go create mode 100644 backend/internal/adapter/handler/stats.go create mode 100644 backend/internal/adapter/handler/utf8_test.go create mode 100644 backend/internal/adapter/imagefile/image.go create mode 100644 backend/internal/adapter/imagefile/image_test.go create mode 100644 backend/internal/adapter/imagefile/stress_test.go create mode 100644 backend/internal/adapter/imagefile/variants.go create mode 100644 backend/internal/adapter/imagefile/variants_integration_test.go create mode 100644 backend/internal/adapter/imagefile/variants_test.go create mode 100644 backend/internal/adapter/password/argon.go create mode 100644 backend/internal/adapter/queue/streams.go create mode 100644 backend/internal/adapter/repository/postgres/attachments.go create mode 100644 backend/internal/adapter/repository/postgres/pipeline.go create mode 100644 backend/internal/adapter/repository/postgres/pipeline_integration_test.go create mode 100644 backend/internal/adapter/repository/postgres/queries/attachments.sql.go create mode 100644 backend/internal/adapter/repository/postgres/queries/core.sql.go create mode 100644 backend/internal/adapter/repository/postgres/queries/db.go create mode 100644 backend/internal/adapter/repository/postgres/queries/idempotency.sql.go create mode 100644 backend/internal/adapter/repository/postgres/queries/models.go create mode 100644 backend/internal/adapter/repository/postgres/queries/pipeline.sql.go create mode 100644 backend/internal/adapter/repository/postgres/repository.go create mode 100644 backend/internal/adapter/storage/bootstrap.go create mode 100644 backend/internal/adapter/storage/readiness_test.go create mode 100644 backend/internal/adapter/storage/s3.go create mode 100644 backend/internal/application/usecase/attachments.go create mode 100644 backend/internal/application/usecase/auth.go create mode 100644 backend/internal/application/usecase/messenger.go create mode 100644 backend/internal/application/usecase/pipeline.go create mode 100644 backend/internal/application/usecase/pipeline_retry_test.go create mode 100644 backend/internal/application/usecase/reconcile.go create mode 100644 backend/internal/application/usecase/reconcile_pipeline_test.go create mode 100644 backend/internal/application/usecase/reconcile_test.go create mode 100644 backend/internal/config/config.go create mode 100644 backend/internal/config/config_test.go create mode 100644 backend/internal/domain/model/attachment.go create mode 100644 backend/internal/domain/model/chat.go create mode 100644 backend/internal/domain/model/errors.go create mode 100644 backend/internal/domain/model/job.go create mode 100644 backend/internal/domain/model/message.go create mode 100644 backend/internal/domain/model/model_test.go create mode 100644 backend/internal/domain/model/page.go create mode 100644 backend/internal/domain/model/postgres_text_test.go create mode 100644 backend/internal/domain/model/text.go create mode 100644 backend/internal/domain/model/user.go create mode 100644 backend/internal/domain/port/auth.go create mode 100644 backend/internal/domain/port/objects.go create mode 100644 backend/internal/domain/port/pipeline.go create mode 100644 backend/internal/domain/repository/repository.go create mode 100644 backend/pkg/logger/logger.go create mode 100644 backend/pkg/server/server.go create mode 100644 compose.yaml create mode 100644 config/haproxy.cfg create mode 100644 evidence/both-replicas.json create mode 100644 evidence/both-replicas.txt create mode 100644 evidence/final-audit-build.txt create mode 100644 evidence/final-checks.txt create mode 100644 evidence/final-runtime.txt create mode 100644 evidence/fresh-db-messenger_stage_70b6ccce.json create mode 100644 evidence/golangci-lint.txt create mode 100644 evidence/green-replay-version2.txt create mode 100644 evidence/openapi-validate.txt create mode 100644 evidence/public-acceptance.txt create mode 100644 evidence/red-completed-cleanup.txt create mode 100644 evidence/red-fit.txt create mode 100644 evidence/red-reconciler.txt create mode 100644 evidence/red-replay-audit.txt create mode 100644 evidence/red-replay-version2.txt create mode 100644 evidence/red-transform-infrastructure.txt create mode 100644 evidence/replay-version2-command.md create mode 100644 evidence/runtime-commit-publish-gap.txt create mode 100644 evidence/runtime-pipeline-faults-after_commit.json create mode 100644 evidence/runtime-pipeline-faults-after_objects.json create mode 100644 evidence/runtime-pipeline-faults-after_original-restart-gap.json create mode 100644 evidence/runtime-pipeline-faults.json create mode 100644 evidence/runtime-reconciler-red.json create mode 100644 evidence/validate-dependencies.txt create mode 100644 evidence/versioned-fault-matrix.txt create mode 100644 scripts/bootstrap.ts create mode 100755 scripts/compose.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/up.sh create mode 100644 student-tests/both-replicas.ts create mode 100644 student-tests/pipeline-faults.ts create mode 100644 student-tests/reconciler-red.ts create mode 100755 student-tests/run-db-matrix.sh create mode 100644 student-tests/wait.ts diff --git a/.dockerignore b/.dockerignore index bc16e556..f8c8530f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,8 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +**/.git +**/.local +**/.env +**/secrets +**/evidence +**/node_modules +**/.venv + diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..a720f245 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,26 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.2' + cache-dependency-path: backend/go.sum + - run: ./scripts/up.sh + - run: bun student-tests/wait.ts + - run: make check test BASE_URL=http://127.0.0.1:28205 + - run: make test-persistence BASE_URL=http://127.0.0.1:28205 ACTION_SCRIPT=./scripts/restart.sh + - run: bun student-tests/both-replicas.ts + - run: ./student-tests/run-db-matrix.sh + - run: bun student-tests/pipeline-faults.ts --point=after_original --restart-gap + - run: bun student-tests/pipeline-faults.ts --point=after_commit + - name: Stop only this stand + if: always() + run: ./scripts/compose.sh stop diff --git a/.gitignore b/.gitignore index b5a71896..14876cce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,6 @@ -# Local/private state +.local/ .env -.env.* -!.env.example -secrets/ -certs/private/ -*.key -*.p12 -*.pfx -*.jks -*.pem -*.tfstate* -volumes/ -data/ -backups/ -# Runtime/build artifacts .venv/ __pycache__/ -*.py[cod] -node_modules/ -dist/ -build/ -target/ -bin/ -obj/ -coverage/ -*.log -.DS_Store -.idea/ -.vscode/ -# Keep small, sanitized evidence summaries in evidence/ under version control. -# Inspect git diff --cached: ignore rules are not a secret scanner. +*.pyc + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..585506e5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build +WORKDIR /src +ENV CGO_ENABLED=0 GOFLAGS=-mod=readonly +COPY backend/ ./ +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -trimpath -ldflags='-s -w' -o /out/messenger ./cmd/app && go build -trimpath -ldflags='-s -w' -o /out/worker ./cmd/worker +FROM debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f +RUN apt-get update && apt-get install -y --no-install-recommends imagemagick ca-certificates && apt-get clean +COPY --from=build /out/messenger /messenger +COPY --from=build /out/worker /worker +USER 65532:65532 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=384MiB +ENTRYPOINT ["/messenger"] + diff --git a/REPORT.md b/REPORT.md index e98f75e9..820253b9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,67 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: Медведев Андрей Михайлович, БПИ2303. Цель: все требования уровней3–4 и трек2 уровня5 «Доказанная идемпотентность pipeline». Оценка преподавателем не присвоена. Публикация и PR не выполнялись; подготовлен архив для сдачи. -## Запуск из чистого clone +Основа: исходный skeleton ЛР5, commit276dbdad8a34da4c5b57fbf1879ca169b581d75b, и собственная ЛР4 из `/Users/rey/projects/vp-mtuci` (оригиналы не редактировались). Сохранены новые README, COURSE, contracts и публичные tests текущей работы. Реализацию и проверки помог подготовить ИИ; независимые находки reviewer исправлены с RED/GREEN. Первичные источники: [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/), [PostgreSQL UPDATE](https://www.postgresql.org/docs/current/sql-update.html), [ImageMagick limits](https://imagemagick.org/script/resources.php), контракты этого skeleton. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Запуск -## Архитектура и решения +Нужны Docker с Compose, Bun и Python3.9+; для расширенной PostgreSQL matrix нужен Go1.27.2 в PATH (или переменная GO с путём к нему). Проверено на Linux x86_64, Go1.27.2, Bun1.4.4, Docker29.9.2/Compose5.6.1. Все сервисы запускаются этим репозиторием. Образы Go1.27.2-alpine, Debian, PostgreSQL18.6, Valkey9, RustFS1.0.0-rc.1 и HAProxy3.2 закреплены digest в Dockerfile/compose.yaml. Runtime устанавливает ImageMagick из Debian; версию пакета фиксирует выбранный базовый release и его репозиторий, отдельного immutable snapshot apt нет. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```sh +./scripts/up.sh +bun student-tests/wait.ts +make check +make test BASE_URL=http://127.0.0.1:28205 +make test-persistence BASE_URL=http://127.0.0.1:28205 ACTION_SCRIPT=./scripts/restart.sh +make test-failover BASE_URL=http://127.0.0.1:28205 ACTION_SCRIPT=./scripts/stop-one.sh +bun student-tests/both-replicas.ts +./student-tests/run-db-matrix.sh +bun student-tests/pipeline-faults.ts --point=after_original --restart-gap +bun student-tests/pipeline-faults.ts --point=after_objects +bun student-tests/pipeline-faults.ts --point=after_commit +./scripts/compose.sh stop +``` -## Доказательства критериев +`up.sh` генерирует отдельные учебные credentials в `.local` (родитель0700), строит один общий image, запускает зависимости, миграции, S3 bootstrap и две API+worker+LB. `.local` не входит в архив/git. При частично существующей конфигурации bootstrap останавливается, не затирая её. `compose.sh` всегда закрепляет manifest и project, поэтому чужие COMPOSE_* не перенаправят операции. Публичен только LB127.0.0.1:28205; остальные компоненты в private/internal network, edge есть лишь у LB. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Остановка сохраняет volumes. Намеренное удаление только этого учебного стенда и всех его данных: `./scripts/compose.sh down --volumes`. Этот разрушительный сценарий в приёмке не запускался. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Архитектура и ограничения -## Эксперименты и отказы +API сохраняет validated original в S3 `staging/`, затем одной PostgreSQL транзакцией переводит metadata в queued и создаёт outbox/job. Только после COMMIT возвращается202. Отдельный worker переносит intent в Valkey Streams и помечает публикацию в БД; сбой между XADD и DB отметкой создаёт допустимый дубль. Valkey AOF `appendfsync always`, PostgreSQL и S3 имеют отдельные persistent volumes. Недоступность broker после COMMIT не теряет DB intent. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Сообщение содержит только схему/тип, UUID, версию, attempt, created_at и необязательный trace carrier; bytes, filename/URL и credentials в broker не попадают. Job UUID устойчив между повторами. Outbox читает до100 jobs каждые500ms. Stream удерживает примерно10000 записей; его trimming не удаляет источник истины — незавершённое intent повторно публикуется. XREADGROUP/claim count1, один worker/одно изображение, lease30s и обработка context20s. PEL восстанавливается XAUTOCLAIM через30s. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Worker берёт доверенные metadata из БД, читает staging (или сохранённый immutable original при новой версии), повторно проверяет размер и SHA256, сохраняет `originals/` и три `variants//v/`. Все variants публикуются атомарным fenced UPDATE только при текущей версии, lease и processing. До этого content409; ready устанавливается лишь после всех S3 записей. Ack идёт после durable результата либо durable next retry/failed, а не после самого получения сообщения. -## Ограничения и следующие шаги +Ограничения изображения: до10MiB и40M pixels, PNG/JPEG; bounded временный файл и output16MiB на variant, tmpfs256MiB, subprocess context20s. ImageMagick memory/map256MiB, disk0, resize thread1. Thumbnail вписывается в256×256 без увеличения (floor, минимум1), crop128×128 центрирован с допустимым увеличением, optimized WebP вписывается в1280×1280 без увеличения. EXIF orientation применяется, alpha ложится на белый фон, metadata удаляются, quality82. Декодер-процесс и Go имеют также Docker RAM/CPU limits. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +API: две реплики по1CPU/2GiB, MAX_INFLIGHT4, MAX_QUEUE32; worker1CPU/2GiB и GOMEMLIMIT384MiB; PostgreSQL/S3 по768MiB, Valkey/LB по128MiB. Это configured limits, не измеренный расход памяти. Пароли с Argon2 и hashed session tokens сохранены из ЛР4; TLS/OpenBao — отдельная ЛР7. + +Retry по умолчанию5 попыток (конфигурация1..10), экспоненциальный backoff100ms×2^attempt плюс jitter0..99ms с ограниченным exponent. Только typed invalid_image permanent; ENOSPC, отсутствующий executable, S3/DB/timeout остаются transient. Исчерпание даёт failed/retry_exhausted в БД (реестр failed вместо отдельного broker DLQ). `./scripts/compose.sh run --rm worker --replay UUID` явно повторяет только failed/current/source-retained job; ready-дубль не исполняется. Replay атомарно сохраняет replay_count и replayed_at вместе со сбросом attempts. `--advance UUID` создаёт новую pipeline_version для operator-controlled version experiment. + +Cleanup имеет отдельный durable staging_cleaned marker и выполняется независимо каждые1s, а также при redelivery completed job. Ready stage удаляется только при наличии original ожидаемого размера; неуспешный/unbound stage получает retention24h. Ошибка удаления не переводит ready в failed и будет повторена. Legacy reconcile не меняет versioned pipeline metadata; старый путь больше не может отметить original-only запись ready. + +## Проверки и наблюдения + +- `make check`, Go race tests, `go vet`, golangci-lint и non-writing gofmt check: результаты в `evidence/final-checks.txt`, `evidence/final-audit-build.txt` и финальном manifest. Env-gated интеграции отдельно, обычный `go test` не выдаётся за их запуск. +- Публичный smoke, сохранность после PostgreSQL/Valkey/S3/API/worker/LB restart и failover с существующей сессией: `evidence/public-acceptance.txt`. Во время failover126 успешных записей,1 неудачная попытка, первое подтверждённое восстановление1.492s. Это наблюдение конкретного теста, не benchmark/SLO. +- Обе реальные API реплики отдельно остановлены; существующая сессия и все подтверждённые сообщения сохранены, surviving instance отличается от остановленной. `evidence/both-replicas.json`:1025ms и27ms — время наблюдения после stop в двух проверках, не универсальный recovery bound. +- Реальный ImageMagick integration: landscape320×200, tiny1×1, portrait23×67, EXIF rotation, JPEG content/white alpha/geometry/hash и WebP magic. Команда `RUN_VARIANT_TESTS=1 go test -v ./internal/adapter/imagefile`, runtime выполнялся в image с ImageMagick. +- Crash после S3 original/variants и после результата COMMIT: реальный worker `-fault` завершён os.Exit88. Actual legacy `/messenger -reconcile -apply -grace1m` не публикует неполную processing запись. После lease recovery готово3 variants, original byte-exact, staging_cleaned=true. JSON `evidence/runtime-pipeline-faults-after_*.json`. +- COMMIT/publish gap: worker уже остановлен, API202, DB job published_at=epoch, content409, сообщение с attachment создаётся. Затем реальные API+Valkey restart с volumes, запуск fault worker и обычное восстановление того же UUID без создания job вручную. `evidence/runtime-pipeline-faults-after_original-restart-gap.json`. +- Versioned pipeline: настоящий PostgreSQLStore+S3+Pipeline, concurrent current duplicates, старая v1 до/после v2, fabricated future delivery, повтор v2. Детерминированный test Transformer выдаёт разные v1/v2 bytes (остальное реально); v2 metadata и S3 bytes остаются неизменны. ENOSPC×5 даёт failed/retry_exhausted, explicit replay восстанавливает ready. После after_commit completed-redelivery очищает настоящий S3 staging. `evidence/versioned-fault-matrix.txt` и `evidence/final-runtime.txt`. +- Regression готовой v1→normal stage cleanup→advance v2→failed→replay сначала воспроизвёл отказ, затем прошёл на реальном SQL после исправления staging_expired; отдельный RED/GREEN сохранён. Audit replay также проверен сначала RED без сохранённого count/timestamp, затем GREEN. `evidence/{red,green}-replay-version2.txt`, `evidence/red-replay-audit.txt`, финальная DB matrix. + +## Трек2 уровня5 + +Гипотеза: at-least-once доставка допускает дубли и reorder без потери durable принятого intent и без публикации результата старой версии поверх новой, если владельцем результата остаётся PostgreSQL lease+version fence, а ключ S3 эффекта включает version. Проверяемые инварианты: immutable original; один logical job на attachment/version; только действующий lease публикует ready; old/future/completed deliveries не меняют текущий результат; повтор после COMMIT может безопасно завершить cleanup. + +Методика и команды выше автоматизированы. Падение after_original одновременно находится после side effect original и до side effects variants; after_objects — после variants до result COMMIT; after_commit — после durable результата до ack/cleanup. Дубли/reorder проверены вызовом того же реального Pipeline consumer method на настоящих PG/S3 с различимым oracle. Процессные падения и рестарты дополнительно проверены настоящим worker и broker. Тестовый Transformer не заменяет ImageMagick acceptance: native image scenarios проверяются отдельно. + +Exactly-once доставка не заявлена: XADD и DB отметка публикации не атомарны, ack может потеряться, S3 PUT повторяется; exactly-once эффект обеспечивается в пределах logical key/fenced publication, а не отсутствием повторных вычислений/записей. + +## Практические пределы + +Один host/PostgreSQL/Valkey/S3 не защищает от потери host или volumes. Не измерены throughput/p95/RSS под перегрузкой: выбран fault-injection трек, benchmark во время игры запрещён. Retry/retention управляются БД clock; на зависшем host hard wall-clock guarantee отсутствует. Детерминированность разных реализаций ImageMagick не заявлена: версии pipeline и metadata фиксируют выбранный эффект. Старые versioned объекты сохраняются для исследования; автоматическая политика удаления исторических версий здесь не реализована. Публикация в student fork, PR и оценка преподавателем — следующие внешние действия владельца. diff --git a/SUBMISSION-MANIFEST.json b/SUBMISSION-MANIFEST.json new file mode 100644 index 00000000..2d0b3518 --- /dev/null +++ b/SUBMISSION-MANIFEST.json @@ -0,0 +1,163 @@ +{ + "createdAt": "2026-11-15T17:44:00.621Z", + "assignment": "VP Lab5", + "student": "Медведев Андрей Михайлович, БПИ2303", + "skeleton": "276dbdad8a34da4c5b57fbf1879ca169b581d75b", + "validation": "Go1.27.2 Linux unit/race/vet/lint/build; full OpenAPI; public smoke/persistence/both-replica failures; actual worker crashes; real PostgreSQL/S3 fault matrix and audit", + "fileHashes": { + ".dockerignore": "62eeb54240e2ee9fda932ed40912aa78983f3be6abc48e44612df399645f5dbb", + ".env.example": "bf057714d05158b66620099ec06b5d81430fd3ed371fabaac47143b5fd4c3a3a", + ".github/pull_request_template.md": "7cfeabfbcb19f34d85ed99cbcd6797977d6d722e65f7533e859147d5ab4d5e9b", + ".github/workflows/application.yml": "fd75a11c8fe9c9c07d2a68078a92df4077516955e8d75cb120358f4c947196cd", + ".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8", + ".gitignore": "385e0e7c81ca13ebf0bab2ebcaf51c1dbfe9af14a01714886d98b5689718b195", + "CONTRIBUTING.md": "2c01dd698cc9653dece49a94c9fd3f58d36e84764aaa643b46e26d7e599dc12e", + "COURSE.md": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971", + "Dockerfile": "40095fcce1fc5c43eea0f7cbc0cce55ab6e630969abafdaddc64ce4f9a778fb4", + "Dockerfile.example": "fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709", + "MAINTAINERS.md": "2ac3103bc622557ab5e550d716dbdb62bcccf8fc4c0eb410bcbfe5a9f127e7ff", + "Makefile": "c05bd07058ae0949dd4905afdae7674bc7e3e63863a6cdf8b72661140d445826", + "README.md": "0651974eebbd4f34a04cd7c06df167cf61ae686cc67ceb2ac977a115ad70b866", + "REPORT.md": "2dfdb23f1837c614d3d35f25ba85e2363701a21af86f0e629493575761f571bf", + "SUBMISSION.md": "768525e53d40e8541b9a95c2202abe0ee3ab38f29ab496c869ac461a8511cff6", + "backend/.golangci.yaml": "215578dee14bcef1b66759a68543083e0a624611efcdb785197307b50cb7725d", + "backend/cmd/app/main.go": "89f8cdf64c4a22a2ae65fb5b88f97dc7bfff7a1aff6cb0c766d3f495ea14fcd3", + "backend/cmd/worker/main.go": "97ae2cafaf861443d0a56a635739f80f66b878b31fffd52f8136888e17c466ea", + "backend/codegen.yaml": "e4d37ce5c5a9f969f696833cd59a6e1b86f99fb2b0de69101a1aa5f4a539aebe", + "backend/db/migrate.go": "33044d905354f24fff194fefd35ee9327d63d6e5062ded8496e518b6ce38dbea", + "backend/db/migrations/00001_core.sql": "b6bd8c70d51aed6c966e07eebffd262af2e694c8b3479c70bbf9a9eacf5a7aad", + "backend/db/migrations/00002_idempotency.sql": "97c98785547db8391c1fa0a685e4d4bcc810b8f79055c0e2fc6ef473cd38b37b", + "backend/db/migrations/00003_attachments.sql": "6b4f707dde32f0c3f531741c108bcf0f22be05c9e2ede327f27a1d9ef0cfcc7f", + "backend/db/migrations/00005_replay_audit.sql": "234a04177bca08621fa38b135cb8086bcdd55ae9084008b25814751e1dd742a4", + "backend/db/sqlc/queries/attachments.sql": "8e64f0e6169fab7664b0f4a97f2be353553b93c0c928d10678cb930e44de37da", + "backend/db/sqlc/queries/core.sql": "1aed3d0d2fc04650351d86d15837cf3239636844727e799aabf5ddd534429f33", + "backend/db/sqlc/queries/idempotency.sql": "5bdec86e85705f7805fb603e2c792bb87c11cf3156952fa48e7bcee35fd683ca", + "backend/db/sqlc/queries/pipeline.sql": "6c5fa5356024da6e70b7e3116a0765e2865342d7ca474caa4f647c04e4fb9cf4", + "backend/db/sqlc/sqlc.yaml": "803b161e063692398c93152b11a08f9e42e1bf3c72a1e64553b82c494545fd10", + "backend/generator-overlay.yaml": "e666da2959b31befed713a8366237de53070a565f93850d6daae7a582a038819", + "backend/go.mod": "0b8da0a50d49decdd1b36c920363f5908b372a786a85bd6c2b789ce781ac8f91", + "backend/go.sum": "b981fecb6445955924c2162a1ca9a6877820143af458df963624b3ceb583b861", + "backend/integration/object_metadata_test.go": "598e973575d57c6ee563f0b15d0111d37b27de6774c6f486c998dbc70594f15d", + "backend/internal/adapter/handler/api.gen.go": "4034501f5c661c87af3b81d0d1ecb9996050828569bea7e85b3604a2a8c8bdbe", + "backend/internal/adapter/handler/api.go": "9d9f109e13c9322287b29ed18e7913e0e8d5a9ef2dc4be23f10c7686de3f70fe", + "backend/internal/adapter/handler/attachments.go": "608827653c4d3e4252dff863d30007e93b1bf82579bbe71d8b647bf22128b1b2", + "backend/internal/adapter/handler/auth.go": "3889f58ac4c58b798192776ea05455d52735a2c024ba664ab9af5eb405187557", + "backend/internal/adapter/handler/errors.go": "39a5b916112fd9cc2be9979b6e78d6305e7c3c8ab5fa462f1b8e9f18c88c1a4a", + "backend/internal/adapter/handler/gate.go": "b662058ebc59010f30d21da12404837aa2329fe5e5590761efdd9140c1d24e9c", + "backend/internal/adapter/handler/gate_test.go": "f7938bdeb5617ed3611303c54818000de9861dde5915fa2c6fa0f74e1fa33268", + "backend/internal/adapter/handler/handler.go": "dc5a9abe76b57c3514c67feb7b98654bc058d3329c35724aef0b6a50476d0ebf", + "backend/internal/adapter/handler/json.go": "d6458b8e8e4426816ec8c855ff6c8d88a151161bc1560e4b98df0ea4b0dbf6be", + "backend/internal/adapter/handler/message_json_test.go": "8dd976c4cad81ee531c76165787b2c8b6052057d934ae4b09b60d8a261cf30e5", + "backend/internal/adapter/handler/middleware.go": "256221524d19e057ab2a330e08f1c5b0bd5fc95b8d19391f69fd286e5aa8ba54", + "backend/internal/adapter/handler/stats.go": "5cd375e5a0db8cc3596293d323e635b86fef22d8ec680f310ab5a460f8176f64", + "backend/internal/adapter/handler/utf8_test.go": "eac6a14b5f3b0ea9aa5329337afc6db27e17f5c17999874da9ce3592dc7a4205", + "backend/internal/adapter/imagefile/image.go": "8a9b399b22573992fd7fd4c87096f953b46c1bc7cad4e09c023923919d755375", + "backend/internal/adapter/imagefile/image_test.go": "a2dc0d3797a5c2ac7ad46a7453a34ea13aba94c3385d2692032705610026dc71", + "backend/internal/adapter/imagefile/stress_test.go": "3d7b0671f57ce5e01fa7e175a86c16a51d56d8f11ec6d3bfb09b2c8bfe8c04d8", + "backend/internal/adapter/imagefile/variants.go": "50762d1e2af635c9a9344d2721ee145bb9d9d43408f680fc280cdd493aeef886", + "backend/internal/adapter/imagefile/variants_integration_test.go": "d25e3371432128e58073ff61e1b87fa741bc32ee7d9bc6e8cb40ae3f3a5b4434", + "backend/internal/adapter/imagefile/variants_test.go": "79c24a3d5c0cb339a32fdea28eb56689703f35ad3c668c464ea7a4f5e82322e5", + "backend/internal/adapter/password/argon.go": "94643bf4b61ca390f8dd56a241e02e1325a994fc40887ffa0e2b8f8e5aebb4c7", + "backend/internal/adapter/queue/streams.go": "4b9b8def20b61d4dec82afc32b540cc7e712cf4e25a6c7f4685747be7a1f5016", + "backend/internal/adapter/repository/postgres/attachments.go": "88f8b8744d95407421eafc111ecc83bcb4b145f1683618d69cac19a8d1c39500", + "backend/internal/adapter/repository/postgres/pipeline.go": "9a367d7a50a9418e96851c253f61ace4a54753044e7fa672c73a3b4526a22245", + "backend/internal/adapter/repository/postgres/pipeline_integration_test.go": "58274475a02f1098a0f79a65a9e8d7608d05dfa26213f90c706a9239fabb586c", + "backend/internal/adapter/repository/postgres/queries/attachments.sql.go": "ccbdaac42ee599d318eae05bb6e0b3580e43c7437ce2cbc9268aa27068460213", + "backend/internal/adapter/repository/postgres/queries/core.sql.go": "9a0e082263cbea39535de9c7c72c41a39a551efc9ee57fdd7b08856831d074b3", + "backend/internal/adapter/repository/postgres/queries/db.go": "4863732830879d4162be215ff245375b953359de23bcd821ed7abc5c32a36c0f", + "backend/internal/adapter/repository/postgres/queries/idempotency.sql.go": "cf6fbc316e93339e0f938c7ca82007e29c3e32f1a21ee1433a2ce8784697f002", + "backend/internal/adapter/repository/postgres/queries/models.go": "26e72c3e908bd66a2a09baabc567d9eb6f88ac9662f6f1eebe8e77b609aa737b", + "backend/internal/adapter/repository/postgres/queries/pipeline.sql.go": "82c3e339f3aae0b9e12f05eb299e615d6f293bbb88fb8139b36a20de5fe6b395", + "backend/internal/adapter/repository/postgres/repository.go": "e380d7c0c52a2447653db9e4d13db432f4c1330e3205a0396510d957441bb978", + "backend/internal/adapter/storage/bootstrap.go": "44c8f422826b29f85a918029bb71a62cd3018190f8efe889cbeee63d19d7039e", + "backend/internal/adapter/storage/readiness_test.go": "a6c79cba7af74ce67e69678085e5bb267a47f9dd7be57daf003ba03951274a5c", + "backend/internal/adapter/storage/s3.go": "99a717604c71da5c181bef74612a603abe3cfc913ccb3da84ed581a57a11ccd8", + "backend/internal/application/usecase/attachments.go": "9e3e69bad8c988f1c931357336c2b0bfe1b0b2ac17fe6f299c1d9cc66dbd9230", + "backend/internal/application/usecase/auth.go": "93802954db2c8eda3782acc71bca817947fd556986d3f6552c78c4b346d988ca", + "backend/internal/application/usecase/messenger.go": "c58aba0d3f5d0ab92fab5d2771b90a3af5ad06e6afe0c7cc46f481a429e89f48", + "backend/internal/application/usecase/pipeline.go": "4f60f7b96bb39e8b5c13597da953783194e9e2c5f8a4030483628a80496d86e0", + "backend/internal/application/usecase/pipeline_retry_test.go": "6068c07fdaa61fdb0bb0b73bd2094aacdec7a10ff2db1ebd64decb3288f2de5c", + "backend/internal/application/usecase/reconcile.go": "239edee16f767dd16b116791900994540a8f99ae3403fd0308f5e2c0797a892d", + "backend/internal/application/usecase/reconcile_pipeline_test.go": "75e1fb5fcc5ffb2188a2368688e5505a4bf54453c2800b5e46c8e9e23de89b7c", + "backend/internal/application/usecase/reconcile_test.go": "fdbcfad86851fc97a3cf635b6d884c41b43084eedc42b3c4847790c5eaad4a6f", + "backend/internal/config/config.go": "58795b18980c4dd938c73721ab937d06410af94edc156b7a7aa9b2c4c2388238", + "backend/internal/config/config_test.go": "390e3b5a4682719a0b25a8eb9aefeda9a9d1c774d3f8d1f08062c79a58d8859a", + "backend/internal/domain/model/attachment.go": "e532f2c8e8c123d0fda6101bff4b21168527a0a446e53f566f5e5919b972133b", + "backend/internal/domain/model/chat.go": "3f7454f48ef442c277fd2741ce5aeeccf03d590bd2d31b9b89fa39f0ba05de7f", + "backend/internal/domain/model/errors.go": "d043547d742e94d9ef9f0799dced7c43dc2adc44d7d90906e891e5afee8de04d", + "backend/internal/domain/model/job.go": "21a2b424b9a72fe098f9a70f34b6137b3e3fcd0129b10ef81071fac11a7c737c", + "backend/internal/domain/model/message.go": "0e0126663d2de04c9d14c8c7427f42a9fb733e3b170ceee7ed86c2138f51b23c", + "backend/internal/domain/model/model_test.go": "77bb8767ac1352bd3551e724eaaeaad4fe93eae248cc8c3008ee8b1c08adddca", + "backend/internal/domain/model/page.go": "cd6a07579bff01aa0d8a11b34fd000a804142b321cef302d386bdb95e9c7f953", + "backend/internal/domain/model/postgres_text_test.go": "8c245eef99a97261daf0aa16fec9edd7bcd3253ec4144c50b4f8e56e45e2186e", + "backend/internal/domain/model/text.go": "c9410cacd9d1b2fd7f3f2cb7e4112e637ecc494f6e4b6da649f936ce0c9a0457", + "backend/internal/domain/model/user.go": "c20c8b8d8a9c0327a3a24afd74f3051d78caed1d2f0851baf340ef61ac502a92", + "backend/internal/domain/port/auth.go": "465c1a1f19075b6f8301eef8dbd7fd9a8d20255716d72c97e946db04c2850dda", + "backend/internal/domain/port/objects.go": "4d22b5ef5ff6fa13404884c160fe52bd7cef4099f84b29268137c27034a0f3ec", + "backend/internal/domain/port/pipeline.go": "5fe7c6909cbaf6a21a4be5d9ed0c2b1551681894d0054463cdba13ea09c21aac", + "backend/internal/domain/repository/repository.go": "baff51af1cf9c6b273b013f6cb8aa3069eaab615e279dd8fe0a2d19f9bb205a6", + "backend/pkg/logger/logger.go": "d839f31015eee692065c31d2538a4cb4c841b0d89e690f0ab8baf80168510556", + "backend/pkg/server/server.go": "3ce957e4e45a94453610217007a9dbf0e8b9723346ff9cc8a3e5aa8ff3676527", + "compose.example.yaml": "2ae5ec12669a2b4281cddc94a2d1725de1993ce093aea32c290fa026749163cc", + "compose.yaml": "bfd9a9be404e90b0a63bc7be89e0def5fba27fa21d0eb7475ab93e3f2a2e26dd", + "config/haproxy.cfg": "5fbd83685515fa68b463fd887b66f23cd2ffab9283c9ece6d015cc5634c7f5bd", + "contracts/README.md": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f", + "contracts/attachment-states.md": "cf0519e137410e4b34ab632aadb8ad8a381d4b73033e27b354c58eeacbdf86d6", + "contracts/examples.json": "381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37", + "contracts/infrastructure.json": "6d4ca861b160118cef8764ae0ca292ac924b541963ccc7be4857eddbef6d77d8", + "contracts/jobs.schema.json": "9ccf9b11466c0c3a26a941520ac5ff543241fc4935e3a6eebb01ef303779b10d", + "contracts/openapi.json": "62b6e1d43ca0c6115e9c6ad0ffa9be0835e8fdf9ae664d2fea3c258ef745d56d", + "evidence/README.md": "78e4bfd738d034597d15d8846accd8a8a5dc738a23a3d15f243f3992bd530cc3", + "evidence/both-replicas.json": "66ac26922382548de136603cb21b160c0b6c0edf20de9de11204ff418ab1c217", + "evidence/both-replicas.txt": "aa2b09b91bf119b29e5989d0d2e3704177524d6293e03c168c069ca84a46cf3e", + "evidence/final-audit-build.txt": "de5843141ad158447f7a86a229b2e96df603497dd1b609dd37dd7718ee17666d", + "evidence/final-checks.txt": "c24bc53e2383435c4d364767d585ace553e75d968793c3a88041cf82b2d0f5f4", + "evidence/final-runtime.txt": "07bca91c910f4077eb709ab00ed233c86102be319c4cf98f79076dbb434a5edc", + "evidence/fresh-db-messenger_stage_70b6ccce.json": "10fdd7032a24d092c8847c8a1499caaef467cd8685e6a3b3ac7b11d9a24ff259", + "evidence/golangci-lint.txt": "ad482a92858eb599d691c55516cb81a5ae9214c855111c7aa2b15d1a77ed2aaf", + "evidence/green-replay-version2.txt": "abeb344088114195f241f2c823a5c32eb657c95176056499dc661bcb54370328", + "evidence/openapi-validate.txt": "422aec45ba93e5567e0de199fa32116a48c970ca3b82738b40f0ed7681ef621f", + "evidence/public-acceptance.txt": "d7a0e35702acb28bff81e3747c6c51f139326b9807c910c209870a59b147103d", + "evidence/red-completed-cleanup.txt": "cf9c45b46d8ce8b09f27a17801368c87713a0947185863f2a1e70bc1bde68c5f", + "evidence/red-fit.txt": "5ad35817fe12fb2c96c579a37c9d8dfd87e5804446178f855f5e95c41035e2be", + "evidence/red-reconciler.txt": "35de766d6a44714a797cad771814c2f159fa6b4536cb69f9fcf2278066b8a59a", + "evidence/red-replay-audit.txt": "0ea3e2607931f6ccdf236ae057465dae9e1bc87eefdab90dd755a2aeefeaa036", + "evidence/red-replay-version2.txt": "b01671551be3e97c7647105da4351b984df4b7cadeb0bdbe1c23f0b1bfb0ba4c", + "evidence/red-transform-infrastructure.txt": "ecd4e6c93ffce849db7fe6b0fd1b7be89da8b7e9c8602cc2bacd31347c0edb98", + "evidence/replay-version2-command.md": "59aab41601f4c11fefaf89049a434554cd11740216981eb6b05977187609b867", + "evidence/runtime-commit-publish-gap.txt": "5d7abc96a82c2ad0d43b91f5492f7882b35ae659ecbb67c952526b98a76c9b63", + "evidence/runtime-pipeline-faults-after_commit.json": "1029f5c2c7fcbbf8d9798934c64491e78c14d2800fbf989ab9dceacb0c8b7106", + "evidence/runtime-pipeline-faults-after_objects.json": "a119e3950320a73f3d83bcb112f61f81543848132289577cac00bb0fb6abf954", + "evidence/runtime-pipeline-faults-after_original-restart-gap.json": "c3c5ab9ac827aecbcd7a435cd702c50419f1dfabdc4cc4aec059846e0899b5a4", + "evidence/runtime-pipeline-faults.json": "c426438148b854a4e0c2f701def53e833c8eea9b588b164a5288033c9aff51a7", + "evidence/runtime-reconciler-red.json": "0a68fc6c9c5a915c055f6775374eb1deeaba1ebb76e28062aac8ecf7a5d9a785", + "evidence/validate-dependencies.txt": "e2e9b0fdaef59a6becf3c2b4476e264fe1b774ac74fe3cf91474044d52d86b2b", + "evidence/versioned-fault-matrix.txt": "97b8aee040692ecbf0ce2b67e3fe812af8dfe4c6ad782604111b568efa24a761", + "examples/requests.http": "8c086fd540092b3e31a945a416632bcda6c4bea39253f86b076f791b362cf5e4", + "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", + "lab.json": "69c44ec7761e238c6a2d25f121a33328f2bbf7592a9348d85f84312d782ed70c", + "requirements-dev.txt": "4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525", + "scripts/bootstrap.ts": "eec21c7e9d66fce3003d2697dc2ff1394c8011e3d6c0a3312fbf265d2e2a754e", + "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", + "scripts/compose.sh": "a2f7328c38f19c7e58dbadef88b087c5d21c17fb520a8a02e90f4da67b80df87", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "dfbdc591fcfe6b7262207fea240b15341ea6c7dcc4be40388e7e33dabfc9918f", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "8f236a157a4698362e66b5c73ca0150eefe3b33cf8b49e0200bbe8a3cb2efaa6", + "scripts/up.sh": "98af809567105f5a3321ba1ff2b105798e7166cf72834a88ee33dd84b198a8c7", + "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", + "student-tests/both-replicas.ts": "4864ccb73f3522fe50967521f2464a94cfc8f9e3ba2da1f837f79b8b47a9c8cd", + "student-tests/pipeline-faults.ts": "76073314ea6a7cab1c7ae7a60d57e6f26f10ccf91f68be88fc31265e285da9ef", + "student-tests/reconciler-red.ts": "55a618dd647ef96f3932e9d086cd0de4baf3b14a3b9571962fcf899ecf2dec95", + "student-tests/run-db-matrix.sh": "bee4d38300d5af9b266877883170d19f43412a54253d51afe1af70b0bb04f8fd", + "student-tests/wait.ts": "9dff91ee5a9876d5c4758d66a65b3b42c29a6d3071c7e510babae561a84cf35b", + "tests/README.md": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5", + "tests/client.py": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d", + "tests/fixtures/README.md": "c001783760b4f87a7a8441a7ba55f5d080ff48e1397d3581d72a88d7691f96c2", + "tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "tests/image_probe.py": "a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13", + "tests/load.py": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933", + "tests/resilience.py": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b", + "tests/smoke.py": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db" + } +} diff --git a/SUBMISSION.md b/SUBMISSION.md new file mode 100644 index 00000000..fcca118e --- /dev/null +++ b/SUBMISSION.md @@ -0,0 +1,5 @@ +# ЛР5 — БПИ2303 — Медведев — цель5, трек2 + +Асинхронный pipeline сохраняет staging и transactional outbox до202, обрабатывается отдельным durable Valkey worker и публикует original+thumbnail/crop/WebP через lease/version fence. Повторы, исчерпание retry, явный replay с audit и независимый staging cleanup сохраняют ready при redelivery и падении после COMMIT. + +Проверены полный OpenAPI, race/vet/lint/build, публичный smoke, полный restart с существующей сессией и файлами, отказ обеих API реплик, реальные worker crashes, COMMIT/publish gap и настоящие PostgreSQL/S3 regressions дублей/reorder/version/replay. Команды и пределы в REPORT.md, фактические receipts в evidence/, точные файлы в SUBMISSION-MANIFEST.json. Это подготовленный результат для сдачи; PR и преподавательская оценка ещё отсутствуют. diff --git a/backend/.golangci.yaml b/backend/.golangci.yaml new file mode 100644 index 00000000..8610ae17 --- /dev/null +++ b/backend/.golangci.yaml @@ -0,0 +1,36 @@ +version: "2" +run: + allow-parallel-runners: true +linters: + default: standard + enable: [depguard] + settings: + errcheck: + # Close тела ответа и файла, открытого на чтение: ошибке некуда деться. + exclude-functions: + - (io.ReadCloser).Close + - (*os.File).Close + depguard: + rules: + # Гексагон: ядро и use-case'ы зависят от портов, а не от адаптеров и транспорта. + inner-pure: + files: ["**/internal/domain/**", "**/internal/application/**"] + deny: + - pkg: backend/internal/adapter + desc: домен и приложение зависят от портов, не от адаптеров + - pkg: backend/internal/config + desc: конфиг читает только cmd + - pkg: net/http + desc: HTTP только в adapter/handler + - pkg: github.com/go-chi + desc: роутер только в adapter/handler + - pkg: github.com/rs/zerolog + desc: логирование на границе, в адаптерах + domain-core: + files: ["**/internal/domain/**"] + deny: + - pkg: backend/internal/application + desc: домен не знает о use-case'ах + exclusions: + paths: + - ".*\\.gen\\.go$" diff --git a/backend/cmd/app/main.go b/backend/cmd/app/main.go new file mode 100644 index 00000000..5db6e838 --- /dev/null +++ b/backend/cmd/app/main.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "backend/db" + "backend/internal/adapter/handler" + "backend/internal/adapter/password" + "backend/internal/adapter/repository/postgres" + "backend/internal/adapter/storage" + "backend/internal/application/usecase" + "backend/internal/config" + "backend/pkg/logger" + "backend/pkg/server" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +func main() { + healthcheck := flag.Bool("healthcheck", false, "проверить /health/ready и выйти (для Docker HEALTHCHECK)") + migrate := flag.Bool("migrate", false, "apply migrations and exit") + bootstrapS3 := flag.Bool("bootstrap-s3", false, "initialize private bucket and application IAM credentials") + reconcile := flag.Bool("reconcile", false, "reconcile at most 100 metadata rows and 100 objects; dry-run by default") + apply := flag.Bool("apply", false, "apply safe reconciliation changes") + grace := flag.Duration("grace", 10*time.Minute, "minimum safe object age, at least 60 seconds") + after := flag.String("after", "", "metadata UUID cursor") + objectAfter := flag.String("object-after", "", "object key cursor") + flag.Parse() + + cfg, err := config.Load() + if err != nil { + fmt.Fprintln(os.Stderr, "config:", err) + os.Exit(1) + } + if *healthcheck { + os.Exit(probe(cfg.Port)) + } + + log := logger.New(cfg.LogLevel) + dsn, err := config.ReadSecret(cfg.DatabaseURLFile) + if err != nil { + log.Fatal().Msg("database secret unavailable") + } + if *migrate { + if err = db.Migrate(context.Background(), dsn); err != nil { + log.Fatal().Err(err).Msg("migration failed") + } + return + } + s3path := os.Getenv("S3_CONFIG_FILE") + if s3path == "" { + s3path = "/run/secrets/s3_credentials" + } + s3config, err := storage.ReadConfig(s3path) + if err != nil { + log.Fatal().Msg("S3 application credentials unavailable") + } + objects, err := storage.FromConfig(s3config) + if err != nil { + log.Fatal().Msg("invalid S3 endpoint") + } + if *bootstrapS3 { + root, e := storage.ReadConfig("/run/secrets/s3_root") + if e != nil { + log.Fatal().Msg("S3 bootstrap credentials unavailable") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if e = storage.Bootstrap(ctx, root, s3config); e != nil { + log.Fatal().Err(e).Msg("S3 bootstrap failed") + } + return + } + pc, err := pgxpool.ParseConfig(dsn) + if err != nil { + log.Fatal().Msg("invalid database configuration") + } + pc.MaxConns = cfg.MaxConns + pc.MinConns = 0 + pc.ConnConfig.ConnectTimeout = time.Second + pc.ConnConfig.RuntimeParams["statement_timeout"] = "2000" + pc.ConnConfig.RuntimeParams["lock_timeout"] = "1000" + pool, err := pgxpool.NewWithConfig(context.Background(), pc) + if err != nil { + log.Fatal().Msg("database initialization failed") + } + defer pool.Close() + var cache *redis.Client + if url, e := config.ReadSecret(cfg.CacheURLFile); e == nil { + opts, e := redis.ParseURL(url) + if e != nil { + log.Fatal().Msg("invalid cache configuration") + } + opts.MaxRetries = 0 + opts.DialTimeout = 80 * time.Millisecond + opts.ReadTimeout = 80 * time.Millisecond + opts.WriteTimeout = 80 * time.Millisecond + cache = redis.NewClient(opts) + defer func() { _ = cache.Close() }() + } + store := postgres.New(pool, cache) + if *reconcile { + cursor := uuid.Nil + if *after != "" { + cursor, err = uuid.Parse(*after) + if err != nil { + log.Fatal().Msg("invalid metadata cursor") + } + } + if len(*objectAfter) > 64 { + log.Fatal().Msg("object cursor too long") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + report, e := usecase.NewReconciler(store, objects, time.Now).Run(ctx, cursor, *objectAfter, *grace, *apply) + if err = json.NewEncoder(os.Stdout).Encode(report); err != nil { + log.Fatal().Msg("reconciliation report write failed") + } + if e != nil { + log.Fatal().Err(e).Msg("reconciliation incomplete") + } + return + } + app := usecase.NewMessenger(store) + auth := usecase.NewAuth(store, password.New(), app, time.Duration(cfg.SessionTTLSeconds)*time.Second, time.Now) + attachments := usecase.NewAttachments(store, app, objects) + probeDependencies := func(ctx context.Context) error { + if e := pool.Ping(ctx); e != nil { + return e + } + return objects.Ready(ctx) + } + h := handler.New(app, auth, attachments, probeDependencies, handler.Limits{ + MaxBodyBytes: cfg.MaxBodyBytes, + MaxInflight: cfg.MaxInflight, + MaxQueue: cfg.MaxQueue, + QueueTimeout: cfg.QueueTimeout, + }, func() any { + return map[string]any{"pool_total": pool.Stat().TotalConns(), "pool_acquired": pool.Stat().AcquiredConns(), "pool_max": pool.Stat().MaxConns()} + }, &log) + srv := server.New(cfg.Addr(), h.Router()) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + serveErr := make(chan error, 1) + go func() { serveErr <- srv.ListenAndServe() }() + log.Info().Str("addr", cfg.Addr()).Str("env", cfg.Env). + Int("max_inflight", cfg.MaxInflight). + Int("max_queue", cfg.MaxQueue).Msg("server started") + + select { + case err := <-serveErr: + log.Fatal().Err(err).Msg("server failed") + case <-ctx.Done(): + } + + started := time.Now() + log.Info().Dur("timeout", cfg.ShutdownTimeout).Msg("shutdown signal received, draining") + h.Drain() + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Error().Err(err).Msg("shutdown timed out, open requests aborted") + } + if err := <-serveErr; err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error().Err(err).Msg("server error") + } + log.Info().Dur("took", time.Since(started)).Msg("shutdown complete") +} + +// probe — healthcheck без curl в образе distroless: сам бинарник ходит в +// /health/ready и возвращает код выхода для Docker. +func probe(port int) int { + client := http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/health/ready", port)) + if err != nil { + return 1 + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 1 + } + return 0 +} diff --git a/backend/cmd/worker/main.go b/backend/cmd/worker/main.go new file mode 100644 index 00000000..c8c3c860 --- /dev/null +++ b/backend/cmd/worker/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "backend/internal/adapter/imagefile" + "backend/internal/adapter/queue" + "backend/internal/adapter/repository/postgres" + "backend/internal/adapter/storage" + "backend/internal/application/usecase" + "backend/internal/config" + "context" + "flag" + "fmt" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "os" + "os/signal" + "strconv" + "syscall" + "time" +) + +func main() { + replay := flag.String("replay", "", "replay failed attachment UUID") + advance := flag.String("advance", "", "new version for ready/failed attachment UUID") + fault := flag.String("fault", "", "test-only crash point: after_original/after_objects/after_commit") + flag.Parse() + cfg, e := config.Load() + must(e) + dsn, e := config.ReadSecret(cfg.DatabaseURLFile) + must(e) + pc, e := pgxpool.ParseConfig(dsn) + must(e) + pc.MaxConns = 2 + pc.ConnConfig.ConnectTimeout = time.Second + pc.ConnConfig.RuntimeParams["statement_timeout"] = "2000" + pc.ConnConfig.RuntimeParams["lock_timeout"] = "1000" + pool, e := pgxpool.NewWithConfig(context.Background(), pc) + must(e) + defer pool.Close() + repo := postgres.New(pool, nil) + if *replay != "" || *advance != "" { + s := *replay + if s == "" { + s = *advance + } + id, e := uuid.Parse(s) + must(e) + var ok bool + if *replay != "" { + ok, e = repo.ReplayJob(context.Background(), id) + } else { + ok, e = repo.AdvancePipeline(context.Background(), id) + } + must(e) + fmt.Printf("{\"changed\":%v}\n", ok) + return + } + s3path := os.Getenv("S3_CONFIG_FILE") + if s3path == "" { + s3path = "/run/secrets/s3_credentials" + } + s3cfg, e := storage.ReadConfig(s3path) + must(e) + objects, e := storage.FromConfig(s3cfg) + must(e) + url, e := config.ReadSecret(cfg.CacheURLFile) + must(e) + opts, e := redis.ParseURL(url) + must(e) + opts.MaxRetries = 0 + opts.DialTimeout = time.Second + opts.ReadTimeout = 2 * time.Second + opts.WriteTimeout = time.Second + client := redis.NewClient(opts) + defer func() { _ = client.Close() }() + attempts := 5 + if s := os.Getenv("WORKER_MAX_ATTEMPTS"); s != "" { + attempts, e = strconv.Atoi(s) + must(e) + } + if attempts < 1 || attempts > 10 { + must(fmt.Errorf("attempts out of bounds")) + } + pipeline := usecase.NewPipeline(repo, objects, imagefile.NewTransformer(), int32(attempts)) + if *fault != "" { + switch *fault { + case "after_original", "after_objects", "after_commit": + default: + must(fmt.Errorf("invalid fault point")) + } + pipeline.Fault = func(point string) error { + if point == *fault { + os.Exit(88) + } + return nil + } + } + broker := queue.New(client, repo, uuid.NewString()) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + logError := func(error) { fmt.Fprintln(os.Stderr, "{\"event\":\"pipeline_dependency_error\"}") } + cleanupDone := make(chan struct{}) + go func() { + defer close(cleanupDone) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cleanup, cancel := context.WithTimeout(ctx, 5*time.Second) + e := pipeline.Cleanup(cleanup) + cancel() + if e != nil { + logError(e) + } + } + } + }() + done := make(chan struct{}) + go func() { defer close(done); broker.RunPublisher(ctx, logError) }() + broker.RunWorker(ctx, pipeline.Process, logError) + stop() + <-done + <-cleanupDone +} +func must(e error) { + if e != nil { + fmt.Fprintln(os.Stderr, "worker initialization/operation failed") + os.Exit(1) + } +} diff --git a/backend/codegen.yaml b/backend/codegen.yaml new file mode 100644 index 00000000..563b4f30 --- /dev/null +++ b/backend/codegen.yaml @@ -0,0 +1,15 @@ +package: handler +generate: + models: true + chi-server: true + strict-server: true +output: api.gen.go +output-options: + overlay: + path: ../../../generator-overlay.yaml + # date-time -> handler.Timestamp: ровно 6 цифр дробной части (см. timestamp.go) + type-mapping: + string: + formats: + date-time: + type: Timestamp diff --git a/backend/db/migrate.go b/backend/db/migrate.go new file mode 100644 index 00000000..a228082b --- /dev/null +++ b/backend/db/migrate.go @@ -0,0 +1,25 @@ +package db + +import ( + "context" + "database/sql" + "embed" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +func Migrate(ctx context.Context, dsn string) error { + conn, err := sql.Open("pgx", dsn) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + goose.SetBaseFS(migrations) + if err = goose.SetDialect("postgres"); err != nil { + return err + } + return goose.UpContext(ctx, conn, "migrations") +} diff --git a/backend/db/migrations/00001_core.sql b/backend/db/migrations/00001_core.sql new file mode 100644 index 00000000..a09cb7a9 --- /dev/null +++ b/backend/db/migrations/00001_core.sql @@ -0,0 +1,45 @@ +-- +goose Up +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + chat_count integer NOT NULL DEFAULT 0 CHECK (chat_count BETWEEN 0 AND 1000), + message_count integer NOT NULL DEFAULT 0 CHECK (message_count BETWEEN 0 AND 1000000), + attachment_count integer NOT NULL DEFAULT 0 CHECK (attachment_count BETWEEN 0 AND 10000) +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX member_chats ON chat_members(user_id,chat_id); +CREATE INDEX chats_order ON chats(created_at,id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 0 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX messages_history ON messages(chat_id,created_at,id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY CHECK (length(token_hash)=64), + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_user ON sessions(user_id,expires_at); + +-- +goose Down +DROP TABLE sessions; +DROP TABLE messages; +DROP TABLE chat_members; +DROP TABLE chats; +DROP TABLE users; diff --git a/backend/db/migrations/00002_idempotency.sql b/backend/db/migrations/00002_idempotency.sql new file mode 100644 index 00000000..81c5db25 --- /dev/null +++ b/backend/db/migrations/00002_idempotency.sql @@ -0,0 +1,12 @@ +-- +goose Up +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + payload_hash text NOT NULL CHECK (length(payload_hash)=64), + message_id uuid NOT NULL REFERENCES messages(id) DEFERRABLE INITIALLY DEFERRED, + expires_at timestamptz NOT NULL, + PRIMARY KEY(user_id,chat_id,key) +); +-- +goose Down +DROP TABLE idempotency; diff --git a/backend/db/migrations/00003_attachments.sql b/backend/db/migrations/00003_attachments.sql new file mode 100644 index 00000000..32695683 --- /dev/null +++ b/backend/db/migrations/00003_attachments.sql @@ -0,0 +1,49 @@ +-- +goose Up +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'), + status text NOT NULL CHECK(status IN ('uploading','queued','processing','ready','failed','missing','abandoned')), + pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>0), + error_code text NOT NULL DEFAULT '', + variants jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(variants)='array' AND jsonb_array_length(variants)<=3), + staging_expired boolean NOT NULL DEFAULT false, + staging_cleaned boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY(chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX attachments_owner ON attachments(owner_id,id); +CREATE INDEX attachments_staging_cleanup ON attachments(updated_at,id) WHERE NOT staging_cleaned; +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), + attachment_id uuid NOT NULL REFERENCES attachments(id), + ordinal integer NOT NULL CHECK(ordinal BETWEEN 0 AND 9), + PRIMARY KEY(message_id,ordinal), + UNIQUE(message_id,attachment_id) +); +CREATE INDEX attachment_references ON message_attachments(attachment_id); +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK(pipeline_version>0), + attempt integer NOT NULL DEFAULT 0 CHECK(attempt>=0), + traceparent text NOT NULL DEFAULT '' CHECK(char_length(traceparent)<=55), + tracestate text NOT NULL DEFAULT '' CHECK(char_length(tracestate)<=512), + lease_id uuid NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + lease_until timestamptz NOT NULL DEFAULT 'epoch', + next_attempt timestamptz NOT NULL DEFAULT 'epoch', + published_at timestamptz NOT NULL DEFAULT 'epoch', + completed boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX pending_attachment_jobs ON attachment_jobs(next_attempt,published_at) WHERE NOT completed; +-- +goose Down +DROP TABLE attachment_jobs; +DROP TABLE message_attachments; +DROP TABLE attachments; diff --git a/backend/db/migrations/00005_replay_audit.sql b/backend/db/migrations/00005_replay_audit.sql new file mode 100644 index 00000000..b621b9bf --- /dev/null +++ b/backend/db/migrations/00005_replay_audit.sql @@ -0,0 +1,4 @@ +-- +goose Up +ALTER TABLE attachment_jobs ADD COLUMN replay_count bigint NOT NULL DEFAULT 0 CHECK(replay_count>=0), ADD COLUMN replayed_at timestamptz NOT NULL DEFAULT 'epoch'; +-- +goose Down +ALTER TABLE attachment_jobs DROP COLUMN replay_count, DROP COLUMN replayed_at; diff --git a/backend/db/sqlc/queries/attachments.sql b/backend/db/sqlc/queries/attachments.sql new file mode 100644 index 00000000..a9224006 --- /dev/null +++ b/backend/db/sqlc/queries/attachments.sql @@ -0,0 +1,15 @@ +-- name: ChargeAttachment :one +UPDATE users SET attachment_count=attachment_count+1 WHERE id=$1 AND attachment_count<10000 RETURNING id; +-- name: InsertAttachment :one +INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status) +VALUES($1,$2,$3,$4,$5,$6,$7,'uploading') RETURNING *; +-- name: GetAttachment :one +SELECT * FROM attachments WHERE id=$1 FOR SHARE; +-- name: SetAttachmentStatus :execrows +UPDATE attachments SET status=$2,updated_at=clock_timestamp() WHERE id=$1 AND status=$3 AND pipeline_version=0; +-- name: InsertMessageAttachment :exec +INSERT INTO message_attachments(message_id,attachment_id,ordinal) VALUES($1,$2,$3); +-- name: MessageAttachments :many +SELECT attachment_id FROM message_attachments WHERE message_id=$1 ORDER BY ordinal LIMIT 10; +-- name: ReconcileAttachments :many +SELECT * FROM attachments WHERE id>$1 ORDER BY id LIMIT 101; diff --git a/backend/db/sqlc/queries/core.sql b/backend/db/sqlc/queries/core.sql new file mode 100644 index 00000000..d34bc21f --- /dev/null +++ b/backend/db/sqlc/queries/core.sql @@ -0,0 +1,42 @@ +-- name: CreateUser :one +INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) +RETURNING id,username,display_name,created_at; +-- name: GetUser :one +SELECT id,username,display_name,created_at FROM users WHERE id=$1; +-- name: Credentials :one +SELECT id,password_hash FROM users WHERE username=$1; +-- name: ChargeChat :one +UPDATE users SET chat_count=chat_count+1 WHERE id=$1 AND chat_count<1000 RETURNING id; +-- name: ChargeMessage :one +UPDATE users SET message_count=message_count+1 WHERE id=$1 AND message_count<1000000 RETURNING id; +-- name: InsertChat :one +INSERT INTO chats(id,title) VALUES($1,$2) RETURNING id,title,created_at; +-- name: InsertMembers :exec +INSERT INTO chat_members(chat_id,user_id) SELECT $1,unnest($2::uuid[]); +-- name: GetChat :one +SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members +FROM chats c WHERE c.id=$1; +-- name: ListChats :many +SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members +FROM chats c JOIN chat_members m ON m.chat_id=c.id +WHERE m.user_id=sqlc.arg(member_id) AND (c.created_at,c.id)>(sqlc.arg(after_time)::timestamptz,sqlc.arg(after_id)::uuid) +ORDER BY c.created_at,c.id LIMIT sqlc.arg(page_limit); +-- name: InsertMessage :one +INSERT INTO messages(id,chat_id,sender_id,text) VALUES($1,$2,$3,$4) RETURNING id,chat_id,sender_id,text,created_at; +-- name: ListMessages :many +SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at, +ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids +FROM messages m WHERE m.chat_id=$1 AND (m.created_at,m.id)>(sqlc.arg(after_time)::timestamptz,sqlc.arg(after_id)::uuid) +ORDER BY m.created_at,m.id LIMIT sqlc.arg(page_limit); +-- name: LockSessionOwner :one +SELECT id FROM users WHERE id=$1 FOR UPDATE; +-- name: DeleteExpiredSessions :exec +DELETE FROM sessions WHERE user_id=$1 AND expires_at<=clock_timestamp(); +-- name: InsertSession :one +INSERT INTO sessions(token_hash,user_id,expires_at) +SELECT $1,$2,$3 WHERE (SELECT count(*) FROM sessions WHERE user_id=$2)<100 +RETURNING user_id; +-- name: AuthenticateSession :one +SELECT user_id FROM sessions WHERE token_hash=$1 AND expires_at>clock_timestamp(); +-- name: RevokeSession :exec +DELETE FROM sessions WHERE token_hash=$1; diff --git a/backend/db/sqlc/queries/idempotency.sql b/backend/db/sqlc/queries/idempotency.sql new file mode 100644 index 00000000..82be769e --- /dev/null +++ b/backend/db/sqlc/queries/idempotency.sql @@ -0,0 +1,12 @@ +-- name: ReserveKey :one +INSERT INTO idempotency(user_id,chat_id,key,payload_hash,message_id,expires_at) +VALUES($1,$2,$3,$4,$5,clock_timestamp()+interval '24 hours') +ON CONFLICT(user_id,chat_id,key) DO UPDATE SET payload_hash=excluded.payload_hash, +message_id=excluded.message_id,expires_at=excluded.expires_at +WHERE idempotency.expires_at<=clock_timestamp() +RETURNING message_id; +-- name: ReplayMessage :one +SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,i.payload_hash, +ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids +FROM idempotency i JOIN messages m ON m.id=i.message_id +WHERE i.user_id=$1 AND i.chat_id=$2 AND i.key=$3; diff --git a/backend/db/sqlc/queries/pipeline.sql b/backend/db/sqlc/queries/pipeline.sql new file mode 100644 index 00000000..aa56ade8 --- /dev/null +++ b/backend/db/sqlc/queries/pipeline.sql @@ -0,0 +1,50 @@ +-- name: InsertJob :exec +INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,traceparent,tracestate) +VALUES(sqlc.arg(job_id),sqlc.arg(attachment_id),sqlc.arg(pipeline_version),sqlc.arg(traceparent),sqlc.arg(tracestate)); +-- name: QueueAttachment :one +UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=$1 AND status='uploading' AND NOT staging_expired RETURNING *; +-- name: PendingJobs :many +SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version +WHERE NOT j.completed AND a.status IN ('queued','processing') AND j.next_attempt<=clock_timestamp() + AND j.lease_untilclock_timestamp() AND NOT j.completed AND a.status='processing'; +-- name: CompleteJob :exec +UPDATE attachment_jobs SET completed=true,lease_until='epoch' WHERE id=sqlc.arg(job_id) AND lease_id=sqlc.arg(lease_id); +-- name: RetryAttachment :execrows +UPDATE attachments a SET status=sqlc.arg(next_status),error_code=sqlc.arg(error_code),updated_at=clock_timestamp() +FROM attachment_jobs j WHERE j.id=sqlc.arg(job_id) AND j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version + AND j.lease_id=sqlc.arg(lease_id) AND j.lease_until>clock_timestamp() AND NOT j.completed AND a.status='processing'; +-- name: RetryJob :exec +UPDATE attachment_jobs SET lease_until='epoch',next_attempt=clock_timestamp()+sqlc.arg(delay_ms)::int*interval '1 millisecond',completed=sqlc.arg(completed) +WHERE id=sqlc.arg(job_id) AND lease_id=sqlc.arg(lease_id); +-- name: ReplayAttachment :one +UPDATE attachments SET status='queued',error_code='',updated_at=clock_timestamp() WHERE id=$1 AND status='failed' AND NOT staging_expired RETURNING *; +-- name: ReplayJobs :exec +UPDATE attachment_jobs SET replay_count=replay_count+1,replayed_at=clock_timestamp(),completed=false,attempt=0,lease_until='epoch',next_attempt='epoch',published_at='epoch' WHERE attachment_id=$1 AND pipeline_version=$2; +-- name: AdvancePipeline :one +UPDATE attachments SET status='queued',error_code='',variants='[]',pipeline_version=pipeline_version+1,staging_expired=false,updated_at=clock_timestamp() WHERE id=$1 AND (status='ready' OR (status='failed' AND NOT staging_expired)) RETURNING *; +-- name: StagingCleanup :many +WITH candidates AS ( + SELECT id FROM attachments WHERE NOT staging_cleaned AND (status='ready' OR (status IN ('failed','uploading') AND updated_at'ready') FROM candidates c WHERE a.id=c.id AND NOT a.staging_cleaned +AND (a.status='ready' OR (a.status IN ('failed','uploading') AND a.updated_at model.MaxImageBytes { + return nil, errPayloadTooLarge + } + f, err := imagefile.Inspect(r.Body, req.Params.Filename, r.Header.Get("Content-Type")) + if errors.Is(err, imagefile.ErrTooLarge) { + return nil, errPayloadTooLarge + } + if errors.Is(err, imagefile.ErrUnsupported) { + return nil, errUnsupportedImage + } + if err != nil { + return nil, err + } + defer f.Dispose() + a, err := h.attachments.Upload(ctx, caller, req.ChatId, req.Params.Filename, f.ContentType, f.SHA256, f.Size, f.File) + if err != nil { + return nil, err + } + return UploadAttachment202JSONResponse{Body: toAttachment(a)}, nil +} +func (h *API) GetAttachment(ctx context.Context, req GetAttachmentRequestObject) (GetAttachmentResponseObject, error) { + a, e := h.attachments.Get(ctx, currentUser(ctx).ID, req.AttachmentId) + if e != nil { + return nil, e + } + return GetAttachment200JSONResponse{Body: toAttachment(a)}, nil +} + +// Closing in the response visitor keeps the S3 stream alive until serialization completes. +type originalResponse struct { + body io.ReadCloser + size int64 + mime string +} + +func (r originalResponse) VisitDownloadAttachmentResponse(w http.ResponseWriter) error { + defer r.body.Close() + w.Header().Set("Content-Type", r.mime) + w.Header().Set("Content-Length", strconv.FormatInt(r.size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(200) + _, err := io.CopyN(w, r.body, r.size) + if err != nil { + panic(http.ErrAbortHandler) + } + return nil +} +func (h *API) DownloadAttachment(ctx context.Context, req DownloadAttachmentRequestObject) (DownloadAttachmentResponseObject, error) { + if _, e := h.attachments.Get(ctx, currentUser(ctx).ID, req.AttachmentId); e != nil { + return nil, e + } + variant := "original" + if req.Params.Variant != nil { + variant = string(*req.Params.Variant) + } + a, r, e := h.attachments.Content(ctx, currentUser(ctx).ID, req.AttachmentId, variant) + if e != nil { + return nil, e + } + return originalResponse{r, a.SizeBytes, a.ContentType}, nil +} diff --git a/backend/internal/adapter/handler/auth.go b/backend/internal/adapter/handler/auth.go new file mode 100644 index 00000000..b60f1cf8 --- /dev/null +++ b/backend/internal/adapter/handler/auth.go @@ -0,0 +1,47 @@ +package handler + +import ( + "backend/internal/domain/model" + "context" + "net/http" + "strings" + "time" + "unicode/utf8" +) + +type sessionRequestKey struct{} + +func (h *API) CreateSession(ctx context.Context, _ CreateSessionRequestObject) (CreateSessionResponseObject, error) { + r := ctx.Value(sessionRequestKey{}).(*http.Request) + username, password, ok := r.BasicAuth() + if !ok || len(username) > 32 || len(password) > 512 || utf8.RuneCountInString(password) > 128 { + return nil, model.ErrUnauthorized + } + s, err := h.auth.Login(ctx, username, password) + if err != nil { + return nil, err + } + return CreateSession201JSONResponse{Body: Session{Token: s.Token, TokenType: Bearer, ExpiresAt: Timestamp(s.ExpiresAt), User: toUser(s.User)}}, nil +} +func (h *API) RevokeSession(ctx context.Context, _ RevokeSessionRequestObject) (RevokeSessionResponseObject, error) { + r := ctx.Value(sessionRequestKey{}).(*http.Request) + parts := strings.Fields(r.Header.Get("Authorization")) + if err := h.auth.Logout(ctx, parts[1]); err != nil { + return nil, err + } + return RevokeSession204Response{}, nil +} +func (h *API) requestContext(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + duration := 5 * time.Second + if strings.HasSuffix(r.URL.Path, "/attachments") || strings.HasSuffix(r.URL.Path, "/content") { + duration = 20 * time.Second + } + ctx, cancel := context.WithTimeout(r.Context(), duration) + defer cancel() + if r.URL.Path == "/api/v1/auth/sessions" { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } + next.ServeHTTP(w, r.WithContext(context.WithValue(ctx, sessionRequestKey{}, r))) + }) +} diff --git a/backend/internal/adapter/handler/errors.go b/backend/internal/adapter/handler/errors.go new file mode 100644 index 00000000..7a64dc62 --- /dev/null +++ b/backend/internal/adapter/handler/errors.go @@ -0,0 +1,85 @@ +package handler + +import ( + "encoding/json" + "errors" + "net/http" + + "backend/internal/domain/model" +) + +var ( + errRouteNotFound = errors.New("route not found") + errMethodNotAllowed = errors.New("method not allowed") + errPayloadTooLarge = errors.New("payload too large") + errOverloaded = errors.New("overloaded") + errDraining = errors.New("draining") + errUnsupportedImage = errors.New("unsupported image") +) + +type apiError struct { + status int + code ErrorErrorCode + message string +} + +// classify — единственное место, где ошибка превращается в HTTP-статус. +func classify(err error) apiError { + var invalid *model.ValidationError + switch { + case errors.As(err, &invalid): + return apiError{http.StatusBadRequest, InvalidRequest, invalid.Message} + case errors.Is(err, model.ErrUnauthorized): + return apiError{http.StatusUnauthorized, Unauthorized, "нужна действующая сессия"} + case errors.Is(err, model.ErrNotFound), errors.Is(err, errRouteNotFound): + return apiError{http.StatusNotFound, NotFound, "объект не найден"} + case errors.Is(err, errMethodNotAllowed): + return apiError{http.StatusMethodNotAllowed, InvalidRequest, "метод не поддерживается"} + case errors.Is(err, model.ErrUsernameTaken): + return apiError{http.StatusConflict, Conflict, "username уже занят"} + case errors.Is(err, model.ErrConflict): + return apiError{http.StatusConflict, Conflict, "конфликт запроса"} + case errors.Is(err, model.ErrUnavailable): + return apiError{http.StatusServiceUnavailable, Unavailable, "зависимость недоступна"} + case errors.Is(err, errPayloadTooLarge): + return apiError{http.StatusRequestEntityTooLarge, PayloadTooLarge, "тело запроса больше допустимого"} + case errors.Is(err, errUnsupportedImage): + return apiError{http.StatusUnsupportedMediaType, UnsupportedMediaType, "нужен корректный PNG или JPEG с верным Content-Type"} + case errors.Is(err, model.ErrStorageFull): + return apiError{http.StatusServiceUnavailable, Unavailable, "хранилище заполнено, запись невозможна; чтение работает"} + case errors.Is(err, errOverloaded): + return apiError{http.StatusServiceUnavailable, Unavailable, "сервер перегружен, повторите позже"} + case errors.Is(err, errDraining): + return apiError{http.StatusServiceUnavailable, Unavailable, "сервер останавливается"} + default: + return apiError{http.StatusInternalServerError, InternalError, "внутренняя ошибка"} + } +} + +func (h *API) writeError(w http.ResponseWriter, r *http.Request, err error) { + e := classify(err) + if e.status == http.StatusInternalServerError { + h.log.Error().Err(err).Str("request_id", requestIDFrom(r.Context())).Msg("unexpected error") + } + if errors.Is(err, errOverloaded) { + w.Header().Set("Retry-After", "1") + } + var body Error + body.Error.Code = e.code + body.Error.Message = e.message + body.Error.RequestId = requestIDFrom(r.Context()) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(e.status) + _ = json.NewEncoder(w).Encode(body) +} + +// bodyError — JSON синтаксически верен (это проверил jsonBody), но не +// подходит к схеме: лишнее поле или неверный тип. +func (h *API) bodyError(w http.ResponseWriter, r *http.Request, _ error) { + h.writeError(w, r, &model.ValidationError{Message: "тело запроса не соответствует схеме: лишнее поле или неверный тип"}) +} + +// paramError — path или query параметр не разобрался (UUID, целое число). +func (h *API) paramError(w http.ResponseWriter, r *http.Request, _ error) { + h.writeError(w, r, &model.ValidationError{Message: "некорректный параметр запроса"}) +} diff --git a/backend/internal/adapter/handler/gate.go b/backend/internal/adapter/handler/gate.go new file mode 100644 index 00000000..e9675f26 --- /dev/null +++ b/backend/internal/adapter/handler/gate.go @@ -0,0 +1,78 @@ +package handler + +import ( + "net/http" + "sync/atomic" + "time" +) + +// gate ограничивает одновременную обработку запросов API: не больше inflight +// в работе и не больше queue в ожидании слота. Лишние сразу получают 503 с +// Retry-After, ждавшие дольше timeout — тоже 503. Так перегрузка превращается +// в быстрый предсказуемый отказ, а не в рост горутин, памяти и задержки. +type gate struct { + slots chan struct{} // nil — без лимита + maxQueue int64 + timeout time.Duration + waiting atomic.Int64 + rejected atomic.Int64 +} + +func newGate(inflight, queue int, timeout time.Duration) *gate { + g := &gate{maxQueue: int64(queue), timeout: timeout} + if inflight > 0 { + g.slots = make(chan struct{}, inflight) + } + return g +} + +func (g *gate) middleware(reject func(http.ResponseWriter, *http.Request, error)) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if g.slots == nil { + next.ServeHTTP(w, r) + return + } + if !g.acquire(r) { + g.rejected.Add(1) + reject(w, r, errOverloaded) + return + } + defer func() { <-g.slots }() + next.ServeHTTP(w, r) + }) + } +} + +func (g *gate) acquire(r *http.Request) bool { + select { + case g.slots <- struct{}{}: + return true + default: + } + if g.waiting.Add(1) > g.maxQueue { + g.waiting.Add(-1) + return false + } + defer g.waiting.Add(-1) + timer := time.NewTimer(g.timeout) + defer timer.Stop() + select { + case g.slots <- struct{}{}: + return true + case <-timer.C: + return false + case <-r.Context().Done(): + return false + } +} + +type gateStats struct { + Inflight int `json:"inflight"` + Waiting int64 `json:"waiting"` + Rejected int64 `json:"rejected"` +} + +func (g *gate) stats() gateStats { + return gateStats{Inflight: len(g.slots), Waiting: g.waiting.Load(), Rejected: g.rejected.Load()} +} diff --git a/backend/internal/adapter/handler/gate_test.go b/backend/internal/adapter/handler/gate_test.go new file mode 100644 index 00000000..7f988ece --- /dev/null +++ b/backend/internal/adapter/handler/gate_test.go @@ -0,0 +1,102 @@ +package handler + +import ( + "errors" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" +) + +// gatedServer пропускает запросы через gate в обработчик, который держит слот, +// пока тест не закроет release. +func gatedServer(t *testing.T, g *gate) (*httptest.Server, chan struct{}, chan struct{}) { + t.Helper() + entered, release := make(chan struct{}, 10), make(chan struct{}) + reject := func(w http.ResponseWriter, _ *http.Request, err error) { + if !errors.Is(err, errOverloaded) { + t.Errorf("unexpected error %v", err) + } + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusServiceUnavailable) + } + h := g.middleware(reject)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entered <- struct{}{} + <-release + w.WriteHeader(http.StatusOK) + })) + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return srv, entered, release +} + +func get(t *testing.T, url string) int { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Error(err) + return 0 + } + resp.Body.Close() + return resp.StatusCode +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not reached") + } + time.Sleep(time.Millisecond) + } +} + +func TestGateRejectsBeyondQueueAndRecovers(t *testing.T) { + t.Parallel() + g := newGate(1, 1, time.Minute) + srv, entered, release := gatedServer(t, g) + + statuses := make(chan int, 2) + var wg sync.WaitGroup + wg.Go(func() { statuses <- get(t, srv.URL) }) // занимает слот + <-entered + wg.Go(func() { statuses <- get(t, srv.URL) }) // ждёт в очереди + waitFor(t, func() bool { return g.waiting.Load() == 1 }) + + if code := get(t, srv.URL); code != http.StatusServiceUnavailable { + t.Fatalf("third request: %d, want 503", code) + } + close(release) + wg.Wait() + close(statuses) + for code := range statuses { + if code != http.StatusOK { + t.Fatalf("admitted request got %d", code) + } + } + if code := get(t, srv.URL); code != http.StatusOK { + t.Fatalf("after load: %d, want recovery to 200", code) + } + if g.stats().Rejected != 1 { + t.Fatalf("rejected %d", g.stats().Rejected) + } +} + +func TestGateTimesOutQueuedRequest(t *testing.T) { + t.Parallel() + g := newGate(1, 10, 20*time.Millisecond) + srv, entered, release := gatedServer(t, g) + defer close(release) + + go get(t, srv.URL) + <-entered + start := time.Now() + if code := get(t, srv.URL); code != http.StatusServiceUnavailable { + t.Fatalf("queued request: %d, want 503 after timeout", code) + } + if time.Since(start) > time.Second { + t.Fatal("queue timeout not applied") + } +} diff --git a/backend/internal/adapter/handler/handler.go b/backend/internal/adapter/handler/handler.go new file mode 100644 index 00000000..8543d832 --- /dev/null +++ b/backend/internal/adapter/handler/handler.go @@ -0,0 +1,102 @@ +package handler + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + "backend/internal/application/usecase" + + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog" +) + +// Limits — защитные лимиты HTTP-слоя (трек 5 «управляемая деградация»). +// Нулевое значение отключает лимит. +type Limits struct { + MaxBodyBytes int64 // больше — 413 + MaxInflight int // одновременно обрабатываемые запросы API + MaxQueue int // ждущие свободного слота; больше — сразу 503 + QueueTimeout time.Duration // дольше ждал слот — 503 +} + +// API — driving HTTP-адаптер: реализует StrictServerInterface из api.gen.go. +// +//go:generate go tool oapi-codegen -config ../../../codegen.yaml ../../../../contracts/openapi.json +type API struct { + app *usecase.Messenger + auth *usecase.Auth + attachments *usecase.Attachments + uploadGate chan struct{} + probe func(context.Context) error + limits Limits + gate *gate + storeStats func() any + ready atomic.Bool + log *zerolog.Logger +} + +// New собирает HTTP-адаптер. storeStats отдаёт состояние хранилища для +// /debug/stats. +func New(app *usecase.Messenger, auth *usecase.Auth, attachments *usecase.Attachments, probe func(context.Context) error, limits Limits, storeStats func() any, log *zerolog.Logger) *API { + h := &API{ + app: app, + auth: auth, + attachments: attachments, + uploadGate: make(chan struct{}, 1), + probe: probe, + limits: limits, + gate: newGate(limits.MaxInflight, limits.MaxQueue, limits.QueueTimeout), + storeStats: storeStats, + log: log, + } + h.ready.Store(true) + return h +} + +// Drain переводит /health/ready в 503 перед остановкой сервера. +func (h *API) Drain() { h.ready.Store(false) } + +func (h *API) Router() http.Handler { + r := chi.NewRouter() + r.Use(requestID, h.recoverer, h.accessLog, h.requestContext) + r.NotFound(func(w http.ResponseWriter, r *http.Request) { + h.writeError(w, r, errRouteNotFound) + }) + r.MethodNotAllowed(func(w http.ResponseWriter, r *http.Request) { + h.writeError(w, r, errMethodNotAllowed) + }) + + strict := NewStrictHandlerWithOptions(h, nil, StrictHTTPServerOptions{ + RequestErrorHandlerFunc: h.bodyError, + ResponseErrorHandlerFunc: h.writeError, + }) + api := ServerInterfaceWrapper{Handler: strict, ErrorHandlerFunc: h.paramError} + + // Health и статистика — мимо очереди: под перегрузкой они должны отвечать. + r.Get("/health/live", api.Liveness) + r.Get("/health/ready", api.Readiness) + r.Get("/debug/stats", h.stats) + + r.Route("/api/v1", func(r chi.Router) { + r.Use(h.gate.middleware(h.writeError)) + r.With(h.jsonBody).Post("/users", api.RegisterUser) + r.Post("/auth/sessions", api.CreateSession) + // Сначала аутентификация, потом тело и доступ к ресурсу. + r.Group(func(r chi.Router) { + r.Use(h.authenticate, h.validQuery, h.jsonBody) + r.Delete("/auth/sessions/current", api.RevokeSession) + r.Get("/users/me", api.CurrentUser) + r.Post("/chats", api.CreateChat) + r.Get("/chats", api.ListChats) + r.Get("/chats/{chat_id}", api.GetChat) + r.Post("/chats/{chat_id}/messages", api.CreateMessage) + r.Get("/chats/{chat_id}/messages", api.ListMessages) + r.Post("/chats/{chat_id}/attachments", api.UploadAttachment) + r.Get("/attachments/{attachment_id}", api.GetAttachment) + r.Get("/attachments/{attachment_id}/content", api.DownloadAttachment) + }) + }) + return r +} diff --git a/backend/internal/adapter/handler/json.go b/backend/internal/adapter/handler/json.go new file mode 100644 index 00000000..6b162ced --- /dev/null +++ b/backend/internal/adapter/handler/json.go @@ -0,0 +1,65 @@ +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "time" +) + +// Timestamp подставляется кодогенератором вместо time.Time (codegen.yaml, +// type-mapping): RFC 3339 UTC с ровно шестью цифрами дробной части. +// time.Time срезает хвостовые нули, а fromisoformat в Python 3.9–3.10 +// (на нём публичные тесты) понимает дробную часть только из 3 или 6 цифр. +type Timestamp time.Time + +const timestampLayout = "2006-01-02T15:04:05.000000Z" + +func (t Timestamp) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Time(t).UTC().Format(timestampLayout) + `"`), nil +} + +func (t *Timestamp) UnmarshalJSON(data []byte) error { + var parsed time.Time + if err := json.Unmarshal(data, &parsed); err != nil { + return err + } + *t = Timestamp(parsed) + return nil +} + +// Тела запросов с лишними полями — 400 по контракту, а сгенерированный +// декодер их молча игнорирует. Строгий разбор подключается методами на +// сгенерированных типах. + +func (c *CreateUser) UnmarshalJSON(data []byte) error { + type plain CreateUser + return decodeStrict(data, (*plain)(c)) +} + +func (c *CreateChat) UnmarshalJSON(data []byte) error { + type plain CreateChat + return decodeStrict(data, (*plain)(c)) +} + +func (c *CreateMessage) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + raw, ok := fields["text"] + if !ok || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("text must be a present string") + } + if ids, ok := fields["attachment_ids"]; ok && bytes.Equal(bytes.TrimSpace(ids), []byte("null")) { + return fmt.Errorf("attachment_ids must be an array") + } + type plain CreateMessage + return decodeStrict(data, (*plain)(c)) +} + +func decodeStrict(data []byte, dst any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + return dec.Decode(dst) +} diff --git a/backend/internal/adapter/handler/message_json_test.go b/backend/internal/adapter/handler/message_json_test.go new file mode 100644 index 00000000..dcca504e --- /dev/null +++ b/backend/internal/adapter/handler/message_json_test.go @@ -0,0 +1,17 @@ +package handler + +import ( + "encoding/json" + "testing" +) + +func TestMessageJSONRequiresStringTextAndArrayIDs(t *testing.T) { + for _, body := range []string{`{"attachment_ids":[]}`, `{"text":null,"attachment_ids":[]}`, `{"text":"ok","attachment_ids":null}`} { + t.Run(body, func(t *testing.T) { + var message CreateMessage + if err := json.Unmarshal([]byte(body), &message); err == nil { + t.Fatal("accepted a missing/null required field") + } + }) + } +} diff --git a/backend/internal/adapter/handler/middleware.go b/backend/internal/adapter/handler/middleware.go new file mode 100644 index 00000000..6e10be00 --- /dev/null +++ b/backend/internal/adapter/handler/middleware.go @@ -0,0 +1,161 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "os" + "runtime/debug" + "strings" + "time" + "unicode/utf8" + + "backend/internal/domain/model" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/google/uuid" +) + +type ctxKey int + +const ( + requestIDKey ctxKey = iota + userKey +) + +// requestID ставит X-Request-Id в каждый ответ, включая ошибки и health. +func requestID(next http.Handler) http.Handler { + instance, _ := os.Hostname() + if instance == "" { + instance = uuid.NewString() + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := uuid.NewString() + w.Header().Set("X-Request-Id", id) + w.Header().Set("X-Instance-Id", instance) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), requestIDKey, id))) + }) +} + +func requestIDFrom(ctx context.Context) string { + id, _ := ctx.Value(requestIDKey).(string) + return id +} + +// recoverer превращает панику в 500 без деталей; стек — только в лог. +func (h *API) recoverer(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + rec := recover() + if rec == nil { + return + } + if rec == http.ErrAbortHandler { + panic(rec) + } + h.log.Error().Interface("panic", rec).Bytes("stack", debug.Stack()). + Str("request_id", requestIDFrom(r.Context())).Msg("panic") + h.writeError(w, r, errors.New("panic")) + }() + next.ServeHTTP(w, r) + }) +} + +// accessLog пишет метод, шаблон маршрута, статус и длительность. Тело, query +// и заголовки не логируются: там текст сообщений и идентификаторы. +func (h *API) accessLog(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(ww, r) + route := chi.RouteContext(r.Context()).RoutePattern() + if route == "" { + route = "unmatched" + } + event := h.log.Info() + if ww.Status() >= http.StatusInternalServerError { + event = h.log.Warn() + } + event.Str("request_id", requestIDFrom(r.Context())). + Str("method", r.Method). + Str("route", route). + Int("status", ww.Status()). + Dur("duration", time.Since(start)). + Msg("request") + }) +} + +// authenticate verifies persisted Bearer sessions for protected business routes. +// The session-creation endpoint separately validates Basic credentials. +func (h *API) authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + parts := strings.Fields(r.Header.Get("Authorization")) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + h.writeError(w, r, model.ErrUnauthorized) + return + } + user, err := h.auth.Authenticate(r.Context(), parts[1]) + if err != nil { + if errors.Is(err, model.ErrUnauthorized) { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + h.writeError(w, r, err) + return + } + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), userKey, user))) + }) +} + +// validQuery отсекает query с битым %-экранированием: url.Values молча +// выбросил бы такой параметр, и cursor=%%% превратился бы в «с начала». +func (h *API) validQuery(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, err := url.ParseQuery(r.URL.RawQuery); err != nil { + h.writeError(w, r, &model.ValidationError{Message: "некорректная строка запроса"}) + return + } + next.ServeHTTP(w, r) + }) +} + +func currentUser(ctx context.Context) model.User { + user, _ := ctx.Value(userKey).(model.User) + return user +} + +// jsonBody читает тело POST с лимитом размера (иначе 413) и отсекает +// синтаксически неверный JSON (400) до кодогенерированного декодера: тот +// читает только первое JSON-значение и пропустил бы мусор после него. +func (h *API) jsonBody(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || strings.HasSuffix(r.URL.Path, "/attachments") { + next.ServeHTTP(w, r) + return + } + reader := io.Reader(r.Body) + if h.limits.MaxBodyBytes > 0 { + reader = http.MaxBytesReader(w, r.Body, h.limits.MaxBodyBytes) + } + body, err := io.ReadAll(reader) + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + h.writeError(w, r, errPayloadTooLarge) + return + case err != nil: + h.writeError(w, r, &model.ValidationError{Message: "тело запроса не прочитано"}) + return + case !utf8.Valid(body) || !json.Valid(body): + h.writeError(w, r, &model.ValidationError{Message: "некорректный JSON"}) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + next.ServeHTTP(w, r) + }) +} diff --git a/backend/internal/adapter/handler/stats.go b/backend/internal/adapter/handler/stats.go new file mode 100644 index 00000000..244dfbf2 --- /dev/null +++ b/backend/internal/adapter/handler/stats.go @@ -0,0 +1,47 @@ +package handler + +import ( + "bufio" + "encoding/json" + "net/http" + "os" + "runtime" + "runtime/debug" + "strconv" + "strings" +) + +// stats — расширение за пределами контракта (contracts/extensions.md): память +// процесса, учтённый объём хранилища и очередь. Нужен для графиков трека 5. +func (h *API) stats(w http.ResponseWriter, _ *http.Request) { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "rss_bytes": rssBytes(), + "heap_alloc": mem.HeapAlloc, + "heap_sys": mem.HeapSys, + "go_memory_limit": debug.SetMemoryLimit(-1), + "num_gc": mem.NumGC, + "goroutines": runtime.NumGoroutine(), + "gate": h.gate.stats(), + "store": h.storeStats(), + }) +} + +// rssBytes читает VmRSS из /proc (только Linux, то есть в контейнере); иначе 0. +func rssBytes() int64 { + f, err := os.Open("/proc/self/status") + if err != nil { + return 0 + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + if kb, ok := strings.CutPrefix(sc.Text(), "VmRSS:"); ok { + n, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimSuffix(kb, "kB")), 10, 64) + return n * 1024 + } + } + return 0 +} diff --git a/backend/internal/adapter/handler/utf8_test.go b/backend/internal/adapter/handler/utf8_test.go new file mode 100644 index 00000000..70d33f9c --- /dev/null +++ b/backend/internal/adapter/handler/utf8_test.go @@ -0,0 +1,21 @@ +package handler + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRejectsInvalidUTF8BeforeJSONDecoding(t *testing.T) { + body := append([]byte(`{"display_name":"`), 0xff) + body = append(body, []byte(`"}`)...) + request := httptest.NewRequest(http.MethodPost, "/api/v1/users", bytes.NewReader(body)) + response := httptest.NewRecorder() + h := &API{limits: Limits{MaxBodyBytes: 65536}} + called := false + requestID(h.jsonBody(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true; w.WriteHeader(204) }))).ServeHTTP(response, request) + if response.Code != 400 || called { + t.Fatalf("invalid UTF-8 reached decoder: status=%d called=%t", response.Code, called) + } +} diff --git a/backend/internal/adapter/imagefile/image.go b/backend/internal/adapter/imagefile/image.go new file mode 100644 index 00000000..8c1a4aed --- /dev/null +++ b/backend/internal/adapter/imagefile/image.go @@ -0,0 +1,72 @@ +package imagefile + +import ( + "backend/internal/domain/model" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "os" +) + +var ErrTooLarge = errors.New("image too large") +var ErrUnsupported = errors.New("invalid image or MIME") + +type File struct { + *os.File + Size int64 + SHA256, ContentType string +} + +func (f *File) Dispose() { _ = f.Close(); _ = os.Remove(f.Name()) } +func Inspect(body io.Reader, filename, mime string) (out *File, err error) { + if err = model.ValidateFilename(filename); err != nil { + return nil, err + } + if mime != "image/png" && mime != "image/jpeg" { + return nil, ErrUnsupported + } + f, e := os.CreateTemp("", "messenger-upload-*") + if e != nil { + return nil, e + } + defer func() { + if err != nil { + _ = f.Close() + _ = os.Remove(f.Name()) + } + }() + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(body, model.MaxImageBytes+1)) + if e != nil { + return nil, &model.ValidationError{Message: "incomplete image upload"} + } + if n > model.MaxImageBytes { + return nil, ErrTooLarge + } + if _, e = f.Seek(0, io.SeekStart); e != nil { + return nil, e + } + cfg, format, e := image.DecodeConfig(f) + if e != nil || "image/"+format != mime { + return nil, ErrUnsupported + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if cfg.Width < 1 || cfg.Height < 1 || pixels > model.MaxImagePixels { + return nil, ErrTooLarge + } + if _, e = f.Seek(0, io.SeekStart); e != nil { + return nil, e + } + decoded, actual, e := image.Decode(f) + if e != nil || actual != format || decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height { + return nil, ErrUnsupported + } + if _, e = f.Seek(0, io.SeekStart); e != nil { + return nil, e + } + return &File{f, n, hex.EncodeToString(hash.Sum(nil)), mime}, nil +} diff --git a/backend/internal/adapter/imagefile/image_test.go b/backend/internal/adapter/imagefile/image_test.go new file mode 100644 index 00000000..76b3db55 --- /dev/null +++ b/backend/internal/adapter/imagefile/image_test.go @@ -0,0 +1,81 @@ +package imagefile + +import ( + "backend/internal/domain/model" + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "image" + "image/png" + "io" + "testing" +) + +func pngBytes(t *testing.T) []byte { + t.Helper() + var b bytes.Buffer + if e := png.Encode(&b, image.NewNRGBA(image.Rect(0, 0, 3, 2))); e != nil { + t.Fatal(e) + } + return b.Bytes() +} +func TestOriginalBytesArePreserved(t *testing.T) { + raw := pngBytes(t) + f, e := Inspect(bytes.NewReader(raw), "пример.png", "image/png") + if e != nil { + t.Fatal(e) + } + defer f.Dispose() + got, e := io.ReadAll(f) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(got, raw) { + t.Fatal("original changed") + } +} +func TestRejectsInvalidImageAndFilename(t *testing.T) { + for _, tc := range []struct { + name, mime string + body []byte + want error + }{ + {"a.png", "image/jpeg", pngBytes(t), ErrUnsupported}, + {"a.png", "image/png", []byte("not an image"), ErrUnsupported}, + {"a.png", "application/octet-stream", pngBytes(t), ErrUnsupported}, + {"a.png", "image/png", append(pngBytes(t)[:20], 0), ErrUnsupported}, + } { + t.Run(tc.mime+string(tc.body[:1]), func(t *testing.T) { + _, e := Inspect(bytes.NewReader(tc.body), tc.name, tc.mime) + if !errors.Is(e, tc.want) { + t.Fatalf("got %v, want %v", e, tc.want) + } + }) + } + for _, name := range []string{"../x.png", "a\\b", "a\x00b", "a\nb", ""} { + if _, e := Inspect(bytes.NewReader(pngBytes(t)), name, "image/png"); e == nil { + t.Fatalf("accepted unsafe filename %q", name) + } + } +} +func TestRejectsOversizedStreamBeforeDecoding(t *testing.T) { + _, e := Inspect(io.LimitReader(zeroReader{}, model.MaxImageBytes+1), "a.png", "image/png") + if !errors.Is(e, ErrTooLarge) { + t.Fatalf("got %v", e) + } +} + +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } +func TestRejectsPixelBombBeforeDecoding(t *testing.T) { + raw := pngBytes(t) + binary.BigEndian.PutUint32(raw[16:20], 50000) + binary.BigEndian.PutUint32(raw[20:24], 50000) + binary.BigEndian.PutUint32(raw[29:33], crc32.ChecksumIEEE(raw[12:29])) + _, e := Inspect(bytes.NewReader(raw), "huge.png", "image/png") + if !errors.Is(e, ErrTooLarge) { + t.Fatalf("pixel limit did not apply before decode: %v", e) + } +} diff --git a/backend/internal/adapter/imagefile/stress_test.go b/backend/internal/adapter/imagefile/stress_test.go new file mode 100644 index 00000000..ac9482f6 --- /dev/null +++ b/backend/internal/adapter/imagefile/stress_test.go @@ -0,0 +1,135 @@ +package imagefile + +import ( + "backend/internal/adapter/password" + "bytes" + "compress/zlib" + "encoding/binary" + "hash/crc32" + "io" + "os" + "runtime" + "runtime/debug" + "strconv" + "strings" + "sync" + "testing" +) + +// A normal PNG encoder allocates several full-width filter rows itself. +// Construct a tiny, highly compressible adversarial fixture as a stream instead. +func extremePNG(t *testing.T, interlaced bool) []byte { + t.Helper() + const width = 40_000_000 + const height = 1 + var idat bytes.Buffer + z := zlib.NewWriter(&idat) + passes := [][4]int{{0, 0, 1, 1}} + if interlaced { + passes = [][4]int{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}} + } + for _, p := range passes { + w := (width - p[0] + p[2] - 1) / p[2] + h := (height - p[1] + p[3] - 1) / p[3] + if width <= p[0] || height <= p[1] || w <= 0 || h <= 0 { + continue + } + for row := 0; row < h; row++ { + if _, e := z.Write([]byte{0}); e != nil { + t.Fatal(e) + } + if _, e := io.CopyN(z, zeroReader{}, int64(w)*8); e != nil { + t.Fatal(e) + } + } + } + if e := z.Close(); e != nil { + t.Fatal(e) + } + var out bytes.Buffer + out.Write([]byte{137, 80, 78, 71, 13, 10, 26, 10}) + chunk := func(kind string, data []byte) { + var word [4]byte + binary.BigEndian.PutUint32(word[:], uint32(len(data))) + out.Write(word[:]) + out.WriteString(kind) + out.Write(data) + crc := crc32.NewIEEE() + _, _ = crc.Write([]byte(kind)) + _, _ = crc.Write(data) + binary.BigEndian.PutUint32(word[:], crc.Sum32()) + out.Write(word[:]) + } + header := make([]byte, 13) + binary.BigEndian.PutUint32(header[:4], width) + binary.BigEndian.PutUint32(header[4:8], height) + header[8] = 16 + header[9] = 6 + if interlaced { + header[12] = 1 + } + chunk("IHDR", header) + chunk("IDAT", idat.Bytes()) + chunk("IEND", nil) + return out.Bytes() +} +func TestWorstCaseImagesWithinBudget(t *testing.T) { + if os.Getenv("RUN_IMAGE_STRESS") != "1" { + t.Skip("explicit isolated memory experiment required") + } + old := debug.SetMemoryLimit(1536 * 1024 * 1024) + defer debug.SetMemoryLimit(old) + for _, interlaced := range []bool{false, true} { + raw := extremePNG(t, interlaced) + runtime.GC() + stop := make(chan struct{}) + var workers sync.WaitGroup + for i := 0; i < 3; i++ { + workers.Add(1) + go func() { + defer workers.Done() + for { + select { + case <-stop: + return + default: + if _, e := password.New().Hash("synthetic-concurrent-KDF"); e != nil { + t.Error(e) + return + } + } + } + }() + } + for repeat := 0; repeat < 2; repeat++ { + f, e := Inspect(bytes.NewReader(raw), "wide.png", "image/png") + if e != nil { + close(stop) + workers.Wait() + t.Fatal(e) + } + f.Dispose() + } + close(stop) + workers.Wait() + runtime.GC() + } + raw, e := os.ReadFile("/proc/self/status") + if e != nil { + t.Fatal(e) + } + peak := int64(0) + for _, line := range strings.Split(string(raw), "\n") { + if strings.HasPrefix(line, "VmHWM:") { + fields := strings.Fields(line) + peak, e = strconv.ParseInt(fields[1], 10, 64) + if e != nil { + t.Fatal(e) + } + } + } + if peak <= 0 || peak >= 2*1024*1024 { + t.Fatalf("RSS high-water mark out of budget: %d KiB", peak) + } + t.Logf("PASS native RGBA64 40000000x1, normal+Adam7, two sequential validations each, 3 concurrent Argon2 workers; peak_rss_kib=%d hard_limit_kib=2097152", peak) +} diff --git a/backend/internal/adapter/imagefile/variants.go b/backend/internal/adapter/imagefile/variants.go new file mode 100644 index 00000000..e30bab6d --- /dev/null +++ b/backend/internal/adapter/imagefile/variants.go @@ -0,0 +1,129 @@ +package imagefile + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "image" + "io" + "os" + "os/exec" +) + +func FitDimensions(w, h, max int) (int, int) { + if w <= max && h <= max { + return w, h + } + if w >= h { + return max, maxInt(1, h*max/w) + } + return maxInt(1, w*max/h), max +} +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +type Transformer struct{} + +func NewTransformer() port.Transformer { return Transformer{} } + +// ImageMagick is a constrained subprocess; no input names or shell fragments are accepted. +func (Transformer) Transform(ctx context.Context, r io.Reader) ([]port.ProcessedVariant, error) { + input, err := os.CreateTemp("", "pipeline-original-*") + if err != nil { + return nil, err + } + defer func() { _ = input.Close(); _ = os.Remove(input.Name()) }() + n, err := io.Copy(input, io.LimitReader(r, model.MaxImageBytes+1)) + if err != nil { + return nil, err + } + if n > model.MaxImageBytes { + return nil, model.ErrInvalidImage + } + if _, err = input.Seek(0, 0); err != nil { + return nil, err + } + config, _, err := image.DecodeConfig(input) + if err != nil || int64(config.Width)*int64(config.Height) > model.MaxImagePixels { + return nil, model.ErrInvalidImage + } + probe := exec.CommandContext(ctx, "convert", "-limit", "memory", "256MiB", "-limit", "map", "256MiB", "-limit", "disk", "0", input.Name()+"[0]", "-auto-orient", "-format", "%w %h", "info:") + raw, err := probe.Output() + if err != nil { + return nil, fmt.Errorf("image subprocess unavailable: %w", err) + } + var originalW, originalH int + if _, err = fmt.Sscanf(string(raw), "%d %d", &originalW, &originalH); err != nil || int64(originalW)*int64(originalH) > model.MaxImagePixels { + return nil, model.ErrInvalidImage + } + var variants []port.ProcessedVariant + for _, v := range []struct { + name, mime, geometry, format string + crop bool + }{{"thumbnail", "image/jpeg", "256x256>", "jpeg", false}, {"crop", "image/jpeg", "128x128^", "jpeg", true}, {"optimized", "image/webp", "1280x1280>", "webp", false}} { + geometry := v.geometry + if !v.crop { + max := 256 + if v.name == "optimized" { + max = 1280 + } + w, h := FitDimensions(originalW, originalH, max) + geometry = fmt.Sprintf("%dx%d!", w, h) + } + args := []string{"-limit", "memory", "256MiB", "-limit", "map", "256MiB", "-limit", "disk", "0", "-limit", "thread", "1", input.Name() + "[0]", "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "-resize", geometry} + if v.crop { + args = append(args, "-gravity", "center", "-extent", "128x128") + } + args = append(args, "-quality", "82", v.format+":-") + cmd := exec.CommandContext(ctx, "convert", args...) + var out limitedBuffer + cmd.Stdout = &out + if err = cmd.Run(); err != nil { + return nil, fmt.Errorf("image subprocess failed: %w", err) + } + b := out.Bytes() + if len(b) == 0 { + return nil, fmt.Errorf("empty variant") + } + w, h := 0, 0 + if v.format == "webp" { + // Read only geometry through the native decoder; never dump diagnostics/payload. + probe := exec.CommandContext(ctx, "identify", "-format", "%w %h", "webp:-") + probe.Stdin = bytes.NewReader(b) + raw, e := probe.Output() + if e != nil { + return nil, fmt.Errorf("invalid generated WebP") + } + if _, e = fmt.Sscanf(string(raw), "%d %d", &w, &h); e != nil { + return nil, e + } + } else { + // JPEG dimensions are read through the same strict decoder used for uploads. + cfg, _, e := image.DecodeConfig(bytes.NewReader(b)) + if e != nil { + return nil, e + } + w, h = cfg.Width, cfg.Height + } + sum := sha256.Sum256(b) + variants = append(variants, port.ProcessedVariant{Metadata: model.Variant{Name: v.name, ContentType: v.mime, SHA256: hex.EncodeToString(sum[:]), SizeBytes: int64(len(b)), Width: w, Height: h}, Bytes: b}) + } + return variants, nil +} + +type limitedBuffer struct{ bytes.Buffer } + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if b.Len()+len(p) > 16*1024*1024 { + return 0, fmt.Errorf("variant too large") + } + return b.Buffer.Write(p) +} diff --git a/backend/internal/adapter/imagefile/variants_integration_test.go b/backend/internal/adapter/imagefile/variants_integration_test.go new file mode 100644 index 00000000..82676ca1 --- /dev/null +++ b/backend/internal/adapter/imagefile/variants_integration_test.go @@ -0,0 +1,93 @@ +package imagefile + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "testing" + "time" +) + +func TestActualVariantsHonorDimensionsAlphaAndEXIF(t *testing.T) { + if os.Getenv("RUN_VARIANT_TESTS") != "1" { + t.Skip("requires actual ImageMagick runtime") + } + for _, c := range []struct { + name string + w, h int + orientation bool + }{{"landscape", 320, 200, false}, {"tiny", 1, 1, false}, {"portrait", 23, 67, false}, {"exif", 40, 20, true}} { + t.Run(c.name, func(t *testing.T) { + img := image.NewNRGBA(image.Rect(0, 0, c.w, c.h)) + img.Set(0, 0, color.NRGBA{R: 255, A: 0}) + var encoded bytes.Buffer + if c.orientation { + if e := jpeg.Encode(&encoded, img, &jpeg.Options{Quality: 82}); e != nil { + t.Fatal(e) + } + } else { + if e := png.Encode(&encoded, img); e != nil { + t.Fatal(e) + } + } + input := encoded.Bytes() + if c.orientation { + payload := []byte{'E', 'x', 'i', 'f', 0, 0, 'I', 'I', 42, 0, 8, 0, 0, 0, 1, 0, 18, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0} + marker := append([]byte{255, 225, 0, byte(len(payload) + 2)}, payload...) + input = append(append(append([]byte{}, input[:2]...), marker...), input[2:]...) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + variants, e := NewTransformer().Transform(ctx, bytes.NewReader(input)) + if e != nil { + t.Fatal(e) + } + if len(variants) != 3 { + t.Fatalf("want three variants, got %d", len(variants)) + } + w, h := c.w, c.h + if c.orientation { + w, h = h, w + } + for _, v := range variants { + s := sha256.Sum256(v.Bytes) + if v.Metadata.SHA256 != hex.EncodeToString(s[:]) || v.Metadata.SizeBytes != int64(len(v.Bytes)) { + t.Fatal("content digest/size mismatch") + } + wantW, wantH := w, h + if v.Metadata.Name == "thumbnail" { + wantW, wantH = FitDimensions(w, h, 256) + } + if v.Metadata.Name == "crop" { + wantW, wantH = 128, 128 + } + if v.Metadata.Width != wantW || v.Metadata.Height != wantH { + t.Fatalf("%s: got %dx%d want %dx%d", v.Metadata.Name, v.Metadata.Width, v.Metadata.Height, wantW, wantH) + } + if v.Metadata.ContentType == "image/jpeg" { + decoded, e := jpeg.Decode(bytes.NewReader(v.Bytes)) + if e != nil { + t.Fatal(e) + } + if decoded.Bounds().Dx() != wantW || decoded.Bounds().Dy() != wantH { + t.Fatal("actual JPEG geometry mismatch") + } + if !c.orientation { + r, g, b, _ := decoded.At(0, 0).RGBA() + if r < 64000 || g < 64000 || b < 64000 { + t.Fatal("alpha was not composited on white") + } + } + } else if len(v.Bytes) < 16 || string(v.Bytes[:4]) != "RIFF" || string(v.Bytes[8:12]) != "WEBP" { + t.Fatal("optimized file is not WebP") + } + } + }) + } +} diff --git a/backend/internal/adapter/imagefile/variants_test.go b/backend/internal/adapter/imagefile/variants_test.go new file mode 100644 index 00000000..5de37853 --- /dev/null +++ b/backend/internal/adapter/imagefile/variants_test.go @@ -0,0 +1,12 @@ +package imagefile + +import "testing" + +func TestFitDimensionsPreservesAspectAndDoesNotUpscale(t *testing.T) { + for _, c := range []struct{ w, h, max, wantW, wantH int }{{320, 200, 256, 256, 160}, {1, 1, 256, 1, 1}, {1, 400, 256, 1, 256}, {400, 1, 256, 256, 1}, {200, 320, 256, 160, 256}, {1280, 1280, 1280, 1280, 1280}} { + w, h := FitDimensions(c.w, c.h, c.max) + if w != c.wantW || h != c.wantH { + t.Errorf("%+v: got %dx%d", c, w, h) + } + } +} diff --git a/backend/internal/adapter/password/argon.go b/backend/internal/adapter/password/argon.go new file mode 100644 index 00000000..a58ca6f0 --- /dev/null +++ b/backend/internal/adapter/password/argon.go @@ -0,0 +1,36 @@ +package password + +import ( + "backend/internal/domain/port" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "golang.org/x/crypto/argon2" + "strings" +) + +type Argon struct{} + +func New() port.Passwords { return Argon{} } +func (Argon) Hash(password string) (string, error) { + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, 2, 32*1024, 1, 32) + return fmt.Sprintf("argon2id$v19$m32768t2p1$%s$%s", base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(hash)), nil +} +func (Argon) Verify(password, encoded string) bool { + parts := strings.Split(encoded, "$") + if len(parts) != 5 || parts[0] != "argon2id" || parts[1] != "v19" || parts[2] != "m32768t2p1" { + return false + } + salt, e1 := base64.RawStdEncoding.DecodeString(parts[3]) + want, e2 := base64.RawStdEncoding.DecodeString(parts[4]) + if e1 != nil || e2 != nil || len(salt) != 16 || len(want) != 32 { + return false + } + got := argon2.IDKey([]byte(password), salt, 2, 32*1024, 1, 32) + return subtle.ConstantTimeCompare(got, want) == 1 +} diff --git a/backend/internal/adapter/queue/streams.go b/backend/internal/adapter/queue/streams.go new file mode 100644 index 00000000..5bb815ee --- /dev/null +++ b/backend/internal/adapter/queue/streams.go @@ -0,0 +1,133 @@ +package queue + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "bytes" + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + "time" +) + +const stream = "attachment-jobs" + +type Broker struct { + client *redis.Client + store port.PipelineStore + consumer string +} + +func New(client *redis.Client, store port.PipelineStore, consumer string) *Broker { + return &Broker{client, store, consumer} +} +func (b *Broker) Publish(ctx context.Context) error { + jobs, e := b.store.PendingJobs(ctx) + if e != nil { + return e + } + for _, j := range jobs { + j.Attempt++ + payload := struct { + Schema int `json:"schema_version"` + Type string `json:"type"` + model.Job + }{1, "attachment.process.v1", j} + raw, e := json.Marshal(payload) + if e != nil { + return e + } + if e = b.client.XAdd(ctx, &redis.XAddArgs{Stream: stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(raw)}}).Err(); e != nil { + return e + } + if e = b.store.Published(ctx, j.ID); e != nil { + return e + } + } + return nil +} +func (b *Broker) RunPublisher(ctx context.Context, onError func(error)) { + tick := time.NewTicker(500 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if e := b.Publish(ctx); e != nil { + onError(e) + } + } + } +} +func (b *Broker) ack(ctx context.Context, id string) { + if b.client.XAck(ctx, stream, "workers", id).Err() == nil { + _ = b.client.XDel(ctx, stream, id).Err() + } +} +func (b *Broker) consume(ctx context.Context, m redis.XMessage, process func(context.Context, model.Job) error, onError func(error)) { + raw, ok := m.Values["job"].(string) + if !ok || len(raw) > 2048 { + b.ack(ctx, m.ID) + return + } + var envelope struct { + Schema int `json:"schema_version"` + Type string `json:"type"` + model.Job + } + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(&envelope) != nil || envelope.Schema != 1 || envelope.Type != "attachment.process.v1" || envelope.ID == uuid.Nil || envelope.AttachmentID == uuid.Nil || envelope.PipelineVersion < 1 || envelope.Attempt < 1 { + b.ack(ctx, m.ID) + return + } + work, cancel := context.WithTimeout(ctx, 20*time.Second) + e := process(work, envelope.Job) + cancel() + if e != nil { + onError(e) + } else { + b.ack(ctx, m.ID) + } +} +func (b *Broker) RunWorker(ctx context.Context, process func(context.Context, model.Job) error, onError func(error)) { + for ctx.Err() == nil { + e := b.client.XGroupCreateMkStream(ctx, stream, "workers", "0").Err() + if e != nil && e.Error() != "BUSYGROUP Consumer Group name already exists" { + onError(e) + pause(ctx) + continue + } + // Reclaim abandoned PEL entries; the DB lease and version fence are authoritative. + claimed, _, e := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: stream, Group: "workers", Consumer: b.consumer, MinIdle: 30 * time.Second, Start: "0-0", Count: 1}).Result() + if e == nil && len(claimed) > 0 { + for _, m := range claimed { + b.consume(ctx, m, process, onError) + } + continue + } + rows, e := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: "workers", Consumer: b.consumer, Streams: []string{stream, ">"}, Count: 1, Block: time.Second}).Result() + if errors.Is(e, redis.Nil) { + continue + } + if e != nil { + onError(e) + pause(ctx) + continue + } + for _, row := range rows { + for _, m := range row.Messages { + b.consume(ctx, m, process, onError) + } + } + } +} +func pause(ctx context.Context) { + select { + case <-ctx.Done(): + case <-time.After(200 * time.Millisecond): + } +} diff --git a/backend/internal/adapter/repository/postgres/attachments.go b/backend/internal/adapter/repository/postgres/attachments.go new file mode 100644 index 00000000..32259a1c --- /dev/null +++ b/backend/internal/adapter/repository/postgres/attachments.go @@ -0,0 +1,57 @@ +package postgres + +import ( + "backend/internal/adapter/repository/postgres/queries" + "backend/internal/domain/model" + "context" + "encoding/json" + "github.com/google/uuid" +) + +func attachment(r queries.Attachment) model.Attachment { + var variants []model.Variant + _ = json.Unmarshal(r.Variants, &variants) + return model.Attachment{ID: r.ID, ChatID: r.ChatID, OwnerID: r.OwnerID, Filename: r.Filename, ContentType: r.ContentType, SizeBytes: r.SizeBytes, SHA256: r.Sha256, Status: r.Status, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, PipelineVersion: r.PipelineVersion, ErrorCode: r.ErrorCode, Variants: variants, StagingExpired: r.StagingExpired, StagingCleaned: r.StagingCleaned} +} +func (s *Store) ReserveAttachment(ctx context.Context, a model.Attachment) (model.Attachment, error) { + tx, e := s.pool.Begin(ctx) + if e != nil { + return model.Attachment{}, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + if _, e = q.ChargeAttachment(ctx, a.OwnerID); e != nil { + return model.Attachment{}, model.ErrStorageFull + } + r, e := q.InsertAttachment(ctx, queries.InsertAttachmentParams{ID: a.ID, ChatID: a.ChatID, OwnerID: a.OwnerID, Filename: a.Filename, ContentType: a.ContentType, SizeBytes: a.SizeBytes, Sha256: a.SHA256}) + if e != nil { + return model.Attachment{}, mapErr(e) + } + if e = tx.Commit(ctx); e != nil { + return model.Attachment{}, mapErr(e) + } + return attachment(r), nil +} +func (s *Store) GetAttachment(ctx context.Context, id uuid.UUID) (model.Attachment, error) { + r, e := s.q.GetAttachment(ctx, id) + return attachment(r), mapErr(e) +} +func (s *Store) SetAttachmentStatus(ctx context.Context, id uuid.UUID, from, to string) (bool, error) { + n, e := s.q.SetAttachmentStatus(ctx, queries.SetAttachmentStatusParams{ID: id, Status: to, Status_2: from}) + return n == 1, mapErr(e) +} +func (s *Store) ReconcileAttachments(ctx context.Context, after uuid.UUID) ([]model.Attachment, bool, error) { + rows, e := s.q.ReconcileAttachments(ctx, after) + if e != nil { + return nil, false, mapErr(e) + } + more := len(rows) > 100 + if more { + rows = rows[:100] + } + out := make([]model.Attachment, len(rows)) + for i, r := range rows { + out[i] = attachment(r) + } + return out, more, nil +} diff --git a/backend/internal/adapter/repository/postgres/pipeline.go b/backend/internal/adapter/repository/postgres/pipeline.go new file mode 100644 index 00000000..d214a679 --- /dev/null +++ b/backend/internal/adapter/repository/postgres/pipeline.go @@ -0,0 +1,171 @@ +package postgres + +import ( + "backend/internal/adapter/repository/postgres/queries" + "backend/internal/domain/model" + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +func job(r queries.AttachmentJob) model.Job { + return model.Job{ID: r.ID, AttachmentID: r.AttachmentID, PipelineVersion: r.PipelineVersion, Attempt: r.Attempt, CreatedAt: r.CreatedAt, Traceparent: r.Traceparent, Tracestate: r.Tracestate} +} +func (s *Store) QueueAttachment(ctx context.Context, a model.Attachment, tp, ts string) (model.Attachment, error) { + tx, e := s.pool.Begin(ctx) + if e != nil { + return model.Attachment{}, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + r, e := q.QueueAttachment(ctx, a.ID) + if e != nil { + return model.Attachment{}, mapErr(e) + } + if e = q.InsertJob(ctx, queries.InsertJobParams{JobID: uuid.New(), AttachmentID: r.ID, PipelineVersion: r.PipelineVersion, Traceparent: tp, Tracestate: ts}); e != nil { + return model.Attachment{}, mapErr(e) + } + if e = tx.Commit(ctx); e != nil { + return model.Attachment{}, mapErr(e) + } + return attachment(r), nil +} +func (s *Store) PendingJobs(ctx context.Context) ([]model.Job, error) { + rows, e := s.q.PendingJobs(ctx) + if e != nil { + return nil, mapErr(e) + } + out := make([]model.Job, len(rows)) + for i, r := range rows { + out[i] = job(r) + } + return out, nil +} +func (s *Store) Published(ctx context.Context, id uuid.UUID) error { + return mapErr(s.q.Published(ctx, id)) +} +func (s *Store) AcquireJob(ctx context.Context, j model.Job, lease uuid.UUID, d time.Duration) (model.Job, model.Attachment, bool, error) { + tx, e := s.pool.Begin(ctx) + if e != nil { + return j, model.Attachment{}, false, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + r, e := q.AcquireJob(ctx, queries.AcquireJobParams{LeaseID: lease, LeaseSeconds: int32(d / time.Second), JobID: j.ID, PipelineVersion: j.PipelineVersion}) + if errors.Is(e, pgx.ErrNoRows) { + return j, model.Attachment{}, false, nil + } + if e != nil { + return j, model.Attachment{}, false, mapErr(e) + } + a, e := q.ProcessingAttachment(ctx, queries.ProcessingAttachmentParams{AttachmentID: r.AttachmentID, PipelineVersion: r.PipelineVersion}) + if e != nil { + return j, model.Attachment{}, false, mapErr(e) + } + if e = tx.Commit(ctx); e != nil { + return j, model.Attachment{}, false, mapErr(e) + } + return job(r), attachment(a), true, nil +} +func (s *Store) CompleteJob(ctx context.Context, j model.Job, lease uuid.UUID, v []model.Variant) (bool, error) { + raw, e := json.Marshal(v) + if e != nil { + return false, e + } + tx, e := s.pool.Begin(ctx) + if e != nil { + return false, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + n, e := q.CompleteAttachment(ctx, queries.CompleteAttachmentParams{Variants: raw, JobID: j.ID, LeaseID: lease}) + if e != nil { + return false, mapErr(e) + } + if n != 1 { + return false, nil + } + if e = q.CompleteJob(ctx, queries.CompleteJobParams{JobID: j.ID, LeaseID: lease}); e != nil { + return false, mapErr(e) + } + return true, mapErr(tx.Commit(ctx)) +} +func (s *Store) RetryJob(ctx context.Context, j model.Job, lease uuid.UUID, code string, delay time.Duration, failed bool) error { + status := "queued" + if failed { + status = "failed" + } + tx, e := s.pool.Begin(ctx) + if e != nil { + return mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + n, e := q.RetryAttachment(ctx, queries.RetryAttachmentParams{NextStatus: status, ErrorCode: code, JobID: j.ID, LeaseID: lease}) + if e != nil { + return mapErr(e) + } + if n == 0 { + return nil + } + if e = q.RetryJob(ctx, queries.RetryJobParams{DelayMs: int32(delay / time.Millisecond), Completed: failed, JobID: j.ID, LeaseID: lease}); e != nil { + return mapErr(e) + } + return mapErr(tx.Commit(ctx)) +} +func (s *Store) ReplayJob(ctx context.Context, id uuid.UUID) (bool, error) { + tx, e := s.pool.Begin(ctx) + if e != nil { + return false, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + a, e := q.ReplayAttachment(ctx, id) + if errors.Is(e, pgx.ErrNoRows) { + return false, nil + } + if e != nil { + return false, mapErr(e) + } + if e = q.ReplayJobs(ctx, queries.ReplayJobsParams{AttachmentID: id, PipelineVersion: a.PipelineVersion}); e != nil { + return false, mapErr(e) + } + return true, mapErr(tx.Commit(ctx)) +} +func (s *Store) AdvancePipeline(ctx context.Context, id uuid.UUID) (bool, error) { + tx, e := s.pool.Begin(ctx) + if e != nil { + return false, mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + a, e := q.AdvancePipeline(ctx, id) + if errors.Is(e, pgx.ErrNoRows) { + return false, nil + } + if e != nil { + return false, mapErr(e) + } + if e = q.InsertJob(ctx, queries.InsertJobParams{JobID: uuid.New(), AttachmentID: id, PipelineVersion: a.PipelineVersion}); e != nil { + return false, mapErr(e) + } + return true, mapErr(tx.Commit(ctx)) +} + +func (s *Store) StagingCleanup(ctx context.Context) ([]model.Attachment, error) { + rows, e := s.q.StagingCleanup(ctx) + if e != nil { + return nil, mapErr(e) + } + out := make([]model.Attachment, len(rows)) + for i, r := range rows { + out[i] = attachment(r) + } + return out, nil +} +func (s *Store) StagingCleaned(ctx context.Context, id uuid.UUID) error { + return mapErr(s.q.StagingCleaned(ctx, id)) +} diff --git a/backend/internal/adapter/repository/postgres/pipeline_integration_test.go b/backend/internal/adapter/repository/postgres/pipeline_integration_test.go new file mode 100644 index 00000000..d4de8c0d --- /dev/null +++ b/backend/internal/adapter/repository/postgres/pipeline_integration_test.go @@ -0,0 +1,331 @@ +package postgres + +import ( + "backend/internal/adapter/storage" + "backend/internal/application/usecase" + "backend/internal/config" + "backend/internal/domain/model" + "backend/internal/domain/port" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "io" + "os" + "reflect" + "sync" + "syscall" + "testing" + "time" +) + +func TestReplayAfterCompletedPreviousVersionCleanup(t *testing.T) { + if os.Getenv("RUN_PIPELINE_DB_TESTS") != "1" { + t.Skip("explicit isolated database required") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + raw, e := config.ReadSecret("/run/secrets/database_url") + if e != nil { + t.Fatal("database file unavailable") + } + pool, e := pgxpool.New(ctx, raw) + if e != nil { + t.Fatal("database connection unavailable") + } + defer pool.Close() + s := New(pool, nil) + user, e := s.CreateUser(ctx, model.UserDraft{Username: "v2_" + uuid.NewString()[:8], DisplayName: "Synthetic version test", PasswordHash: "disabled-synthetic"}) + if e != nil { + t.Fatal(e) + } + var chatID, attachmentID uuid.UUID + defer func() { + cleanup, c := context.WithTimeout(context.Background(), 5*time.Second) + defer c() + for _, q := range []struct { + sql string + id uuid.UUID + }{{"DELETE FROM attachment_jobs WHERE attachment_id=$1", attachmentID}, {"DELETE FROM attachments WHERE id=$1", attachmentID}, {"DELETE FROM chat_members WHERE chat_id=$1", chatID}, {"DELETE FROM chats WHERE id=$1", chatID}, {"DELETE FROM users WHERE id=$1", user.ID}} { + if _, e := pool.Exec(cleanup, q.sql, q.id); e != nil { + t.Error("version fixture cleanup failed") + } + } + }() + chat, e := s.CreateChat(ctx, model.ChatDraft{Title: "Owned version fixture", Creator: user.ID, MemberIDs: []uuid.UUID{user.ID}}) + if e != nil { + t.Fatal(e) + } + chatID = chat.ID + attachmentID = uuid.New() + a, e := s.ReserveAttachment(ctx, model.Attachment{ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: "test.png", ContentType: "image/png", SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}) + if e != nil { + t.Fatal(e) + } + if _, e = s.QueueAttachment(ctx, a, "", ""); e != nil { + t.Fatal(e) + } + jobs, e := s.PendingJobs(ctx) + if e != nil { + t.Fatal(e) + } + var j model.Job + for _, candidate := range jobs { + if candidate.AttachmentID == attachmentID { + j = candidate + } + } + lease := uuid.New() + j, _, ok, e := s.AcquireJob(ctx, j, lease, 30*time.Second) + if e != nil || !ok { + t.Fatal("version1 acquisition failed") + } + if ok, e = s.CompleteJob(ctx, j, lease, []model.Variant{}); e != nil || !ok { + t.Fatal("version1 completion failed") + } + if _, e = s.StagingCleanup(ctx); e != nil { + t.Fatal(e) + } + if e = s.StagingCleaned(ctx, attachmentID); e != nil { + t.Fatal(e) + } + if ok, e = s.AdvancePipeline(ctx, attachmentID); e != nil || !ok { + t.Fatal("version2 advance failed") + } + jobs, e = s.PendingJobs(ctx) + if e != nil { + t.Fatal(e) + } + for _, candidate := range jobs { + if candidate.AttachmentID == attachmentID && candidate.PipelineVersion == 2 { + j = candidate + } + } + lease = uuid.New() + j, _, ok, e = s.AcquireJob(ctx, j, lease, 30*time.Second) + if e != nil || !ok { + t.Fatal("version2 acquisition failed") + } + if e = s.RetryJob(ctx, j, lease, "retry_exhausted", 0, true); e != nil { + t.Fatal(e) + } + ok, e = s.ReplayJob(ctx, attachmentID) + if e != nil { + t.Fatal(e) + } + if !ok { + t.Fatal("ready version1 staging cleanup permanently blocked replay of failed version2 despite retained original") + } + var replays int + var replayed time.Time + if e = pool.QueryRow(ctx, "SELECT replay_count,replayed_at FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=2", attachmentID).Scan(&replays, &replayed); e != nil || replays != 1 || replayed.IsZero() { + t.Fatal("explicit replay has no durable audit count/timestamp") + } +} + +// This deterministic transformer makes v1/v2 effects distinguishable; the Store, +// pipeline ownership/leases, transactions and S3 operations remain real. +type versionTransformer struct { + body string + err error +} + +func (v versionTransformer) Transform(context.Context, io.Reader) ([]port.ProcessedVariant, error) { + if v.err != nil { + return nil, v.err + } + out := []port.ProcessedVariant{} + for _, name := range []string{"thumbnail", "crop", "optimized"} { + body := []byte(v.body + name) + sum := sha256.Sum256(body) + out = append(out, port.ProcessedVariant{Metadata: model.Variant{Name: name, ContentType: "image/jpeg", SHA256: hex.EncodeToString(sum[:]), SizeBytes: int64(len(body)), Width: 1, Height: 1}, Bytes: body}) + } + return out, nil +} +func TestVersionedPipelineDuplicateReorderRetryAndCleanup(t *testing.T) { + if os.Getenv("RUN_PIPELINE_DB_TESTS") != "1" { + t.Skip("explicit isolated database and S3 required") + } + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + raw, e := config.ReadSecret("/run/secrets/database_url") + if e != nil { + t.Fatal("database file unavailable") + } + pool, e := pgxpool.New(ctx, raw) + if e != nil { + t.Fatal("database unavailable") + } + defer pool.Close() + s := New(pool, nil) + cfg, e := storage.ReadConfig("/run/secrets/s3_credentials") + if e != nil { + t.Fatal("S3 config unavailable") + } + objects, e := storage.FromConfig(cfg) + if e != nil { + t.Fatal(e) + } + user, e := s.CreateUser(ctx, model.UserDraft{Username: "matrix_" + uuid.NewString()[:8], DisplayName: "Synthetic fault matrix", PasswordHash: "disabled"}) + if e != nil { + t.Fatal(e) + } + chat, e := s.CreateChat(ctx, model.ChatDraft{Title: "Owned pipeline matrix", Creator: user.ID, MemberIDs: []uuid.UUID{user.ID}}) + if e != nil { + t.Fatal(e) + } + id := uuid.New() + body := []byte("immutable-original") + sum := sha256.Sum256(body) + defer func() { + c, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + for _, key := range []string{"staging/" + id.String(), usecase.ObjectKey(id)} { + if e := objects.Remove(c, key); e != nil { + t.Error("object fixture cleanup failed") + } + } + for version := int32(1); version <= 3; version++ { + for _, name := range []string{"thumbnail", "crop", "optimized"} { + if e := objects.Remove(c, usecase.VariantKey(id, version, name)); e != nil { + t.Error("variant fixture cleanup failed") + } + } + } + for _, q := range []struct { + sql string + id uuid.UUID + }{{"DELETE FROM attachment_jobs WHERE attachment_id=$1", id}, {"DELETE FROM attachments WHERE id=$1", id}, {"DELETE FROM chat_members WHERE chat_id=$1", chat.ID}, {"DELETE FROM chats WHERE id=$1", chat.ID}, {"DELETE FROM users WHERE id=$1", user.ID}} { + if _, e := pool.Exec(c, q.sql, q.id); e != nil { + t.Error("database fixture cleanup failed") + } + } + }() + a, e := s.ReserveAttachment(ctx, model.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "matrix.png", ContentType: "image/png", SizeBytes: int64(len(body)), SHA256: hex.EncodeToString(sum[:])}) + if e != nil { + t.Fatal(e) + } + if e = objects.Put(ctx, "staging/"+id.String(), bytes.NewReader(body), int64(len(body)), "image/png"); e != nil { + t.Fatal(e) + } + if _, e = s.QueueAttachment(ctx, a, "", ""); e != nil { + t.Fatal(e) + } + next := func(version int32) model.Job { + t.Helper() + jobs, e := s.PendingJobs(ctx) + if e != nil { + t.Fatal(e) + } + for _, j := range jobs { + if j.AttachmentID == id && j.PipelineVersion == version { + return j + } + } + t.Fatal("owned current job not found") + return model.Job{} + } + j1 := next(1) + p1 := usecase.NewPipeline(s, objects, versionTransformer{body: "version1-"}, 5) + p1.Fault = func(point string) error { + if point == "after_commit" { + return errors.New("synthetic crash after commit") + } + return nil + } + if e = p1.Process(ctx, j1); e == nil { + t.Fatal("fault did not interrupt after commit") + } + a, e = s.GetAttachment(ctx, id) + if e != nil || a.Status != "ready" || a.StagingCleaned { + t.Fatal("commit fault did not preserve ready/pending cleanup") + } + if e = p1.Process(ctx, j1); e != nil { + t.Fatal(e) + } + if _, e = objects.Get(ctx, "staging/"+id.String()); !errors.Is(e, model.ErrNotFound) { + t.Fatal("completed redelivery did not remove staging") + } + if ok, e := s.AdvancePipeline(ctx, id); e != nil || !ok { + t.Fatal("advance v2 failed") + } + j2 := next(2) + // Old delivery arriving before v2 and a fabricated future delivery are no-ops. + if e = p1.Process(ctx, j1); e != nil { + t.Fatal(e) + } + future := j2 + future.PipelineVersion = 3 + if e = p1.Process(ctx, future); e != nil { + t.Fatal(e) + } + p2 := usecase.NewPipeline(s, objects, versionTransformer{body: "version2-"}, 5) + var group sync.WaitGroup + errs := make(chan error, 2) + for i := 0; i < 2; i++ { + group.Add(1) + go func() { defer group.Done(); errs <- p2.Process(ctx, j2) }() + } + group.Wait() + close(errs) + for e := range errs { + if e != nil { + t.Fatal(e) + } + } + a, e = s.GetAttachment(ctx, id) + if e != nil || a.PipelineVersion != 2 || a.Status != "ready" || len(a.Variants) != 3 { + t.Fatal("duplicate current job lost result") + } + snapshot := a.Variants + for _, j := range []model.Job{j1, j2, j1, future, j2} { + if e = p1.Process(ctx, j); e != nil { + t.Fatal(e) + } + } + a, e = s.GetAttachment(ctx, id) + if e != nil || !reflect.DeepEqual(snapshot, a.Variants) { + t.Fatal("stale or duplicate job overwrote current metadata") + } + for _, v := range a.Variants { + r, e := objects.Get(ctx, usecase.VariantKey(id, 2, v.Name)) + if e != nil { + t.Fatal(e) + } + got, e := io.ReadAll(r) + r.Close() + if e != nil || string(got) != "version2-"+v.Name { + t.Fatal("old transformation overwrote v2 objects") + } + } + if ok, e := s.AdvancePipeline(ctx, id); e != nil || !ok { + t.Fatal("advance v3 failed") + } + j3 := next(3) + broken := usecase.NewPipeline(s, objects, versionTransformer{err: syscall.ENOSPC}, 5) + for i := 0; i < 5; i++ { + if _, e = pool.Exec(ctx, "UPDATE attachment_jobs SET next_attempt='epoch' WHERE id=$1", j3.ID); e != nil { + t.Fatal(e) + } + if e = broken.Process(ctx, j3); e != nil { + t.Fatal(e) + } + } + a, e = s.GetAttachment(ctx, id) + if e != nil || a.Status != "failed" || a.ErrorCode != "retry_exhausted" { + t.Fatal("transient infrastructure failure did not exhaust bounded retry") + } + if ok, e := s.ReplayJob(ctx, id); e != nil || !ok { + t.Fatal("explicit replay blocked despite retained immutable source") + } + if e = p2.Process(ctx, j3); e != nil { + t.Fatal(e) + } + a, e = s.GetAttachment(ctx, id) + if e != nil || a.Status != "ready" || a.PipelineVersion != 3 { + t.Fatal("replayed job did not recover") + } +} diff --git a/backend/internal/adapter/repository/postgres/queries/attachments.sql.go b/backend/internal/adapter/repository/postgres/queries/attachments.sql.go new file mode 100644 index 00000000..525c698b --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/attachments.sql.go @@ -0,0 +1,193 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachments.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const chargeAttachment = `-- name: ChargeAttachment :one +UPDATE users SET attachment_count=attachment_count+1 WHERE id=$1 AND attachment_count<10000 RETURNING id +` + +func (q *Queries) ChargeAttachment(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, chargeAttachment, id) + var id_2 uuid.UUID + err := row.Scan(&id_2) + return id_2, err +} + +const getAttachment = `-- name: GetAttachment :one +SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at FROM attachments WHERE id=$1 FOR SHARE +` + +func (q *Queries) GetAttachment(ctx context.Context, id uuid.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachment, id) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.PipelineVersion, + &i.ErrorCode, + &i.Variants, + &i.StagingExpired, + &i.StagingCleaned, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const insertAttachment = `-- name: InsertAttachment :one +INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status) +VALUES($1,$2,$3,$4,$5,$6,$7,'uploading') RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at +` + +type InsertAttachmentParams struct { + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + Filename string + ContentType string + SizeBytes int64 + Sha256 string +} + +func (q *Queries) InsertAttachment(ctx context.Context, arg InsertAttachmentParams) (Attachment, error) { + row := q.db.QueryRow(ctx, insertAttachment, + arg.ID, + arg.ChatID, + arg.OwnerID, + arg.Filename, + arg.ContentType, + arg.SizeBytes, + arg.Sha256, + ) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.PipelineVersion, + &i.ErrorCode, + &i.Variants, + &i.StagingExpired, + &i.StagingCleaned, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const insertMessageAttachment = `-- name: InsertMessageAttachment :exec +INSERT INTO message_attachments(message_id,attachment_id,ordinal) VALUES($1,$2,$3) +` + +type InsertMessageAttachmentParams struct { + MessageID uuid.UUID + AttachmentID uuid.UUID + Ordinal int32 +} + +func (q *Queries) InsertMessageAttachment(ctx context.Context, arg InsertMessageAttachmentParams) error { + _, err := q.db.Exec(ctx, insertMessageAttachment, arg.MessageID, arg.AttachmentID, arg.Ordinal) + return err +} + +const messageAttachments = `-- name: MessageAttachments :many +SELECT attachment_id FROM message_attachments WHERE message_id=$1 ORDER BY ordinal LIMIT 10 +` + +func (q *Queries) MessageAttachments(ctx context.Context, messageID uuid.UUID) ([]uuid.UUID, error) { + rows, err := q.db.Query(ctx, messageAttachments, messageID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []uuid.UUID{} + for rows.Next() { + var attachment_id uuid.UUID + if err := rows.Scan(&attachment_id); err != nil { + return nil, err + } + items = append(items, attachment_id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const reconcileAttachments = `-- name: ReconcileAttachments :many +SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at FROM attachments WHERE id>$1 ORDER BY id LIMIT 101 +` + +func (q *Queries) ReconcileAttachments(ctx context.Context, id uuid.UUID) ([]Attachment, error) { + rows, err := q.db.Query(ctx, reconcileAttachments, id) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Attachment{} + for rows.Next() { + var i Attachment + if err := rows.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.PipelineVersion, + &i.ErrorCode, + &i.Variants, + &i.StagingExpired, + &i.StagingCleaned, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setAttachmentStatus = `-- name: SetAttachmentStatus :execrows +UPDATE attachments SET status=$2,updated_at=clock_timestamp() WHERE id=$1 AND status=$3 AND pipeline_version=0 +` + +type SetAttachmentStatusParams struct { + ID uuid.UUID + Status string + Status_2 string +} + +func (q *Queries) SetAttachmentStatus(ctx context.Context, arg SetAttachmentStatusParams) (int64, error) { + result, err := q.db.Exec(ctx, setAttachmentStatus, arg.ID, arg.Status, arg.Status_2) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/backend/internal/adapter/repository/postgres/queries/core.sql.go b/backend/internal/adapter/repository/postgres/queries/core.sql.go new file mode 100644 index 00000000..ba4a5cba --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/core.sql.go @@ -0,0 +1,358 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: core.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const authenticateSession = `-- name: AuthenticateSession :one +SELECT user_id FROM sessions WHERE token_hash=$1 AND expires_at>clock_timestamp() +` + +func (q *Queries) AuthenticateSession(ctx context.Context, tokenHash string) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, authenticateSession, tokenHash) + var user_id uuid.UUID + err := row.Scan(&user_id) + return user_id, err +} + +const chargeChat = `-- name: ChargeChat :one +UPDATE users SET chat_count=chat_count+1 WHERE id=$1 AND chat_count<1000 RETURNING id +` + +func (q *Queries) ChargeChat(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, chargeChat, id) + var id_2 uuid.UUID + err := row.Scan(&id_2) + return id_2, err +} + +const chargeMessage = `-- name: ChargeMessage :one +UPDATE users SET message_count=message_count+1 WHERE id=$1 AND message_count<1000000 RETURNING id +` + +func (q *Queries) ChargeMessage(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, chargeMessage, id) + var id_2 uuid.UUID + err := row.Scan(&id_2) + return id_2, err +} + +const createUser = `-- name: CreateUser :one +INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4) +RETURNING id,username,display_name,created_at +` + +type CreateUserParams struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string +} + +type CreateUserRow struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (CreateUserRow, error) { + row := q.db.QueryRow(ctx, createUser, + arg.ID, + arg.Username, + arg.DisplayName, + arg.PasswordHash, + ) + var i CreateUserRow + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.CreatedAt, + ) + return i, err +} + +const credentials = `-- name: Credentials :one +SELECT id,password_hash FROM users WHERE username=$1 +` + +type CredentialsRow struct { + ID uuid.UUID + PasswordHash string +} + +func (q *Queries) Credentials(ctx context.Context, username string) (CredentialsRow, error) { + row := q.db.QueryRow(ctx, credentials, username) + var i CredentialsRow + err := row.Scan(&i.ID, &i.PasswordHash) + return i, err +} + +const deleteExpiredSessions = `-- name: DeleteExpiredSessions :exec +DELETE FROM sessions WHERE user_id=$1 AND expires_at<=clock_timestamp() +` + +func (q *Queries) DeleteExpiredSessions(ctx context.Context, userID uuid.UUID) error { + _, err := q.db.Exec(ctx, deleteExpiredSessions, userID) + return err +} + +const getChat = `-- name: GetChat :one +SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members +FROM chats c WHERE c.id=$1 +` + +type GetChatRow struct { + ID uuid.UUID + Title string + CreatedAt time.Time + Members []uuid.UUID +} + +func (q *Queries) GetChat(ctx context.Context, id uuid.UUID) (GetChatRow, error) { + row := q.db.QueryRow(ctx, getChat, id) + var i GetChatRow + err := row.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.Members, + ) + return i, err +} + +const getUser = `-- name: GetUser :one +SELECT id,username,display_name,created_at FROM users WHERE id=$1 +` + +type GetUserRow struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +func (q *Queries) GetUser(ctx context.Context, id uuid.UUID) (GetUserRow, error) { + row := q.db.QueryRow(ctx, getUser, id) + var i GetUserRow + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.CreatedAt, + ) + return i, err +} + +const insertChat = `-- name: InsertChat :one +INSERT INTO chats(id,title) VALUES($1,$2) RETURNING id,title,created_at +` + +type InsertChatParams struct { + ID uuid.UUID + Title string +} + +func (q *Queries) InsertChat(ctx context.Context, arg InsertChatParams) (Chat, error) { + row := q.db.QueryRow(ctx, insertChat, arg.ID, arg.Title) + var i Chat + err := row.Scan(&i.ID, &i.Title, &i.CreatedAt) + return i, err +} + +const insertMembers = `-- name: InsertMembers :exec +INSERT INTO chat_members(chat_id,user_id) SELECT $1,unnest($2::uuid[]) +` + +type InsertMembersParams struct { + ChatID uuid.UUID + Column2 []uuid.UUID +} + +func (q *Queries) InsertMembers(ctx context.Context, arg InsertMembersParams) error { + _, err := q.db.Exec(ctx, insertMembers, arg.ChatID, arg.Column2) + return err +} + +const insertMessage = `-- name: InsertMessage :one +INSERT INTO messages(id,chat_id,sender_id,text) VALUES($1,$2,$3,$4) RETURNING id,chat_id,sender_id,text,created_at +` + +type InsertMessageParams struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string +} + +func (q *Queries) InsertMessage(ctx context.Context, arg InsertMessageParams) (Message, error) { + row := q.db.QueryRow(ctx, insertMessage, + arg.ID, + arg.ChatID, + arg.SenderID, + arg.Text, + ) + var i Message + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ) + return i, err +} + +const insertSession = `-- name: InsertSession :one +INSERT INTO sessions(token_hash,user_id,expires_at) +SELECT $1,$2,$3 WHERE (SELECT count(*) FROM sessions WHERE user_id=$2)<100 +RETURNING user_id +` + +type InsertSessionParams struct { + TokenHash string + UserID uuid.UUID + ExpiresAt time.Time +} + +func (q *Queries) InsertSession(ctx context.Context, arg InsertSessionParams) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, insertSession, arg.TokenHash, arg.UserID, arg.ExpiresAt) + var user_id uuid.UUID + err := row.Scan(&user_id) + return user_id, err +} + +const listChats = `-- name: ListChats :many +SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members +FROM chats c JOIN chat_members m ON m.chat_id=c.id +WHERE m.user_id=$1 AND (c.created_at,c.id)>($2::timestamptz,$3::uuid) +ORDER BY c.created_at,c.id LIMIT $4 +` + +type ListChatsParams struct { + MemberID uuid.UUID + AfterTime time.Time + AfterID uuid.UUID + PageLimit int32 +} + +type ListChatsRow struct { + ID uuid.UUID + Title string + CreatedAt time.Time + Members []uuid.UUID +} + +func (q *Queries) ListChats(ctx context.Context, arg ListChatsParams) ([]ListChatsRow, error) { + rows, err := q.db.Query(ctx, listChats, + arg.MemberID, + arg.AfterTime, + arg.AfterID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListChatsRow{} + for rows.Next() { + var i ListChatsRow + if err := rows.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.Members, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMessages = `-- name: ListMessages :many +SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at, +ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids +FROM messages m WHERE m.chat_id=$1 AND (m.created_at,m.id)>($2::timestamptz,$3::uuid) +ORDER BY m.created_at,m.id LIMIT $4 +` + +type ListMessagesParams struct { + ChatID uuid.UUID + AfterTime time.Time + AfterID uuid.UUID + PageLimit int32 +} + +type ListMessagesRow struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + AttachmentIds []uuid.UUID +} + +func (q *Queries) ListMessages(ctx context.Context, arg ListMessagesParams) ([]ListMessagesRow, error) { + rows, err := q.db.Query(ctx, listMessages, + arg.ChatID, + arg.AfterTime, + arg.AfterID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListMessagesRow{} + for rows.Next() { + var i ListMessagesRow + if err := rows.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + &i.AttachmentIds, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockSessionOwner = `-- name: LockSessionOwner :one +SELECT id FROM users WHERE id=$1 FOR UPDATE +` + +func (q *Queries) LockSessionOwner(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, lockSessionOwner, id) + var id_2 uuid.UUID + err := row.Scan(&id_2) + return id_2, err +} + +const revokeSession = `-- name: RevokeSession :exec +DELETE FROM sessions WHERE token_hash=$1 +` + +func (q *Queries) RevokeSession(ctx context.Context, tokenHash string) error { + _, err := q.db.Exec(ctx, revokeSession, tokenHash) + return err +} diff --git a/backend/internal/adapter/repository/postgres/queries/db.go b/backend/internal/adapter/repository/postgres/queries/db.go new file mode 100644 index 00000000..c69f0c53 --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package queries + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/backend/internal/adapter/repository/postgres/queries/idempotency.sql.go b/backend/internal/adapter/repository/postgres/queries/idempotency.sql.go new file mode 100644 index 00000000..e950686d --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/idempotency.sql.go @@ -0,0 +1,81 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: idempotency.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const replayMessage = `-- name: ReplayMessage :one +SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,i.payload_hash, +ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids +FROM idempotency i JOIN messages m ON m.id=i.message_id +WHERE i.user_id=$1 AND i.chat_id=$2 AND i.key=$3 +` + +type ReplayMessageParams struct { + UserID uuid.UUID + ChatID uuid.UUID + Key string +} + +type ReplayMessageRow struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + PayloadHash string + AttachmentIds []uuid.UUID +} + +func (q *Queries) ReplayMessage(ctx context.Context, arg ReplayMessageParams) (ReplayMessageRow, error) { + row := q.db.QueryRow(ctx, replayMessage, arg.UserID, arg.ChatID, arg.Key) + var i ReplayMessageRow + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + &i.PayloadHash, + &i.AttachmentIds, + ) + return i, err +} + +const reserveKey = `-- name: ReserveKey :one +INSERT INTO idempotency(user_id,chat_id,key,payload_hash,message_id,expires_at) +VALUES($1,$2,$3,$4,$5,clock_timestamp()+interval '24 hours') +ON CONFLICT(user_id,chat_id,key) DO UPDATE SET payload_hash=excluded.payload_hash, +message_id=excluded.message_id,expires_at=excluded.expires_at +WHERE idempotency.expires_at<=clock_timestamp() +RETURNING message_id +` + +type ReserveKeyParams struct { + UserID uuid.UUID + ChatID uuid.UUID + Key string + PayloadHash string + MessageID uuid.UUID +} + +func (q *Queries) ReserveKey(ctx context.Context, arg ReserveKeyParams) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, reserveKey, + arg.UserID, + arg.ChatID, + arg.Key, + arg.PayloadHash, + arg.MessageID, + ) + var message_id uuid.UUID + err := row.Scan(&message_id) + return message_id, err +} diff --git a/backend/internal/adapter/repository/postgres/queries/models.go b/backend/internal/adapter/repository/postgres/queries/models.go new file mode 100644 index 00000000..4e2bada7 --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/models.go @@ -0,0 +1,97 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package queries + +import ( + "time" + + "github.com/google/uuid" +) + +type Attachment struct { + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + Filename string + ContentType string + SizeBytes int64 + Sha256 string + Status string + PipelineVersion int32 + ErrorCode string + Variants []byte + StagingExpired bool + StagingCleaned bool + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentJob struct { + ID uuid.UUID + AttachmentID uuid.UUID + PipelineVersion int32 + Attempt int32 + Traceparent string + Tracestate string + LeaseID uuid.UUID + LeaseUntil time.Time + NextAttempt time.Time + PublishedAt time.Time + Completed bool + CreatedAt time.Time + ReplayCount int64 + ReplayedAt time.Time +} + +type Chat struct { + ID uuid.UUID + Title string + CreatedAt time.Time +} + +type ChatMember struct { + ChatID uuid.UUID + UserID uuid.UUID +} + +type Idempotency struct { + UserID uuid.UUID + ChatID uuid.UUID + Key string + PayloadHash string + MessageID uuid.UUID + ExpiresAt time.Time +} + +type Message struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +type MessageAttachment struct { + MessageID uuid.UUID + AttachmentID uuid.UUID + Ordinal int32 +} + +type Session struct { + TokenHash string + UserID uuid.UUID + ExpiresAt time.Time +} + +type User struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time + ChatCount int32 + MessageCount int32 + AttachmentCount int32 +} diff --git a/backend/internal/adapter/repository/postgres/queries/pipeline.sql.go b/backend/internal/adapter/repository/postgres/queries/pipeline.sql.go new file mode 100644 index 00000000..cd6ff87f --- /dev/null +++ b/backend/internal/adapter/repository/postgres/queries/pipeline.sql.go @@ -0,0 +1,393 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: pipeline.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const acquireJob = `-- name: AcquireJob :one +UPDATE attachment_jobs j SET attempt=j.attempt+1,lease_id=$1,lease_until=clock_timestamp()+$2::int*interval '1 second' +WHERE j.id=$3 AND j.pipeline_version=$4 AND NOT j.completed + AND j.next_attempt<=clock_timestamp() AND j.lease_untilclock_timestamp() AND NOT j.completed AND a.status='processing' +` + +type CompleteAttachmentParams struct { + Variants []byte + JobID uuid.UUID + LeaseID uuid.UUID +} + +func (q *Queries) CompleteAttachment(ctx context.Context, arg CompleteAttachmentParams) (int64, error) { + result, err := q.db.Exec(ctx, completeAttachment, arg.Variants, arg.JobID, arg.LeaseID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const completeJob = `-- name: CompleteJob :exec +UPDATE attachment_jobs SET completed=true,lease_until='epoch' WHERE id=$1 AND lease_id=$2 +` + +type CompleteJobParams struct { + JobID uuid.UUID + LeaseID uuid.UUID +} + +func (q *Queries) CompleteJob(ctx context.Context, arg CompleteJobParams) error { + _, err := q.db.Exec(ctx, completeJob, arg.JobID, arg.LeaseID) + return err +} + +const insertJob = `-- name: InsertJob :exec +INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,traceparent,tracestate) +VALUES($1,$2,$3,$4,$5) +` + +type InsertJobParams struct { + JobID uuid.UUID + AttachmentID uuid.UUID + PipelineVersion int32 + Traceparent string + Tracestate string +} + +func (q *Queries) InsertJob(ctx context.Context, arg InsertJobParams) error { + _, err := q.db.Exec(ctx, insertJob, + arg.JobID, + arg.AttachmentID, + arg.PipelineVersion, + arg.Traceparent, + arg.Tracestate, + ) + return err +} + +const pendingJobs = `-- name: PendingJobs :many +SELECT j.id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent, j.tracestate, j.lease_id, j.lease_until, j.next_attempt, j.published_at, j.completed, j.created_at, j.replay_count, j.replayed_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version +WHERE NOT j.completed AND a.status IN ('queued','processing') AND j.next_attempt<=clock_timestamp() + AND j.lease_untilclock_timestamp() AND NOT j.completed AND a.status='processing' +` + +type RetryAttachmentParams struct { + NextStatus string + ErrorCode string + JobID uuid.UUID + LeaseID uuid.UUID +} + +func (q *Queries) RetryAttachment(ctx context.Context, arg RetryAttachmentParams) (int64, error) { + result, err := q.db.Exec(ctx, retryAttachment, + arg.NextStatus, + arg.ErrorCode, + arg.JobID, + arg.LeaseID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const retryJob = `-- name: RetryJob :exec +UPDATE attachment_jobs SET lease_until='epoch',next_attempt=clock_timestamp()+$1::int*interval '1 millisecond',completed=$2 +WHERE id=$3 AND lease_id=$4 +` + +type RetryJobParams struct { + DelayMs int32 + Completed bool + JobID uuid.UUID + LeaseID uuid.UUID +} + +func (q *Queries) RetryJob(ctx context.Context, arg RetryJobParams) error { + _, err := q.db.Exec(ctx, retryJob, + arg.DelayMs, + arg.Completed, + arg.JobID, + arg.LeaseID, + ) + return err +} + +const stagingCleaned = `-- name: StagingCleaned :exec +UPDATE attachments SET staging_cleaned=true WHERE id=$1 AND (staging_expired OR status='ready') +` + +func (q *Queries) StagingCleaned(ctx context.Context, id uuid.UUID) error { + _, err := q.db.Exec(ctx, stagingCleaned, id) + return err +} + +const stagingCleanup = `-- name: StagingCleanup :many +WITH candidates AS ( + SELECT id FROM attachments WHERE NOT staging_cleaned AND (status='ready' OR (status IN ('failed','uploading') AND updated_at'ready') FROM candidates c WHERE a.id=c.id AND NOT a.staging_cleaned +AND (a.status='ready' OR (a.status IN ('failed','uploading') AND a.updated_at p.Limit + if more { + rows = rows[:p.Limit] + } + out := make([]model.Chat, len(rows)) + for i, r := range rows { + out[i] = model.Chat{ID: r.ID, Title: r.Title, CreatedAt: r.CreatedAt, MemberIDs: r.Members} + } + return out, more, nil +} +func (s *Store) CreateMessage(ctx context.Context, d model.MessageDraft) (model.Message, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return model.Message{}, mapErr(err) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + messageID := uuid.New() + reserved, e := q.ReserveKey(ctx, queries.ReserveKeyParams{UserID: d.SenderID, ChatID: d.ChatID, Key: d.Key, PayloadHash: d.PayloadHash, MessageID: messageID}) + if errors.Is(e, pgx.ErrNoRows) { + r, e := q.ReplayMessage(ctx, queries.ReplayMessageParams{UserID: d.SenderID, ChatID: d.ChatID, Key: d.Key}) + if e != nil { + return model.Message{}, mapErr(e) + } + if r.PayloadHash != d.PayloadHash { + return model.Message{}, model.ErrConflict + } + return model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: r.AttachmentIds}, nil + } + if e != nil { + return model.Message{}, mapErr(e) + } + messageID = reserved + for _, id := range d.AttachmentIDs { + a, e := q.GetAttachment(ctx, id) + if errors.Is(e, pgx.ErrNoRows) { + return model.Message{}, model.ErrNotFound + } + if e != nil { + return model.Message{}, mapErr(e) + } + if a.ChatID != d.ChatID { + return model.Message{}, model.ErrNotFound + } + if a.Status != "ready" && a.Status != "queued" && a.Status != "processing" { + return model.Message{}, model.ErrConflict + } + } + if _, err = q.ChargeMessage(ctx, d.SenderID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, model.ErrStorageFull + } + return model.Message{}, mapErr(err) + } + r, err := q.InsertMessage(ctx, queries.InsertMessageParams{ID: messageID, ChatID: d.ChatID, SenderID: d.SenderID, Text: d.Text}) + if err != nil { + return model.Message{}, mapErr(err) + } + for i, id := range d.AttachmentIDs { + if e := q.InsertMessageAttachment(ctx, queries.InsertMessageAttachmentParams{MessageID: messageID, AttachmentID: id, Ordinal: int32(i)}); e != nil { + return model.Message{}, mapErr(e) + } + } + if err = tx.Commit(ctx); err != nil { + return model.Message{}, mapErr(err) + } + return model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: d.AttachmentIDs}, nil +} +func (s *Store) ListMessages(ctx context.Context, id uuid.UUID, p model.PageQuery) ([]model.Message, bool, error) { + at, aid := position(p) + rows, err := s.q.ListMessages(ctx, queries.ListMessagesParams{ChatID: id, AfterTime: at, AfterID: aid, PageLimit: int32(p.Limit + 1)}) + if err != nil { + return nil, false, mapErr(err) + } + more := len(rows) > p.Limit + if more { + rows = rows[:p.Limit] + } + out := make([]model.Message, len(rows)) + for i, r := range rows { + out[i] = model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: r.AttachmentIds} + } + return out, more, nil +} +func (s *Store) Credentials(ctx context.Context, name string) (uuid.UUID, string, error) { + r, e := s.q.Credentials(ctx, name) + if errors.Is(e, pgx.ErrNoRows) { + return uuid.Nil, "", model.ErrUnauthorized + } + return r.ID, r.PasswordHash, mapErr(e) +} +func (s *Store) CreateSession(ctx context.Context, hash string, id uuid.UUID, expiry time.Time) error { + tx, e := s.pool.Begin(ctx) + if e != nil { + return mapErr(e) + } + defer func() { _ = tx.Rollback(ctx) }() + q := s.q.WithTx(tx) + if _, e = q.LockSessionOwner(ctx, id); e != nil { + return mapErr(e) + } + if e = q.DeleteExpiredSessions(ctx, id); e != nil { + return mapErr(e) + } + if _, e = q.InsertSession(ctx, queries.InsertSessionParams{TokenHash: hash, UserID: id, ExpiresAt: expiry}); e != nil { + if errors.Is(e, pgx.ErrNoRows) { + return model.ErrStorageFull + } + return mapErr(e) + } + return mapErr(tx.Commit(ctx)) +} +func (s *Store) AuthenticateSession(ctx context.Context, hash string) (uuid.UUID, error) { + r, e := s.q.AuthenticateSession(ctx, hash) + if errors.Is(e, pgx.ErrNoRows) { + return uuid.Nil, model.ErrUnauthorized + } + return r, mapErr(e) +} +func (s *Store) RevokeSession(ctx context.Context, hash string) error { + return mapErr(s.q.RevokeSession(ctx, hash)) +} diff --git a/backend/internal/adapter/storage/bootstrap.go b/backend/internal/adapter/storage/bootstrap.go new file mode 100644 index 00000000..b238f991 --- /dev/null +++ b/backend/internal/adapter/storage/bootstrap.go @@ -0,0 +1,86 @@ +package storage + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/signer" + "io" + "net/http" + "net/url" + "os" + "time" +) + +type Config struct{ Endpoint, Access, Secret, Bucket string } + +func ReadConfig(path string) (Config, error) { + var c Config + b, e := os.ReadFile(path) + if e != nil { + return c, fmt.Errorf("S3 credentials unavailable") + } + if e = json.Unmarshal(b, &c); e != nil { + return c, fmt.Errorf("invalid S3 credentials format") + } + if c.Endpoint == "" || c.Access == "" || c.Secret == "" || c.Bucket == "" { + return c, fmt.Errorf("incomplete S3 configuration") + } + return c, nil +} +func FromConfig(c Config) (*S3, error) { return New(c.Endpoint, c.Access, c.Secret, c.Bucket) } + +// RustFS provides S3 SDKs, but its documented administrative API is separate. +// Use the SDK's SigV4 signer; do not implement signing or persist credentials in command arguments. +func admin(ctx context.Context, c Config, path string, body []byte) error { + r, e := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+c.Endpoint+"/rustfs/admin/v3/"+path, bytes.NewReader(body)) + if e != nil { + return e + } + sum := sha256.Sum256(body) + r.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:])) + r.Header.Set("Content-Type", "application/json") + signed := signer.SignV4(*r, c.Access, c.Secret, "", "us-east-1") + response, e := (&http.Client{Timeout: 5 * time.Second}).Do(signed) + if e != nil { + return fmt.Errorf("S3 admin transport failed") + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64*1024)) + if response.StatusCode != 200 && response.StatusCode != 204 { + return fmt.Errorf("S3 admin operation rejected: HTTP %d", response.StatusCode) + } + return nil +} +func Bootstrap(ctx context.Context, root, app Config) error { + s, e := FromConfig(root) + if e != nil { + return e + } + exists, e := s.Client.BucketExists(ctx, root.Bucket) + if e != nil { + return fmt.Errorf("bucket availability check failed") + } + if !exists { + if e = s.Client.MakeBucket(ctx, root.Bucket, minio.MakeBucketOptions{Region: "us-east-1"}); e != nil { + return fmt.Errorf("bucket creation failed") + } + } + policy := map[string]any{"Version": "2012-10-17", "Statement": []any{ + map[string]any{"Effect": "Allow", "Action": []string{"s3:GetBucketLocation", "s3:ListBucket"}, "Resource": []string{"arn:aws:s3:::" + app.Bucket}}, + map[string]any{"Effect": "Allow", "Action": []string{"s3:PutObject", "s3:GetObject", "s3:DeleteObject"}, "Resource": []string{"arn:aws:s3:::" + app.Bucket + "/originals/*", "arn:aws:s3:::" + app.Bucket + "/staging/*", "arn:aws:s3:::" + app.Bucket + "/variants/*"}}, + }} + raw, _ := json.Marshal(policy) + if e = admin(ctx, root, "add-canned-policy?name=messenger", raw); e != nil { + return e + } + raw, _ = json.Marshal(map[string]string{"secretKey": app.Secret, "status": "enabled"}) + if e = admin(ctx, root, "add-user?accessKey="+url.QueryEscape(app.Access), raw); e != nil { + return e + } + return admin(ctx, root, "set-user-or-group-policy?policyName=messenger&userOrGroup="+url.QueryEscape(app.Access)+"&isGroup=false", nil) +} diff --git a/backend/internal/adapter/storage/readiness_test.go b/backend/internal/adapter/storage/readiness_test.go new file mode 100644 index 00000000..4617deca --- /dev/null +++ b/backend/internal/adapter/storage/readiness_test.go @@ -0,0 +1,21 @@ +package storage + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestMissingBucketFailsReadiness(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) })) + defer server.Close() + store, err := New(strings.TrimPrefix(server.URL, "http://"), "synthetic-access", "synthetic-secret", "missing") + if err != nil { + t.Fatal(err) + } + if err = store.Ready(context.Background()); err == nil { + t.Fatal("missing bucket reported ready") + } +} diff --git a/backend/internal/adapter/storage/s3.go b/backend/internal/adapter/storage/s3.go new file mode 100644 index 00000000..22dc2a9c --- /dev/null +++ b/backend/internal/adapter/storage/s3.go @@ -0,0 +1,81 @@ +package storage + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "context" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "io" + "net" + "net/http" + "time" +) + +type S3 struct { + Client *minio.Client + Bucket string +} + +func New(endpoint, access, secret, bucket string) (*S3, error) { + client, err := minio.New(endpoint, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: false, Region: "us-east-1", Transport: &http.Transport{DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, ResponseHeaderTimeout: 3 * time.Second, MaxIdleConnsPerHost: 8}}) + if err != nil { + return nil, err + } + return &S3{client, bucket}, nil +} +func mapError(err error) error { + if err == nil { + return nil + } + r := minio.ToErrorResponse(err) + if r.Code == "NoSuchKey" || r.Code == "NoSuchObject" || r.Code == "NotFound" { + return model.ErrNotFound + } + return model.ErrUnavailable +} +func (s *S3) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error { + _, err := s.Client.PutObject(ctx, s.Bucket, key, r, size, minio.PutObjectOptions{ContentType: mime, DisableMultipart: true}) + return mapError(err) +} +func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) { + obj, err := s.Client.GetObject(ctx, s.Bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, mapError(err) + } + if _, err = obj.Stat(); err != nil { + _ = obj.Close() + return nil, mapError(err) + } + return obj, nil +} +func (s *S3) Stat(ctx context.Context, key string) (port.Object, error) { + r, e := s.Client.StatObject(ctx, s.Bucket, key, minio.StatObjectOptions{}) + return port.Object{Key: key, ETag: r.ETag, Size: r.Size, Modified: r.LastModified.UTC().Truncate(time.Second)}, mapError(e) +} +func (s *S3) Remove(ctx context.Context, key string) error { + return mapError(s.Client.RemoveObject(ctx, s.Bucket, key, minio.RemoveObjectOptions{})) +} +func (s *S3) List(ctx context.Context, after string) ([]port.Object, bool, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + out := []port.Object{} + for r := range s.Client.ListObjects(ctx, s.Bucket, minio.ListObjectsOptions{Prefix: "originals/", StartAfter: after, Recursive: true, MaxKeys: 100}) { + if r.Err != nil { + return nil, false, mapError(r.Err) + } + // HTTP Last-Modified has second precision; RustFS inventory also carries milliseconds. + out = append(out, port.Object{Key: r.Key, ETag: r.ETag, Size: r.Size, Modified: r.LastModified.UTC().Truncate(time.Second)}) + if len(out) > 100 { + return out[:100], true, nil + } + } + return out, false, nil +} +func (s *S3) Ready(ctx context.Context) error { + exists, err := s.Client.BucketExists(ctx, s.Bucket) + if err != nil || !exists { + return model.ErrUnavailable + } + return nil +} diff --git a/backend/internal/application/usecase/attachments.go b/backend/internal/application/usecase/attachments.go new file mode 100644 index 00000000..8c1a679f --- /dev/null +++ b/backend/internal/application/usecase/attachments.go @@ -0,0 +1,79 @@ +package usecase + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "backend/internal/domain/repository" + "context" + "github.com/google/uuid" + "io" +) + +type Attachments struct { + repo repository.AsyncAttachments + messenger *Messenger + objects port.Objects +} + +func NewAttachments(r repository.AsyncAttachments, m *Messenger, o port.Objects) *Attachments { + return &Attachments{r, m, o} +} +func ObjectKey(id uuid.UUID) string { return "originals/" + id.String() } +func (a *Attachments) Upload(ctx context.Context, caller, chat uuid.UUID, name, mime, sha string, size int64, body io.Reader) (model.Attachment, error) { + if _, e := a.messenger.GetChat(ctx, caller, chat); e != nil { + return model.Attachment{}, e + } + row, e := a.repo.ReserveAttachment(ctx, model.Attachment{ID: uuid.New(), ChatID: chat, OwnerID: caller, Filename: name, ContentType: mime, SHA256: sha, SizeBytes: size}) + if e != nil { + return model.Attachment{}, e + } + if e = a.objects.Put(ctx, "staging/"+row.ID.String(), body, size, mime); e != nil { + return model.Attachment{}, model.ErrUnavailable + } + return a.repo.QueueAttachment(ctx, row, "", "") +} +func (a *Attachments) Get(ctx context.Context, caller, id uuid.UUID) (model.Attachment, error) { + row, e := a.repo.GetAttachment(ctx, id) + if e != nil { + return model.Attachment{}, e + } + if _, e = a.messenger.GetChat(ctx, caller, row.ChatID); e != nil { + return model.Attachment{}, e + } + return row, nil +} +func (a *Attachments) Content(ctx context.Context, caller, id uuid.UUID, variant string) (model.Attachment, io.ReadCloser, error) { + row, e := a.Get(ctx, caller, id) + if e != nil { + return row, nil, e + } + if row.Status != "ready" { + return row, nil, model.ErrConflict + } + key := ObjectKey(id) + if variant != "original" { + found := false + for _, v := range row.Variants { + if v.Name == variant { + found = true + key = VariantKey(id, row.PipelineVersion, variant) + row.SizeBytes = v.SizeBytes + row.ContentType = v.ContentType + row.SHA256 = v.SHA256 + break + } + } + if !found { + return row, nil, model.ErrNotFound + } + } + info, e := a.objects.Stat(ctx, key) + if e != nil || info.Size != row.SizeBytes { + return row, nil, model.ErrUnavailable + } + r, e := a.objects.Get(ctx, key) + if e != nil { + return row, nil, model.ErrUnavailable + } + return row, r, nil +} diff --git a/backend/internal/application/usecase/auth.go b/backend/internal/application/usecase/auth.go new file mode 100644 index 00000000..ea133922 --- /dev/null +++ b/backend/internal/application/usecase/auth.go @@ -0,0 +1,86 @@ +package usecase + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "time" + "unicode/utf8" + + "backend/internal/domain/model" + "backend/internal/domain/port" +) + +type Auth struct { + sessions port.Sessions + passwords port.Passwords + messenger *Messenger + ttl time.Duration + now func() time.Time +} +type Session struct { + Token string + ExpiresAt time.Time + User model.User +} + +func NewAuth(s port.Sessions, p port.Passwords, m *Messenger, ttl time.Duration, now func() time.Time) *Auth { + return &Auth{s, p, m, ttl, now} +} +func (a *Auth) Register(ctx context.Context, username, name, password string) (model.User, error) { + if n := utf8.RuneCountInString(password); n < 12 || n > 128 || !utf8.ValidString(password) { + return model.User{}, &model.ValidationError{Message: "password: 12–128 Unicode characters"} + } + draft, err := model.NewUserDraft(username, name) + if err != nil { + return model.User{}, err + } + hash, err := a.passwords.Hash(password) + if err != nil { + return model.User{}, err + } + draft.PasswordHash = hash + return a.messenger.repo.CreateUser(ctx, draft) +} +func (a *Auth) Login(ctx context.Context, username, password string) (Session, error) { + id, hash, err := a.sessions.Credentials(ctx, username) + if err != nil { + return Session{}, err + } + if !a.passwords.Verify(password, hash) { + return Session{}, model.ErrUnauthorized + } + user, err := a.messenger.repo.GetUser(ctx, id) + if err != nil { + return Session{}, err + } + raw := make([]byte, 32) + if _, err = rand.Read(raw); err != nil { + return Session{}, err + } + token := base64.RawURLEncoding.EncodeToString(raw) + expires := a.now().UTC().Add(a.ttl) + if err = a.sessions.CreateSession(ctx, TokenHash(token), id, expires); err != nil { + return Session{}, err + } + return Session{token, expires, user}, nil +} +func TokenHash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} +func (a *Auth) Authenticate(ctx context.Context, token string) (model.User, error) { + if len(token) != 43 { + return model.User{}, model.ErrUnauthorized + } + id, err := a.sessions.AuthenticateSession(ctx, TokenHash(token)) + if err != nil { + return model.User{}, err + } + return a.messenger.repo.GetUser(ctx, id) +} +func (a *Auth) Logout(ctx context.Context, token string) error { + return a.sessions.RevokeSession(ctx, TokenHash(token)) +} diff --git a/backend/internal/application/usecase/messenger.go b/backend/internal/application/usecase/messenger.go new file mode 100644 index 00000000..0ac702bb --- /dev/null +++ b/backend/internal/application/usecase/messenger.go @@ -0,0 +1,102 @@ +package usecase + +import ( + "context" + + "backend/internal/domain/model" + "backend/internal/domain/repository" + + "github.com/google/uuid" +) + +const chatsCursorScope = "chats" + +// Page — страница списка; NextCursor == nil, если следующей записи нет. +type Page[T any] struct { + Items []T + NextCursor *string +} + +type Messenger struct { + repo repository.Repository +} + +func NewMessenger(repo repository.Repository) *Messenger { + return &Messenger{repo: repo} +} + +func (m *Messenger) CreateChat(ctx context.Context, caller uuid.UUID, title string, memberIDs []uuid.UUID) (model.Chat, error) { + draft, err := model.NewChatDraft(caller, title, memberIDs) + if err != nil { + return model.Chat{}, err + } + return m.repo.CreateChat(ctx, draft) +} + +func (m *Messenger) ListChats(ctx context.Context, caller uuid.UUID, limit *int, cursor *string) (Page[model.Chat], error) { + q, err := model.NewPageQuery(limit, cursor, chatsCursorScope) + if err != nil { + return Page[model.Chat]{}, err + } + chats, more, err := m.repo.ListChats(ctx, caller, q) + if err != nil { + return Page[model.Chat]{}, err + } + return newPage(chats, more, chatsCursorScope, model.Chat.Position), nil +} + +// GetChat отдаёт чат только участнику; постороннему — ErrNotFound, чтобы не +// раскрывать существование чужого чата. +func (m *Messenger) GetChat(ctx context.Context, caller, chatID uuid.UUID) (model.Chat, error) { + chat, err := m.repo.GetChat(ctx, chatID) + if err != nil { + return model.Chat{}, err + } + if !chat.HasMember(caller) { + return model.Chat{}, model.ErrNotFound + } + return chat, nil +} + +// SendMessage сначала проверяет доступ, затем тело: посторонний получает 404 +// даже на невалидное сообщение. Проверка членства и запись не атомарны, но +// участники чата неизменяемы, а чаты не удаляются, поэтому гонки здесь нет. +func (m *Messenger) SendMessage(ctx context.Context, caller, chatID uuid.UUID, text, key string, attachments []uuid.UUID) (model.Message, error) { + if _, err := m.GetChat(ctx, caller, chatID); err != nil { + return model.Message{}, err + } + draft, err := model.NewMessageDraft(chatID, caller, text, attachments) + if err != nil { + return model.Message{}, err + } + draft, err = model.WithKey(draft, key) + if err != nil { + return model.Message{}, err + } + return m.repo.CreateMessage(ctx, draft) +} + +func (m *Messenger) ListMessages(ctx context.Context, caller, chatID uuid.UUID, limit *int, cursor *string) (Page[model.Message], error) { + if _, err := m.GetChat(ctx, caller, chatID); err != nil { + return Page[model.Message]{}, err + } + scope := "messages:" + chatID.String() + q, err := model.NewPageQuery(limit, cursor, scope) + if err != nil { + return Page[model.Message]{}, err + } + messages, more, err := m.repo.ListMessages(ctx, chatID, q) + if err != nil { + return Page[model.Message]{}, err + } + return newPage(messages, more, scope, model.Message.Position), nil +} + +func newPage[T any](items []T, more bool, scope string, position func(T) model.Position) Page[T] { + page := Page[T]{Items: items} + if more && len(items) > 0 { + next := model.EncodeCursor(scope, position(items[len(items)-1])) + page.NextCursor = &next + } + return page +} diff --git a/backend/internal/application/usecase/pipeline.go b/backend/internal/application/usecase/pipeline.go new file mode 100644 index 00000000..545ebdb3 --- /dev/null +++ b/backend/internal/application/usecase/pipeline.go @@ -0,0 +1,138 @@ +package usecase + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "github.com/google/uuid" + "io" + "math/rand/v2" + "time" +) + +type Pipeline struct { + store port.PipelineStore + objects port.Objects + transform port.Transformer + MaxAttempts int32 + Fault func(string) error +} + +func NewPipeline(s port.PipelineStore, o port.Objects, t port.Transformer, max int32) *Pipeline { + return &Pipeline{store: s, objects: o, transform: t, MaxAttempts: max, Fault: func(string) error { return nil }} +} +func VariantKey(id uuid.UUID, version int32, name string) string { + return fmt.Sprintf("variants/%s/v%d/%s", id, version, name) +} +func (p *Pipeline) Process(ctx context.Context, j model.Job) error { + lease := uuid.New() + owned, a, ok, e := p.store.AcquireJob(ctx, j, lease, 30*time.Second) + if e != nil { + return e + } + if !ok { + return p.Cleanup(ctx) + } + j = owned + if j.Attempt > p.MaxAttempts { + return p.store.RetryJob(ctx, j, lease, "retry_exhausted", 0, true) + } + // Reprocessing uses the immutable original after staging was safely removed. + key := "staging/" + a.ID.String() + r, e := p.objects.Get(ctx, key) + if errors.Is(e, model.ErrNotFound) { + r, e = p.objects.Get(ctx, ObjectKey(a.ID)) + } + if e != nil { + return p.retry(ctx, j, lease, e) + } + original, e := io.ReadAll(io.LimitReader(r, model.MaxImageBytes+1)) + _ = r.Close() + if e != nil || int64(len(original)) != a.SizeBytes { + return p.retry(ctx, j, lease, model.ErrUnavailable) + } + sum := sha256.Sum256(original) + if hex.EncodeToString(sum[:]) != a.SHA256 { + return p.retry(ctx, j, lease, model.ErrInvalidImage) + } + if e = p.objects.Put(ctx, ObjectKey(a.ID), bytes.NewReader(original), a.SizeBytes, a.ContentType); e != nil { + return p.retry(ctx, j, lease, e) + } + if e = p.Fault("after_original"); e != nil { + return e + } + variants, e := p.transform.Transform(ctx, bytes.NewReader(original)) + if e != nil { + return p.retry(ctx, j, lease, e) + } + result := make([]model.Variant, 0, len(variants)) + for _, v := range variants { + if e = p.objects.Put(ctx, VariantKey(a.ID, j.PipelineVersion, v.Metadata.Name), bytes.NewReader(v.Bytes), int64(len(v.Bytes)), v.Metadata.ContentType); e != nil { + return p.retry(ctx, j, lease, e) + } + result = append(result, v.Metadata) + } + if e = p.Fault("after_objects"); e != nil { + return e + } + ok, e = p.store.CompleteJob(ctx, j, lease, result) + if e != nil { + return e + } + if !ok { + return nil + } + if e = p.Fault("after_commit"); e != nil { + return e + } + // Cleanup has its own durable marker and cannot change a published ready result. + return p.Cleanup(ctx) +} +func (p *Pipeline) retry(ctx context.Context, j model.Job, lease uuid.UUID, cause error) error { + failed := j.Attempt >= p.MaxAttempts + code := "processing_failed" + if failed { + code = "retry_exhausted" + } + if errors.Is(cause, model.ErrInvalidImage) { + failed = true + code = "invalid_image" + } + delay := time.Duration((1<= grace && age >= 0 } + rows, more, e := r.repo.ReconcileAttachments(ctx, after) + if e != nil { + return report, e + } + for _, a := range rows { + report.Metadata++ + if more { + report.NextMetadata = a.ID.String() + } + if !old(a.CreatedAt) { + report.Fresh++ + continue + } + // Only the fenced pipeline can publish versioned results. The legacy + // reconciler knows nothing about required variants, retries or outbox jobs. + if a.PipelineVersion > 0 { + report.PipelineSkipped++ + continue + } + info, e := r.objects.Stat(ctx, ObjectKey(a.ID)) + target := a.Status + if errors.Is(e, model.ErrNotFound) { + switch a.Status { + case "ready": + report.Missing++ + target = "missing" + case "uploading": + report.Missing++ + target = "abandoned" + } + } else if e != nil { + return report, e + } else if a.Status != "ready" { + valid := false + if info.Size == a.SizeBytes { + body, err := r.objects.Get(ctx, ObjectKey(a.ID)) + if err != nil { + return report, err + } + hash := sha256.New() + n, err := io.Copy(hash, io.LimitReader(body, a.SizeBytes+1)) + _ = body.Close() + if err != nil { + return report, model.ErrUnavailable + } + valid = n == a.SizeBytes && hex.EncodeToString(hash.Sum(nil)) == a.SHA256 + } + if valid { + report.Recovered++ + target = "ready" + } else { + report.Missing++ + target = "missing" + } + } else if info.Size != a.SizeBytes { + report.Missing++ + target = "missing" + } + if apply && target != a.Status { + changed, e := r.repo.SetAttachmentStatus(ctx, a.ID, a.Status, target) + if e != nil { + return report, e + } + if changed { + report.Changed++ + } + } + } + objects, more, e := r.objects.List(ctx, objectAfter) + if e != nil { + return report, e + } + for _, o := range objects { + report.Objects++ + if more { + report.NextObject = o.Key + } + if !old(o.Modified) { + report.Fresh++ + continue + } + raw, ok := strings.CutPrefix(o.Key, "originals/") + if !ok { + continue + } + id, e := uuid.Parse(raw) + if e != nil || ObjectKey(id) != o.Key { + continue + } + if _, e = r.repo.GetAttachment(ctx, id); e == nil { + continue + } else if !errors.Is(e, model.ErrNotFound) { + return report, e + } + report.Orphans++ + if !apply { + continue + } + fresh, e := r.objects.Stat(ctx, o.Key) + if errors.Is(e, model.ErrNotFound) { + continue + } + if e != nil { + return report, e + } + if fresh.ETag != o.ETag || fresh.Size != o.Size || !fresh.Modified.Equal(o.Modified) || !old(fresh.Modified) { + report.Fresh++ + continue + } + if _, e = r.repo.GetAttachment(ctx, id); e == nil { + continue + } else if !errors.Is(e, model.ErrNotFound) { + return report, e + } + // Object keys are random UUIDs, immutable and never reused by the upload API. + // An absent metadata row cannot later adopt an old inventory key. + if e = r.objects.Remove(ctx, o.Key); e != nil { + return report, e + } + report.Deleted++ + } + return report, nil +} diff --git a/backend/internal/application/usecase/reconcile_pipeline_test.go b/backend/internal/application/usecase/reconcile_pipeline_test.go new file mode 100644 index 00000000..070c113e --- /dev/null +++ b/backend/internal/application/usecase/reconcile_pipeline_test.go @@ -0,0 +1,31 @@ +package usecase + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "context" + "crypto/sha256" + "encoding/hex" + "github.com/google/uuid" + "testing" + "time" +) + +func TestReconcilerCannotPublishVersionedPipelineResult(t *testing.T) { + for _, status := range []string{"uploading", "queued", "processing", "failed", "ready"} { + t.Run(status, func(t *testing.T) { + id := uuid.New() + raw := []byte("only original survived worker crash") + sum := sha256.Sum256(raw) + repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: status, PipelineVersion: 1, SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:]), CreatedAt: fixedNow.Add(-2 * time.Minute)}}} + objects := &fixtureObjects{items: map[string]port.Object{ObjectKey(id): {Key: ObjectKey(id), Size: int64(len(raw)), Modified: fixedNow.Add(-2 * time.Minute)}}, content: raw} + report, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true) + if e != nil { + t.Fatal(e) + } + if repo.rows[id].Status != status || report.Recovered != 0 || report.Changed != 0 { + t.Fatalf("pipeline state was changed by legacy reconciler: status %s -> %s, report %+v", status, repo.rows[id].Status, report) + } + }) + } +} diff --git a/backend/internal/application/usecase/reconcile_test.go b/backend/internal/application/usecase/reconcile_test.go new file mode 100644 index 00000000..1ac21cf4 --- /dev/null +++ b/backend/internal/application/usecase/reconcile_test.go @@ -0,0 +1,184 @@ +package usecase + +import ( + "backend/internal/domain/model" + "backend/internal/domain/port" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/google/uuid" + "io" + "testing" + "time" +) + +type fixtureRepo struct { + rows map[uuid.UUID]model.Attachment + failure error + more bool +} + +func (f *fixtureRepo) ReserveAttachment(context.Context, model.Attachment) (model.Attachment, error) { + panic("not used") +} +func (f *fixtureRepo) GetAttachment(_ context.Context, id uuid.UUID) (model.Attachment, error) { + if f.failure != nil { + return model.Attachment{}, f.failure + } + a, ok := f.rows[id] + if !ok { + return a, model.ErrNotFound + } + return a, nil +} +func (f *fixtureRepo) SetAttachmentStatus(_ context.Context, id uuid.UUID, from, to string) (bool, error) { + a := f.rows[id] + if a.Status != from { + return false, nil + } + a.Status = to + f.rows[id] = a + return true, nil +} +func (f *fixtureRepo) ReconcileAttachments(context.Context, uuid.UUID) ([]model.Attachment, bool, error) { + out := []model.Attachment{} + for _, a := range f.rows { + out = append(out, a) + } + return out, f.more, f.failure +} + +type fixtureObjects struct { + items map[string]port.Object + content []byte + failure error + removed int + changed bool +} + +func (f *fixtureObjects) Put(context.Context, string, io.Reader, int64, string) error { + panic("not used") +} +func (f *fixtureObjects) Get(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(f.content)), f.failure +} +func (f *fixtureObjects) Stat(_ context.Context, key string) (port.Object, error) { + if f.failure != nil { + return port.Object{}, f.failure + } + o, ok := f.items[key] + if !ok { + return o, model.ErrNotFound + } + if f.changed { + o.ETag = "changed" + } + return o, nil +} +func (f *fixtureObjects) Remove(_ context.Context, key string) error { + delete(f.items, key) + f.removed++ + return nil +} +func (f *fixtureObjects) List(context.Context, string) ([]port.Object, bool, error) { + out := []port.Object{} + for _, o := range f.items { + out = append(out, o) + } + return out, false, f.failure +} + +var fixedNow = time.Date(2026, 11, 12, 12, 0, 0, 0, time.UTC) + +func TestOrphanCleanupHonorsGraceAndDryRun(t *testing.T) { + for _, tc := range []struct { + name string + age time.Duration + apply, used, changed bool + wantDeleted int + }{ + {name: "old dry run", age: 2 * time.Minute}, + {name: "old apply", age: 2 * time.Minute, apply: true, wantDeleted: 1}, + {name: "fresh protected", age: 10 * time.Second, apply: true}, + {name: "future timestamp protected", age: -time.Minute, apply: true}, + {name: "referenced protected", age: 2 * time.Minute, apply: true, used: true}, + {name: "changed inventory protected", age: 2 * time.Minute, apply: true, changed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + key := ObjectKey(id) + repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{}} + if tc.used { + repo.rows[id] = model.Attachment{ID: id, Status: "ready", CreatedAt: fixedNow} + } + objects := &fixtureObjects{items: map[string]port.Object{key: {Key: key, ETag: "original", Modified: fixedNow.Add(-tc.age)}}, changed: tc.changed} + report, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, tc.apply) + if e != nil { + t.Fatal(e) + } + if objects.removed != tc.wantDeleted { + t.Fatalf("deleted %d, want %d; report=%+v", objects.removed, tc.wantDeleted, report) + } + if tc.wantDeleted > 0 { + _, e = NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true) + if e != nil { + t.Fatal(e) + } + if objects.removed != 1 { + t.Fatal("second run deleted something twice") + } + } + }) + } +} +func TestMissingAndPendingMetadataRepair(t *testing.T) { + for _, status := range []string{"ready", "uploading", "missing"} { + t.Run(status, func(t *testing.T) { + id := uuid.New() + raw := []byte("original bytes") + sum := sha256.Sum256(raw) + repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: status, CreatedAt: fixedNow.Add(-2 * time.Minute), UpdatedAt: fixedNow.Add(-2 * time.Minute), SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])}}} + objects := &fixtureObjects{items: map[string]port.Object{}, content: raw} + want := "missing" + if status != "ready" { + objects.items[ObjectKey(id)] = port.Object{Key: ObjectKey(id), Size: int64(len(raw)), Modified: fixedNow.Add(-2 * time.Minute)} + want = "ready" + } + _, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true) + if e != nil { + t.Fatal(e) + } + if got := repo.rows[id].Status; got != want { + t.Fatalf("status %s, want %s", got, want) + } + if objects.removed != 0 { + t.Fatal("metadata repair removed original") + } + }) + } +} +func TestDependencyFailureNeverDeletes(t *testing.T) { + repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{}, failure: model.ErrUnavailable} + objects := &fixtureObjects{items: map[string]port.Object{}} + _, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true) + if !errors.Is(e, model.ErrUnavailable) { + t.Fatalf("expected dependency error, got %v", e) + } + if objects.removed != 0 { + t.Fatal("deleted during failed inventory") + } +} +func TestFailureDoesNotReturnCursorPastFailedRecord(t *testing.T) { + id := uuid.New() + repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: "ready", CreatedAt: fixedNow.Add(-2 * time.Minute)}}, more: true} + objects := &fixtureObjects{failure: model.ErrUnavailable} + report, err := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true) + if !errors.Is(err, model.ErrUnavailable) { + t.Fatalf("expected failure, got %v", err) + } + if report.NextMetadata != "" || report.NextObject != "" { + t.Fatal("failed inventory must restart rather than skip the failed item") + } +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 00000000..4aab77ce --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,86 @@ +package config + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/spf13/viper" +) + +// Config приходит только из окружения (12-factor): в контейнер не копируются +// ни .env, ни файлы конфигурации. Значения по умолчанию — ниже. +type Config struct { + Host string `mapstructure:"APP_HOST"` + Port int `mapstructure:"APP_PORT"` + Env string `mapstructure:"APP_ENV"` + LogLevel string `mapstructure:"LOG_LEVEL"` + ShutdownTimeout time.Duration `mapstructure:"SHUTDOWN_TIMEOUT"` + MaxBodyBytes int64 `mapstructure:"MAX_BODY_BYTES"` + MaxInflight int `mapstructure:"MAX_INFLIGHT"` + MaxQueue int `mapstructure:"MAX_QUEUE"` + QueueTimeout time.Duration `mapstructure:"QUEUE_TIMEOUT"` + DatabaseURLFile string `mapstructure:"DATABASE_URL_FILE"` + CacheURLFile string `mapstructure:"CACHE_URL_FILE"` + SessionTTLSeconds int `mapstructure:"SESSION_TTL_SECONDS"` + MaxConns int32 `mapstructure:"PG_MAX_CONNS"` +} + +var defaults = map[string]any{ + "APP_HOST": "0.0.0.0", + "APP_PORT": 8080, + "APP_ENV": "lab", + "LOG_LEVEL": "info", + "SHUTDOWN_TIMEOUT": "8s", + "MAX_BODY_BYTES": 64 << 10, + "MAX_INFLIGHT": 4, + "MAX_QUEUE": 32, + "QUEUE_TIMEOUT": "1s", + "DATABASE_URL_FILE": "/run/secrets/database_url", + "CACHE_URL_FILE": "/run/secrets/cache_url", + "SESSION_TTL_SECONDS": 3600, + "PG_MAX_CONNS": 8, +} + +func Load() (Config, error) { + v := viper.New() + for key, value := range defaults { + v.SetDefault(key, value) + } + v.AutomaticEnv() + var cfg Config + if err := v.Unmarshal(&cfg); err != nil { + return Config{}, fmt.Errorf("parse config: %w", err) + } + if cfg.ShutdownTimeout <= 0 || cfg.ShutdownTimeout > 9*time.Second { + return Config{}, fmt.Errorf("SHUTDOWN_TIMEOUT must be in (0, 9s]: docker stop ждёт 10 с") + } + if cfg.SessionTTLSeconds < 1 || cfg.SessionTTLSeconds > 86400 { + return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be 1..86400") + } + if cfg.MaxConns < 1 || cfg.MaxConns > 64 { + return Config{}, fmt.Errorf("PG_MAX_CONNS must be 1..64") + } + if cfg.MaxInflight < 1 || cfg.MaxInflight > 4 || cfg.MaxQueue < 0 || cfg.MaxQueue > 128 || cfg.QueueTimeout <= 0 || cfg.QueueTimeout > 2*time.Second { + return Config{}, fmt.Errorf("HTTP gate exceeds the tested resource budget") + } + if cfg.MaxBodyBytes < 1 || cfg.MaxBodyBytes > 65536 { + return Config{}, fmt.Errorf("MAX_BODY_BYTES must be 1..65536") + } + return cfg, nil +} + +func (c Config) Addr() string { return fmt.Sprintf("%s:%d", c.Host, c.Port) } + +func ReadSecret(path string) (string, error) { + b, e := os.ReadFile(path) + if e != nil { + return "", fmt.Errorf("required secret file unavailable") + } + v := strings.TrimSpace(string(b)) + if v == "" { + return "", fmt.Errorf("required secret is empty") + } + return v, nil +} diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go new file mode 100644 index 00000000..7a82caa9 --- /dev/null +++ b/backend/internal/config/config_test.go @@ -0,0 +1,24 @@ +package config + +import "testing" + +func TestRejectsConfigurationOutsideResourceBudget(t *testing.T) { + for _, tc := range []struct{ key, value string }{{"MAX_INFLIGHT", "0"}, {"MAX_INFLIGHT", "5"}, {"MAX_QUEUE", "-1"}, {"MAX_QUEUE", "129"}, {"QUEUE_TIMEOUT", "0s"}, {"QUEUE_TIMEOUT", "3s"}, {"MAX_BODY_BYTES", "0"}, {"MAX_BODY_BYTES", "65537"}, {"PG_MAX_CONNS", "0"}, {"PG_MAX_CONNS", "65"}, {"SESSION_TTL_SECONDS", "0"}, {"SESSION_TTL_SECONDS", "86401"}} { + t.Run(tc.key+tc.value, func(t *testing.T) { + t.Setenv(tc.key, tc.value) + if _, e := Load(); e == nil { + t.Fatal("accepted configuration beyond tested bounds") + } + }) + } +} +func TestPoolSizeIsConfigurableWithinBounds(t *testing.T) { + t.Setenv("PG_MAX_CONNS", "1") + cfg, e := Load() + if e != nil { + t.Fatal(e) + } + if cfg.MaxConns != 1 { + t.Fatal("configured pool size ignored") + } +} diff --git a/backend/internal/domain/model/attachment.go b/backend/internal/domain/model/attachment.go new file mode 100644 index 00000000..55cbc229 --- /dev/null +++ b/backend/internal/domain/model/attachment.go @@ -0,0 +1,35 @@ +package model + +import ( + "github.com/google/uuid" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const MaxImageBytes int64 = 10 * 1024 * 1024 +const MaxImagePixels int64 = 40_000_000 + +type Attachment struct { + ID, ChatID, OwnerID uuid.UUID + Filename, ContentType, SHA256, Status string + SizeBytes int64 + CreatedAt, UpdatedAt time.Time + PipelineVersion int32 + ErrorCode string + Variants []Variant + StagingExpired, StagingCleaned bool +} + +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || utf8.RuneCountInString(name) < 1 || utf8.RuneCountInString(name) > 255 || strings.TrimSpace(name) == "" { + return invalidf("filename: 1–255 characters required") + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return invalidf("filename: invalid character") + } + } + return nil +} diff --git a/backend/internal/domain/model/chat.go b/backend/internal/domain/model/chat.go new file mode 100644 index 00000000..609b152b --- /dev/null +++ b/backend/internal/domain/model/chat.go @@ -0,0 +1,52 @@ +package model + +import ( + "slices" + "time" + + "github.com/google/uuid" +) + +const maxChatMembers = 100 + +// Chat неизменяем после создания: участники задаются один раз, удаления в API нет. +type Chat struct { + ID uuid.UUID + Title string + MemberIDs []uuid.UUID + CreatedAt time.Time +} + +func (c Chat) HasMember(userID uuid.UUID) bool { + return slices.Contains(c.MemberIDs, userID) +} + +// ChatDraft — проверенные данные нового чата. Существование участников +// проверяет хранилище атомарно с созданием. +type ChatDraft struct { + Creator uuid.UUID + Title string + MemberIDs []uuid.UUID +} + +func NewChatDraft(creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatDraft, error) { + if err := requireText("title", title, 1, 100); err != nil { + return ChatDraft{}, err + } + if len(memberIDs) == 0 || len(memberIDs) > maxChatMembers { + return ChatDraft{}, invalidf("member_ids: от 1 до %d участников", maxChatMembers) + } + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + for _, id := range memberIDs { + if _, dup := seen[id]; dup { + return ChatDraft{}, invalidf("member_ids: повторяющийся участник") + } + seen[id] = struct{}{} + } + if _, ok := seen[creator]; !ok { + return ChatDraft{}, invalidf("member_ids: должен содержать создателя чата") + } + return ChatDraft{Creator: creator, Title: title, MemberIDs: slices.Clone(memberIDs)}, nil +} + +func (c Chat) Position() Position { return Position{CreatedAt: c.CreatedAt, ID: c.ID} } diff --git a/backend/internal/domain/model/errors.go b/backend/internal/domain/model/errors.go new file mode 100644 index 00000000..54ea5eb7 --- /dev/null +++ b/backend/internal/domain/model/errors.go @@ -0,0 +1,33 @@ +package model + +import ( + "errors" + "fmt" +) + +var ( + ErrUnavailable = errors.New("dependency unavailable") + ErrInvalidImage = errors.New("invalid staged image") + ErrConflict = errors.New("conflict") + ErrNotFound = errors.New("not found") + ErrUnauthorized = errors.New("unauthorized") + ErrUsernameTaken = errors.New("username taken") + // ErrStorageFull — исчерпан бюджет памяти хранилища. Подтверждённые данные + // не вытесняются, поэтому новые записи отклоняются, а чтение продолжает работать. + ErrStorageFull = errors.New("storage full") + // ErrUnknownMember возвращает хранилище, если среди участников нового чата + // есть несуществующий пользователь; для клиента это ошибка ввода (400). + ErrUnknownMember error = &ValidationError{Message: "member_ids: неизвестный пользователь"} +) + +// ValidationError — нарушение правил входных данных; Message безопасно +// показывать клиенту. +type ValidationError struct { + Message string +} + +func (e *ValidationError) Error() string { return e.Message } + +func invalidf(format string, args ...any) error { + return &ValidationError{Message: fmt.Sprintf(format, args...)} +} diff --git a/backend/internal/domain/model/job.go b/backend/internal/domain/model/job.go new file mode 100644 index 00000000..9f085f19 --- /dev/null +++ b/backend/internal/domain/model/job.go @@ -0,0 +1,24 @@ +package model + +import ( + "github.com/google/uuid" + "time" +) + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SHA256 string `json:"sha256"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` +} +type Job struct { + ID uuid.UUID `json:"job_id"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int32 `json:"pipeline_version"` + Attempt int32 `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} diff --git a/backend/internal/domain/model/message.go b/backend/internal/domain/model/message.go new file mode 100644 index 00000000..9bbe55f3 --- /dev/null +++ b/backend/internal/domain/model/message.go @@ -0,0 +1,82 @@ +package model + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "regexp" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +const MaxMessageRunes = 4000 + +type Message struct { + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time + AttachmentIDs []uuid.UUID +} + +// MessageDraft — проверенное сообщение; отправитель берётся из идентификации, +// а не из тела запроса. +type MessageDraft struct { + ChatID uuid.UUID + SenderID uuid.UUID + Text string + Key string + PayloadHash string + AttachmentIDs []uuid.UUID +} + +var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func WithKey(d MessageDraft, key string) (MessageDraft, error) { + if !keyPattern.MatchString(key) { + return MessageDraft{}, invalidf("Idempotency-Key: 1–128 ASCII characters required") + } + canonical, _ := json.Marshal(struct { + Text string + Attachments []uuid.UUID + }{d.Text, d.AttachmentIDs}) + sum := sha256.Sum256(canonical) + d.Key = key + d.PayloadHash = hex.EncodeToString(sum[:]) + return d, nil +} + +func NewMessageDraft(chatID, senderID uuid.UUID, text string, attachments ...[]uuid.UUID) (MessageDraft, error) { + ids := []uuid.UUID{} + if len(attachments) > 0 { + ids = slices.Clone(attachments[0]) + if ids == nil { + ids = []uuid.UUID{} + } + } + if len(ids) > 10 { + return MessageDraft{}, invalidf("attachment_ids: at most 10") + } + seen := map[uuid.UUID]bool{} + for _, id := range ids { + if seen[id] { + return MessageDraft{}, invalidf("attachment_ids: duplicate") + } + seen[id] = true + } + if len(ids) == 0 || strings.TrimSpace(text) != "" { + if err := requireText("text", text, 1, MaxMessageRunes); err != nil { + return MessageDraft{}, err + } + } else if !utf8.ValidString(text) || utf8.RuneCountInString(text) > MaxMessageRunes || strings.ContainsRune(text, 0) { + return MessageDraft{}, invalidf("text: invalid optional text") + } + return MessageDraft{ChatID: chatID, SenderID: senderID, Text: text, AttachmentIDs: ids}, nil +} + +func (m Message) Position() Position { return Position{CreatedAt: m.CreatedAt, ID: m.ID} } diff --git a/backend/internal/domain/model/model_test.go b/backend/internal/domain/model/model_test.go new file mode 100644 index 00000000..e47d1bf2 --- /dev/null +++ b/backend/internal/domain/model/model_test.go @@ -0,0 +1,126 @@ +package model + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func isValidationError(err error) bool { + var v *ValidationError + return errors.As(err, &v) +} + +func TestNewUserDraft(t *testing.T) { + t.Parallel() + cases := []struct { + name, username, display string + ok bool + }{ + {"minimal", "abc", "A", true}, + {"max username 32", strings.Repeat("a", 32), "A", true}, + {"username too short", "ab", "A", false}, + {"username too long", strings.Repeat("a", 33), "A", false}, + {"uppercase rejected", "Alice", "A", false}, + {"cyrillic username rejected", "алиса", "A", false}, + {"display 100 emoji ok", "abc", strings.Repeat("😀", 100), true}, + {"display 101 code points rejected", "abc", strings.Repeat("ж", 101), false}, + {"display whitespace only", "abc", " \t\n", false}, + {"display empty", "abc", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := NewUserDraft(tc.username, tc.display) + if tc.ok && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !tc.ok && !isValidationError(err) { + t.Fatalf("want ValidationError, got %v", err) + } + }) + } +} + +func TestNewChatDraft(t *testing.T) { + t.Parallel() + me, other := uuid.New(), uuid.New() + tooMany := []uuid.UUID{me} + for range 100 { + tooMany = append(tooMany, uuid.New()) + } + cases := []struct { + name string + members []uuid.UUID + ok bool + }{ + {"creator only", []uuid.UUID{me}, true}, + {"two members", []uuid.UUID{me, other}, true}, + {"empty", nil, false}, + {"creator missing", []uuid.UUID{other}, false}, + {"duplicate member", []uuid.UUID{me, other, other}, false}, + {"over 100 members", tooMany, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := NewChatDraft(me, "Чат", tc.members) + if tc.ok != (err == nil) { + t.Fatalf("ok=%v, err=%v", tc.ok, err) + } + }) + } +} + +func TestNewMessageDraftLimitsInCodePoints(t *testing.T) { + t.Parallel() + if _, err := NewMessageDraft(uuid.New(), uuid.New(), strings.Repeat("👋", MaxMessageRunes)); err != nil { + t.Fatalf("4000 four-byte code points must pass: %v", err) + } + if _, err := NewMessageDraft(uuid.New(), uuid.New(), strings.Repeat("a", MaxMessageRunes+1)); !isValidationError(err) { + t.Fatalf("4001 code points must be rejected, got %v", err) + } +} + +func TestCursorRoundTripAndScope(t *testing.T) { + t.Parallel() + pos := Position{CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 123456000, time.UTC), ID: uuid.New()} + cursor := EncodeCursor("messages:a", pos) + + q, err := NewPageQuery(nil, &cursor, "messages:a") + if err != nil || q.After == nil || *q.After != pos || q.Limit != DefaultPageLimit { + t.Fatalf("round trip failed: %+v, %v", q, err) + } + if _, err := NewPageQuery(nil, &cursor, "messages:b"); !isValidationError(err) { + t.Fatalf("cursor of another resource must be rejected, got %v", err) + } + for _, bad := range []string{"", "!!!", EncodeCursor("x", pos)[:5]} { + if _, err := NewPageQuery(nil, &bad, "messages:a"); !isValidationError(err) { + t.Fatalf("garbage cursor %q must be rejected, got %v", bad, err) + } + } +} + +func TestPageLimitBounds(t *testing.T) { + t.Parallel() + for limit, ok := range map[int]bool{0: false, 1: true, 100: true, 101: false, -1: false} { + _, err := NewPageQuery(&limit, nil, "chats") + if ok != (err == nil) { + t.Errorf("limit=%d: ok=%v, err=%v", limit, ok, err) + } + } +} + +func TestPositionOrderTiesBrokenByID(t *testing.T) { + t.Parallel() + at := time.Now() + low := Position{CreatedAt: at, ID: uuid.MustParse("00000000-0000-4000-8000-000000000001")} + high := Position{CreatedAt: at, ID: uuid.MustParse("ffffffff-0000-4000-8000-000000000000")} + later := Position{CreatedAt: at.Add(time.Microsecond), ID: low.ID} + if !high.After(low) || low.After(high) || !later.After(high) || low.After(low) { + t.Fatal("order must be (created_at, id) ascending") + } +} diff --git a/backend/internal/domain/model/page.go b/backend/internal/domain/model/page.go new file mode 100644 index 00000000..4eda2571 --- /dev/null +++ b/backend/internal/domain/model/page.go @@ -0,0 +1,84 @@ +package model + +import ( + "bytes" + "encoding/base64" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + DefaultPageLimit = 50 + MaxPageLimit = 100 +) + +// Position — ключ keyset-пагинации: история упорядочена по (created_at, id). +type Position struct { + CreatedAt time.Time + ID uuid.UUID +} + +// After сообщает, стоит ли p строго после other в порядке (created_at, id). +func (p Position) After(other Position) bool { + if !p.CreatedAt.Equal(other.CreatedAt) { + return p.CreatedAt.After(other.CreatedAt) + } + return bytes.Compare(p.ID[:], other.ID[:]) > 0 +} + +// PageQuery — проверенный запрос страницы: limit 1–100 и позиция после +// последнего элемента предыдущей страницы (nil — с начала). +type PageQuery struct { + Limit int + After *Position +} + +// NewPageQuery проверяет limit и курсор. scope привязывает курсор к ресурсу: +// курсор истории одного чата не подходит к другому чату или к списку чатов. +func NewPageQuery(limit *int, cursor *string, scope string) (PageQuery, error) { + q := PageQuery{Limit: DefaultPageLimit} + if limit != nil { + if *limit < 1 || *limit > MaxPageLimit { + return PageQuery{}, invalidf("limit: от 1 до %d", MaxPageLimit) + } + q.Limit = *limit + } + if cursor != nil { + pos, err := decodeCursor(*cursor, scope) + if err != nil { + return PageQuery{}, err + } + q.After = &pos + } + return q, nil +} + +// EncodeCursor делает непрозрачную для клиента строку из позиции и scope. +func EncodeCursor(scope string, p Position) string { + raw := scope + "|" + strconv.FormatInt(p.CreatedAt.UnixMicro(), 10) + "|" + p.ID.String() + return base64.RawURLEncoding.EncodeToString([]byte(raw)) +} + +func decodeCursor(cursor, scope string) (Position, error) { + errBad := invalidf("cursor: некорректное значение") + raw, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return Position{}, errBad + } + parts := strings.Split(string(raw), "|") + if len(parts) != 3 || parts[0] != scope { + return Position{}, errBad + } + micros, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return Position{}, errBad + } + id, err := uuid.Parse(parts[2]) + if err != nil { + return Position{}, errBad + } + return Position{CreatedAt: time.UnixMicro(micros).UTC(), ID: id}, nil +} diff --git a/backend/internal/domain/model/postgres_text_test.go b/backend/internal/domain/model/postgres_text_test.go new file mode 100644 index 00000000..6739cebc --- /dev/null +++ b/backend/internal/domain/model/postgres_text_test.go @@ -0,0 +1,13 @@ +package model + +import "testing" + +func TestRejectsNULInPersistedText(t *testing.T) { + for _, field := range []string{"display_name", "title", "text"} { + t.Run(field, func(t *testing.T) { + if err := requireText(field, "a\x00b", 1, 4000); err == nil { + t.Fatal("PostgreSQL text must reject NUL as a validation error") + } + }) + } +} diff --git a/backend/internal/domain/model/text.go b/backend/internal/domain/model/text.go new file mode 100644 index 00000000..789696b0 --- /dev/null +++ b/backend/internal/domain/model/text.go @@ -0,0 +1,24 @@ +package model + +import ( + "strings" + "unicode/utf8" +) + +// requireText проверяет обязательную строку: длина в Unicode code points +// (не в байтах) и запрет строк только из пробельных символов. +func requireText(field, value string, minLen, maxLen int) error { + if strings.ContainsRune(value, 0) { + return invalidf("%s: NUL недопустим в PostgreSQL text", field) + } + if !utf8.ValidString(value) { + return invalidf("%s: некорректный UTF-8", field) + } + if strings.TrimSpace(value) == "" { + return invalidf("%s: пустая строка", field) + } + if n := utf8.RuneCountInString(value); n < minLen || n > maxLen { + return invalidf("%s: длина должна быть от %d до %d символов", field, minLen, maxLen) + } + return nil +} diff --git a/backend/internal/domain/model/user.go b/backend/internal/domain/model/user.go new file mode 100644 index 00000000..ccecf355 --- /dev/null +++ b/backend/internal/domain/model/user.go @@ -0,0 +1,34 @@ +package model + +import ( + "regexp" + "time" + + "github.com/google/uuid" +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +// UserDraft — проверенные данные нового пользователя; ID и время назначает хранилище. +type UserDraft struct { + Username string + DisplayName string + PasswordHash string +} + +func NewUserDraft(username, displayName string) (UserDraft, error) { + if !usernamePattern.MatchString(username) { + return UserDraft{}, invalidf("username: допустимо [a-z0-9_], от 3 до 32 символов") + } + if err := requireText("display_name", displayName, 1, 100); err != nil { + return UserDraft{}, err + } + return UserDraft{Username: username, DisplayName: displayName}, nil +} diff --git a/backend/internal/domain/port/auth.go b/backend/internal/domain/port/auth.go new file mode 100644 index 00000000..92b5b279 --- /dev/null +++ b/backend/internal/domain/port/auth.go @@ -0,0 +1,18 @@ +package port + +import ( + "context" + "github.com/google/uuid" + "time" +) + +type Passwords interface { + Hash(string) (string, error) + Verify(string, string) bool +} +type Sessions interface { + Credentials(context.Context, string) (uuid.UUID, string, error) + CreateSession(context.Context, string, uuid.UUID, time.Time) error + AuthenticateSession(context.Context, string) (uuid.UUID, error) + RevokeSession(context.Context, string) error +} diff --git a/backend/internal/domain/port/objects.go b/backend/internal/domain/port/objects.go new file mode 100644 index 00000000..6385df27 --- /dev/null +++ b/backend/internal/domain/port/objects.go @@ -0,0 +1,20 @@ +package port + +import ( + "context" + "io" + "time" +) + +type Object struct { + Key, ETag string + Size int64 + Modified time.Time +} +type Objects interface { + Put(context.Context, string, io.Reader, int64, string) error + Get(context.Context, string) (io.ReadCloser, error) + Stat(context.Context, string) (Object, error) + Remove(context.Context, string) error + List(context.Context, string) ([]Object, bool, error) +} diff --git a/backend/internal/domain/port/pipeline.go b/backend/internal/domain/port/pipeline.go new file mode 100644 index 00000000..9dc4a418 --- /dev/null +++ b/backend/internal/domain/port/pipeline.go @@ -0,0 +1,28 @@ +package port + +import ( + "backend/internal/domain/model" + "context" + "github.com/google/uuid" + "io" + "time" +) + +type PipelineStore interface { + PendingJobs(context.Context) ([]model.Job, error) + Published(context.Context, uuid.UUID) error + AcquireJob(context.Context, model.Job, uuid.UUID, time.Duration) (model.Job, model.Attachment, bool, error) + CompleteJob(context.Context, model.Job, uuid.UUID, []model.Variant) (bool, error) + RetryJob(context.Context, model.Job, uuid.UUID, string, time.Duration, bool) error + ReplayJob(context.Context, uuid.UUID) (bool, error) + AdvancePipeline(context.Context, uuid.UUID) (bool, error) + StagingCleanup(context.Context) ([]model.Attachment, error) + StagingCleaned(context.Context, uuid.UUID) error +} +type Transformer interface { + Transform(context.Context, io.Reader) ([]ProcessedVariant, error) +} +type ProcessedVariant struct { + Metadata model.Variant + Bytes []byte +} diff --git a/backend/internal/domain/repository/repository.go b/backend/internal/domain/repository/repository.go new file mode 100644 index 00000000..440b894a --- /dev/null +++ b/backend/internal/domain/repository/repository.go @@ -0,0 +1,64 @@ +// Package repository — порт персистентности. В lab1 его реализует хранилище +// в памяти процесса, с lab2 — PostgreSQL; ядро от этой замены не меняется. +package repository + +import ( + "context" + + "backend/internal/domain/model" + "backend/internal/domain/port" + + "github.com/google/uuid" +) + +// Repository assigns IDs and database timestamps. Keyset order is (created_at, id); +// concurrent commits do not provide insertion order or a traversal snapshot. +// Каждый метод атомарен сам по себе. +type Repository interface { + UserRepository + ChatRepository + MessageRepository + AttachmentRepository +} + +type AttachmentRepository interface { + ReserveAttachment(context.Context, model.Attachment) (model.Attachment, error) + GetAttachment(context.Context, uuid.UUID) (model.Attachment, error) + SetAttachmentStatus(context.Context, uuid.UUID, string, string) (bool, error) + ReconcileAttachments(context.Context, uuid.UUID) ([]model.Attachment, bool, error) +} + +type Persistent interface { + Repository + port.Sessions + AsyncAttachments + port.PipelineStore +} + +type AsyncAttachments interface { + AttachmentRepository + QueueAttachment(context.Context, model.Attachment, string, string) (model.Attachment, error) +} + +type UserRepository interface { + // CreateUser: model.ErrUsernameTaken, model.ErrStorageFull. + CreateUser(ctx context.Context, draft model.UserDraft) (model.User, error) + // GetUser: model.ErrNotFound. + GetUser(ctx context.Context, id uuid.UUID) (model.User, error) +} + +type ChatRepository interface { + // CreateChat: model.ErrUnknownMember, model.ErrStorageFull. + CreateChat(ctx context.Context, draft model.ChatDraft) (model.Chat, error) + // GetChat: model.ErrNotFound. + GetChat(ctx context.Context, id uuid.UUID) (model.Chat, error) + // ListChats возвращает чаты участника и признак, что есть следующая страница. + ListChats(ctx context.Context, memberID uuid.UUID, q model.PageQuery) ([]model.Chat, bool, error) +} + +type MessageRepository interface { + // CreateMessage: model.ErrNotFound (нет чата), model.ErrStorageFull. + CreateMessage(ctx context.Context, draft model.MessageDraft) (model.Message, error) + // ListMessages возвращает страницу истории и признак следующей страницы. + ListMessages(ctx context.Context, chatID uuid.UUID, q model.PageQuery) ([]model.Message, bool, error) +} diff --git a/backend/pkg/logger/logger.go b/backend/pkg/logger/logger.go new file mode 100644 index 00000000..e76c339d --- /dev/null +++ b/backend/pkg/logger/logger.go @@ -0,0 +1,18 @@ +package logger + +import ( + "os" + "time" + + "github.com/rs/zerolog" +) + +// New — JSON-логи в stdout, их собирает docker logs. +func New(level string) zerolog.Logger { + lvl, err := zerolog.ParseLevel(level) + if err != nil || level == "" { + lvl = zerolog.InfoLevel + } + zerolog.TimeFieldFormat = time.RFC3339Nano + return zerolog.New(os.Stdout).Level(lvl).With().Timestamp().Logger() +} diff --git a/backend/pkg/server/server.go b/backend/pkg/server/server.go new file mode 100644 index 00000000..85d496f0 --- /dev/null +++ b/backend/pkg/server/server.go @@ -0,0 +1,29 @@ +package server + +import ( + "context" + "net/http" + "time" +) + +type Server struct { + httpServer *http.Server +} + +func New(addr string, handler http.Handler) *Server { + return &Server{httpServer: &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 16 << 10, + }} +} + +func (s *Server) ListenAndServe() error { return s.httpServer.ListenAndServe() } + +// Shutdown сразу закрывает слушающий сокет (новые соединения не принимаются) +// и ждёт завершения начатых запросов, но не дольше ctx. +func (s *Server) Shutdown(ctx context.Context) error { return s.httpServer.Shutdown(ctx) } diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0ad6f7fb --- /dev/null +++ b/compose.yaml @@ -0,0 +1,122 @@ +name: mtuci-vp5-medvedev +x-api: &api + image: mtuci-vp5:local + build: . + cpus: 1 + mem_limit: 2g + stop_grace_period: 10s + read_only: true + security_opt: [no-new-privileges:true] + cap_drop: [ALL] + tmpfs: ['/tmp:uid=65532,gid=65532,size=256m,mode=1777'] + environment: + APP_PORT: 8080 + SESSION_TTL_SECONDS: 3600 + PG_MAX_CONNS: 8 + GOMEMLIMIT: 1536MiB + secrets: [database_url, cache_url, s3_credentials] + depends_on: + migrate: {condition: service_completed_successfully} + valkey: {condition: service_started} + bootstrap: {condition: service_completed_successfully} + networks: [private] +services: + postgres: + image: postgres:18.6@sha256:74935e72241653ca55e0414067e6d8763aceb8a810eb51b452253ec3dcfc4336 + cpus: 1 + mem_limit: 768m + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + secrets: [pg_password] + volumes: [postgres:/var/lib/postgresql] + healthcheck: + test: [CMD-SHELL, pg_isready -U messenger -d messenger] + interval: 2s + timeout: 2s + retries: 30 + networks: [private] + valkey: + image: valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd + cpus: 0.25 + mem_limit: 128m + command: [valkey-server, /run/secrets/valkey_conf] + secrets: [valkey_conf] + volumes: [valkey:/data] + networks: [private] + s3: + image: rustfs/rustfs:1.0.0-rc.1@sha256:f53d700fc16809070326f6ab3fb565d48e722364a7bd5a12e47d0a5abe079ce6 + cpus: 1 + mem_limit: 768m + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret + RUSTFS_VOLUMES: /data + RUSTFS_CONSOLE_ENABLE: 'false' + secrets: [s3_access,s3_secret] + volumes: [s3:/data] + healthcheck: + test: [CMD,curl,-f,http://127.0.0.1:9000/health] + interval: 2s + timeout: 2s + retries: 40 + networks: [private] + migrate: + image: mtuci-vp5:local + build: . + command: [-migrate] + cpus: 0.5 + mem_limit: 256m + secrets: [database_url] + depends_on: + postgres: {condition: service_healthy} + networks: [private] + bootstrap: + image: mtuci-vp5:local + build: . + command: [-bootstrap-s3] + cpus: 0.5 + mem_limit: 256m + secrets: [database_url,s3_root,s3_credentials] + depends_on: + s3: {condition: service_healthy} + networks: [private] + api: + <<: *api + deploy: {replicas: 2} + worker: + <<: *api + entrypoint: [/worker] + cpus: 1 + mem_limit: 2g + tmpfs: ['/tmp:uid=65532,gid=65532,size=256m,mode=1777'] + environment: + GOMEMLIMIT: 384MiB + WORKER_MAX_ATTEMPTS: 5 + lb: + image: haproxy:3.2-alpine@sha256:1c3568e520453cc8ba98c39501febcf450429c80e6cc27ecad779961d76e597c + cpus: 0.25 + mem_limit: 128m + ports: ['127.0.0.1:28205:8080'] + volumes: ['./config/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: + api: {condition: service_started} + networks: [private,edge] +networks: + private: {internal: true} + edge: {} +volumes: + postgres: {} + valkey: {} + s3: {} +secrets: + pg_password: {file: .local/pg_password} + database_url: {file: .local/database_url} + cache_url: {file: .local/cache_url} + valkey_conf: {file: .local/valkey.conf} + s3_access: {file: .local/s3_access} + s3_secret: {file: .local/s3_secret} + s3_root: {file: .local/s3_root} + s3_credentials: {file: .local/s3_credentials} + diff --git a/config/haproxy.cfg b/config/haproxy.cfg new file mode 100644 index 00000000..fe94b25a --- /dev/null +++ b/config/haproxy.cfg @@ -0,0 +1,27 @@ +global + log stdout format raw local0 + maxconn 256 +defaults + mode http + log global + timeout connect 1s + timeout client 20s + timeout server 25s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s +frontend public + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server inter 500ms fall 2 rise 1 + server-template replica 1-4 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/evidence/both-replicas.json b/evidence/both-replicas.json new file mode 100644 index 00000000..cf74c9cf --- /dev/null +++ b/evidence/both-replicas.json @@ -0,0 +1,19 @@ +{ + "checked_at": "2026-11-15T17:22:02.090Z", + "results": [ + { + "stopped_instance": "3e051955bcf1", + "surviving_instance": "168a40088e2e", + "session_preserved": true, + "confirmed_messages": 2, + "recovery_observed_ms": 1025 + }, + { + "stopped_instance": "168a40088e2e", + "surviving_instance": "3e051955bcf1", + "session_preserved": true, + "confirmed_messages": 4, + "recovery_observed_ms": 27 + } + ] +} diff --git a/evidence/both-replicas.txt b/evidence/both-replicas.txt new file mode 100644 index 00000000..0c07e086 --- /dev/null +++ b/evidence/both-replicas.txt @@ -0,0 +1,2 @@ +Running as unit: run-p1145148-i22068280.scope; invocation ID: 9e77b7f9f1504b5d92284f2490cd0a95 +{"checked_at":"2026-11-15T17:22:02.090Z","results":[{"stopped_instance":"3e051955bcf1","surviving_instance":"168a40088e2e","session_preserved":true,"confirmed_messages":2,"recovery_observed_ms":1025},{"stopped_instance":"168a40088e2e","surviving_instance":"3e051955bcf1","session_preserved":true,"confirmed_messages":4,"recovery_observed_ms":27}]} diff --git a/evidence/final-audit-build.txt b/evidence/final-audit-build.txt new file mode 100644 index 00000000..87592340 --- /dev/null +++ b/evidence/final-audit-build.txt @@ -0,0 +1,149 @@ +Running as unit: run-p1218114-i22148514.scope; invocation ID: 259d09db44e74bd38eead4c7940a0c94 +? backend/cmd/app [no test files] +? backend/cmd/worker [no test files] +? backend/db [no test files] +ok backend/integration (cached) +ok backend/internal/adapter/handler (cached) +ok backend/internal/adapter/imagefile (cached) +? backend/internal/adapter/password [no test files] +? backend/internal/adapter/queue [no test files] +ok backend/internal/adapter/repository/postgres 1.011s +? backend/internal/adapter/repository/postgres/queries [no test files] +ok backend/internal/adapter/storage (cached) +ok backend/internal/application/usecase 1.009s +ok backend/internal/config (cached) +ok backend/internal/domain/model (cached) +? backend/internal/domain/port [no test files] +? backend/internal/domain/repository [no test files] +? backend/pkg/logger [no test files] +? backend/pkg/server [no test files] +0 issues. + Image mtuci-vp5:local Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.53kB done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 839B done +#2 DONE 0.1s + +#3 resolve image config for docker-image://docker.io/docker/dockerfile:1 +#3 DONE 4.9s + +#4 docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e +#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e +#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.2s done +#4 CACHED + +#5 [internal] load metadata for docker.io/library/debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f +#5 ... + +#6 [internal] load metadata for docker.io/library/golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d +#6 DONE 1.2s + +#5 [internal] load metadata for docker.io/library/debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f +#5 DONE 1.3s + +#7 [internal] load .dockerignore +#7 transferring context: 115B done +#7 DONE 0.1s + +#8 [internal] load build context +#8 DONE 0.0s + +#9 [stage-1 1/4] FROM docker.io/library/debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f +#9 resolve docker.io/library/debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f +#9 resolve docker.io/library/debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f 0.4s done +#9 DONE 0.5s + +#10 [build 1/4] FROM docker.io/library/golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d +#10 resolve docker.io/library/golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d +#10 resolve docker.io/library/golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d 0.4s done +#10 DONE 0.5s + +#8 [internal] load build context +#8 transferring context: 55.42kB done +#8 DONE 0.1s + +#11 [build 2/4] WORKDIR /src +#11 CACHED + +#12 [build 3/4] COPY backend/ ./ +#12 DONE 0.7s + +#13 [build 4/4] RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -trimpath -ldflags='-s -w' -o /out/messenger ./cmd/app && go build -trimpath -ldflags='-s -w' -o /out/worker ./cmd/worker +#13 DONE 18.8s + +#14 [stage-1 2/4] RUN apt-get update && apt-get install -y --no-install-recommends imagemagick ca-certificates && apt-get clean +#14 CACHED + +#15 [stage-1 3/4] COPY --from=build /out/messenger /messenger +#15 DONE 0.9s + +#16 [stage-1 4/4] COPY --from=build /out/worker /worker +#16 DONE 1.1s + +#17 exporting to image +#17 exporting layers +#17 exporting layers 2.2s done +#17 exporting manifest sha256:11b531203ff037cef9fa60e89ebbf8b98c597cabb043a2f39d3452a00097d29a +#17 exporting manifest sha256:11b531203ff037cef9fa60e89ebbf8b98c597cabb043a2f39d3452a00097d29a 0.2s done +#17 exporting config sha256:6eb1958a181c51ce939df15c467beedb96a9c2b9629b355460d7bbfa1bab3824 +#17 exporting config sha256:6eb1958a181c51ce939df15c467beedb96a9c2b9629b355460d7bbfa1bab3824 0.2s done +#17 exporting attestation manifest sha256:82091c7efa4ace5668dfab0a8afe102c4292e83937d06c8c63c2ed0ab6cfb995 +#17 exporting attestation manifest sha256:82091c7efa4ace5668dfab0a8afe102c4292e83937d06c8c63c2ed0ab6cfb995 0.3s done +#17 exporting manifest list sha256:22d0a63d522e7a322c7ea0a0319be7cb1f303089bc1ab1c26e1238da36f06050 +#17 exporting manifest list sha256:22d0a63d522e7a322c7ea0a0319be7cb1f303089bc1ab1c26e1238da36f06050 0.2s done +#17 naming to docker.io/library/mtuci-vp5:local +#17 naming to docker.io/library/mtuci-vp5:local 0.0s done +#17 unpacking to docker.io/library/mtuci-vp5:local +#17 unpacking to docker.io/library/mtuci-vp5:local 0.7s done +#17 DONE 4.1s + +#18 resolving provenance for metadata file +#18 DONE 0.0s + Image mtuci-vp5:local Built +time="2026-11-15T20:29:02+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-postgres-1 Running + Container mtuci-vp5-medvedev-postgres-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Healthy + Container mtuci-vp5-medvedev-migrate-run-1e87f28d2b77 Creating + Container mtuci-vp5-medvedev-migrate-run-1e87f28d2b77 Created +2026/11/15 17:29:05 OK 00005_replay_audit.sql (534.27ms) +2026/11/15 17:29:05 goose: successfully migrated database to version: 5 +time="2026-11-15T20:29:07+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-postgres-1 Running + Container mtuci-vp5-medvedev-valkey-1 Running + Container mtuci-vp5-medvedev-s3-1 Running + Container mtuci-vp5-medvedev-migrate-1 Recreate + Container mtuci-vp5-medvedev-bootstrap-1 Recreate + Container mtuci-vp5-medvedev-migrate-1 Recreated + Container mtuci-vp5-medvedev-bootstrap-1 Recreated + Container mtuci-vp5-medvedev-worker-1 Recreate + Container mtuci-vp5-medvedev-api-2 Recreate + Container mtuci-vp5-medvedev-api-1 Recreate + Container mtuci-vp5-medvedev-worker-1 Recreated + Container mtuci-vp5-medvedev-api-2 Recreated + Container mtuci-vp5-medvedev-api-1 Recreated + Container mtuci-vp5-medvedev-s3-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Healthy + Container mtuci-vp5-medvedev-migrate-1 Starting + Container mtuci-vp5-medvedev-s3-1 Healthy + Container mtuci-vp5-medvedev-bootstrap-1 Starting + Container mtuci-vp5-medvedev-migrate-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Waiting + Container mtuci-vp5-medvedev-bootstrap-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Exited + Container mtuci-vp5-medvedev-migrate-1 Exited + Container mtuci-vp5-medvedev-bootstrap-1 Exited + Container mtuci-vp5-medvedev-api-1 Starting + Container mtuci-vp5-medvedev-bootstrap-1 Exited + Container mtuci-vp5-medvedev-worker-1 Starting + Container mtuci-vp5-medvedev-api-1 Started + Container mtuci-vp5-medvedev-api-2 Starting + Container mtuci-vp5-medvedev-api-2 Started + Container mtuci-vp5-medvedev-worker-1 Started diff --git a/evidence/final-checks.txt b/evidence/final-checks.txt new file mode 100644 index 00000000..b452950a --- /dev/null +++ b/evidence/final-checks.txt @@ -0,0 +1,59 @@ +Running as unit: run-p1353899-i22195833.scope; invocation ID: b3b71e390dd94959b46dd7bbc7897868 +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +? backend/cmd/app [no test files] +? backend/cmd/worker [no test files] +? backend/db [no test files] +ok backend/integration (cached) +ok backend/internal/adapter/handler (cached) +ok backend/internal/adapter/imagefile (cached) +? backend/internal/adapter/password [no test files] +? backend/internal/adapter/queue [no test files] +ok backend/internal/adapter/repository/postgres (cached) +? backend/internal/adapter/repository/postgres/queries [no test files] +ok backend/internal/adapter/storage (cached) +ok backend/internal/application/usecase (cached) +ok backend/internal/config (cached) +ok backend/internal/domain/model (cached) +? backend/internal/domain/port [no test files] +? backend/internal/domain/repository [no test files] +? backend/pkg/logger [no test files] +? backend/pkg/server [no test files] +0 issues. +time="2026-11-15T20:42:06+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-api-run-4077310676b4, mtuci-vp5-medvedev-worker-run-771b781db111, mtuci-vp5-medvedev-api-run-b290bf928939, mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-883dcfd1c83f Creating + Container mtuci-vp5-medvedev-worker-run-883dcfd1c83f Created +=== RUN TestOriginalBytesArePreserved +--- PASS: TestOriginalBytesArePreserved (0.00s) +=== RUN TestRejectsInvalidImageAndFilename +=== RUN TestRejectsInvalidImageAndFilename/image/jpeg� +=== RUN TestRejectsInvalidImageAndFilename/image/pngn +=== RUN TestRejectsInvalidImageAndFilename/application/octet-stream� +=== RUN TestRejectsInvalidImageAndFilename/image/png� +--- PASS: TestRejectsInvalidImageAndFilename (0.00s) + --- PASS: TestRejectsInvalidImageAndFilename/image/jpeg� (0.00s) + --- PASS: TestRejectsInvalidImageAndFilename/image/pngn (0.00s) + --- PASS: TestRejectsInvalidImageAndFilename/application/octet-stream� (0.00s) + --- PASS: TestRejectsInvalidImageAndFilename/image/png� (0.00s) +=== RUN TestRejectsOversizedStreamBeforeDecoding +--- PASS: TestRejectsOversizedStreamBeforeDecoding (0.01s) +=== RUN TestRejectsPixelBombBeforeDecoding +--- PASS: TestRejectsPixelBombBeforeDecoding (0.00s) +=== RUN TestWorstCaseImagesWithinBudget + stress_test.go:78: explicit isolated memory experiment required +--- SKIP: TestWorstCaseImagesWithinBudget (0.00s) +=== RUN TestActualVariantsHonorDimensionsAlphaAndEXIF +=== RUN TestActualVariantsHonorDimensionsAlphaAndEXIF/landscape +=== RUN TestActualVariantsHonorDimensionsAlphaAndEXIF/tiny +=== RUN TestActualVariantsHonorDimensionsAlphaAndEXIF/portrait +=== RUN TestActualVariantsHonorDimensionsAlphaAndEXIF/exif +--- PASS: TestActualVariantsHonorDimensionsAlphaAndEXIF (0.18s) + --- PASS: TestActualVariantsHonorDimensionsAlphaAndEXIF/landscape (0.12s) + --- PASS: TestActualVariantsHonorDimensionsAlphaAndEXIF/tiny (0.02s) + --- PASS: TestActualVariantsHonorDimensionsAlphaAndEXIF/portrait (0.02s) + --- PASS: TestActualVariantsHonorDimensionsAlphaAndEXIF/exif (0.02s) +=== RUN TestFitDimensionsPreservesAspectAndDoesNotUpscale +--- PASS: TestFitDimensionsPreservesAspectAndDoesNotUpscale (0.00s) +PASS diff --git a/evidence/final-runtime.txt b/evidence/final-runtime.txt new file mode 100644 index 00000000..3be7046c --- /dev/null +++ b/evidence/final-runtime.txt @@ -0,0 +1,37 @@ +Running as unit: run-p1232569-i22229982.scope; invocation ID: 161ac53c367c43708ce75fb300253e7d +Ready: 2 API process(es) + Container mtuci-vp5-medvedev-worker-1 Stopping + Container mtuci-vp5-medvedev-worker-1 Stopped +time="2026-11-15T20:29:47+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-9c6243811fa8 Creating + Container mtuci-vp5-medvedev-worker-run-9c6243811fa8 Created +=== RUN TestReplayAfterCompletedPreviousVersionCleanup +--- PASS: TestReplayAfterCompletedPreviousVersionCleanup (0.26s) +=== RUN TestVersionedPipelineDuplicateReorderRetryAndCleanup +--- PASS: TestVersionedPipelineDuplicateReorderRetryAndCleanup (2.53s) +PASS +time="2026-11-15T20:29:52+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-postgres-1 Running + Container mtuci-vp5-medvedev-valkey-1 Running + Container mtuci-vp5-medvedev-s3-1 Running + Container mtuci-vp5-medvedev-s3-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Healthy + Container mtuci-vp5-medvedev-migrate-1 Starting + Container mtuci-vp5-medvedev-s3-1 Healthy + Container mtuci-vp5-medvedev-bootstrap-1 Starting + Container mtuci-vp5-medvedev-migrate-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Started + Container mtuci-vp5-medvedev-migrate-1 Waiting + Container mtuci-vp5-medvedev-bootstrap-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Exited + Container mtuci-vp5-medvedev-bootstrap-1 Exited + Container mtuci-vp5-medvedev-worker-1 Starting + Container mtuci-vp5-medvedev-worker-1 Started +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:28205" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/resilience.py --mode persistence --action-script "./scripts/restart.sh" --base-url "http://127.0.0.1:28205" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/fresh-db-messenger_stage_70b6ccce.json b/evidence/fresh-db-messenger_stage_70b6ccce.json new file mode 100644 index 00000000..63fc4775 --- /dev/null +++ b/evidence/fresh-db-messenger_stage_70b6ccce.json @@ -0,0 +1 @@ +{"phase":"created_and_selected","database":"messenger_stage_70b6ccce","old_database":"messenger","container":"3b0bdf554f6f648b0846697eac2b2f20512205d5fd373742dcce7a608696c5c9","old_database_preserved":true} diff --git a/evidence/golangci-lint.txt b/evidence/golangci-lint.txt new file mode 100644 index 00000000..450ea8c0 --- /dev/null +++ b/evidence/golangci-lint.txt @@ -0,0 +1,6 @@ +Running as unit: run-p1209455-i22190718.scope; invocation ID: 788efe50bc3c434f9f3d4ab94cbb2bb3 +internal/application/usecase/pipeline_retry_test.go:23:2: field cleanup is unused (unused) + cleanup []model.Attachment + ^ +1 issues: +* unused: 1 diff --git a/evidence/green-replay-version2.txt b/evidence/green-replay-version2.txt new file mode 100644 index 00000000..781a7026 --- /dev/null +++ b/evidence/green-replay-version2.txt @@ -0,0 +1,6 @@ +time="2026-11-15T20:01:08+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-1db5a9894a25 Creating + Container mtuci-vp5-medvedev-worker-run-1db5a9894a25 Created +=== RUN TestReplayAfterCompletedPreviousVersionCleanup +--- PASS: TestReplayAfterCompletedPreviousVersionCleanup (0.24s) +PASS diff --git a/evidence/openapi-validate.txt b/evidence/openapi-validate.txt new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/openapi-validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/public-acceptance.txt b/evidence/public-acceptance.txt new file mode 100644 index 00000000..50ec55e4 --- /dev/null +++ b/evidence/public-acceptance.txt @@ -0,0 +1,10 @@ +Running as unit: run-p1084962-i22065896.scope; invocation ID: d1fe2b7306ca4399bdf630f35468acc2 +python3 tests/smoke.py --base-url "http://127.0.0.1:28205" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/resilience.py --mode persistence --action-script "./scripts/restart.sh" --base-url "http://127.0.0.1:28205" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +python3 tests/resilience.py --mode failover --action-script "./scripts/stop-one.sh" --base-url "http://127.0.0.1:28205" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 1, "first_recovery_seconds": 1.492} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/red-completed-cleanup.txt b/evidence/red-completed-cleanup.txt new file mode 100644 index 00000000..2878817b --- /dev/null +++ b/evidence/red-completed-cleanup.txt @@ -0,0 +1,6 @@ +=== RUN TestCompletedDeliveryRecoversStagingCleanupAfterCrash + pipeline_retry_test.go:49: completed redelivery did not recover cleanup: removed=0 durable_cleaned=0 status=ready +--- FAIL: TestCompletedDeliveryRecoversStagingCleanupAfterCrash (0.00s) +FAIL +FAIL backend/internal/application/usecase 0.008s +FAIL diff --git a/evidence/red-fit.txt b/evidence/red-fit.txt new file mode 100644 index 00000000..e9c307ab --- /dev/null +++ b/evidence/red-fit.txt @@ -0,0 +1,6 @@ +Command: go test ./internal/adapter/imagefile +Before FitDimensions implementation, on omarchy: +# backend/internal/adapter/imagefile [backend/internal/adapter/imagefile.test] +internal/adapter/imagefile/variants_test.go:7:10: undefined: FitDimensions +FAIL backend/internal/adapter/imagefile [build failed] +FAIL diff --git a/evidence/red-reconciler.txt b/evidence/red-reconciler.txt new file mode 100644 index 00000000..d5c34099 --- /dev/null +++ b/evidence/red-reconciler.txt @@ -0,0 +1,19 @@ +=== RUN TestReconcilerCannotPublishVersionedPipelineResult +=== RUN TestReconcilerCannotPublishVersionedPipelineResult/uploading + reconcile_pipeline_test.go:9: pipeline state was changed by legacy reconciler: status uploading -> ready, report {Metadata:1 Objects:1 Orphans:0 Missing:0 Recovered:1 Fresh:0 Deleted:0 Changed:1 NextMetadata: NextObject:} +=== RUN TestReconcilerCannotPublishVersionedPipelineResult/queued + reconcile_pipeline_test.go:9: pipeline state was changed by legacy reconciler: status queued -> ready, report {Metadata:1 Objects:1 Orphans:0 Missing:0 Recovered:1 Fresh:0 Deleted:0 Changed:1 NextMetadata: NextObject:} +=== RUN TestReconcilerCannotPublishVersionedPipelineResult/processing + reconcile_pipeline_test.go:9: pipeline state was changed by legacy reconciler: status processing -> ready, report {Metadata:1 Objects:1 Orphans:0 Missing:0 Recovered:1 Fresh:0 Deleted:0 Changed:1 NextMetadata: NextObject:} +=== RUN TestReconcilerCannotPublishVersionedPipelineResult/failed + reconcile_pipeline_test.go:9: pipeline state was changed by legacy reconciler: status failed -> ready, report {Metadata:1 Objects:1 Orphans:0 Missing:0 Recovered:1 Fresh:0 Deleted:0 Changed:1 NextMetadata: NextObject:} +=== RUN TestReconcilerCannotPublishVersionedPipelineResult/ready +--- FAIL: TestReconcilerCannotPublishVersionedPipelineResult (0.00s) + --- FAIL: TestReconcilerCannotPublishVersionedPipelineResult/uploading (0.00s) + --- FAIL: TestReconcilerCannotPublishVersionedPipelineResult/queued (0.00s) + --- FAIL: TestReconcilerCannotPublishVersionedPipelineResult/processing (0.00s) + --- FAIL: TestReconcilerCannotPublishVersionedPipelineResult/failed (0.00s) + --- PASS: TestReconcilerCannotPublishVersionedPipelineResult/ready (0.00s) +FAIL +FAIL backend/internal/application/usecase 0.003s +FAIL diff --git a/evidence/red-replay-audit.txt b/evidence/red-replay-audit.txt new file mode 100644 index 00000000..1550b4b7 --- /dev/null +++ b/evidence/red-replay-audit.txt @@ -0,0 +1,31 @@ +Running as unit: run-p1213525-i22184445.scope; invocation ID: bcb6a76b860c44cf98eea6bdd13aa1ac + Container mtuci-vp5-medvedev-worker-1 Stopping + Container mtuci-vp5-medvedev-worker-1 Stopped +time="2026-11-15T20:27:58+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-dcb0b433f76b Creating + Container mtuci-vp5-medvedev-worker-run-dcb0b433f76b Created +=== RUN TestReplayAfterCompletedPreviousVersionCleanup + pipeline_integration_test.go:125: explicit replay has no durable audit count/timestamp +--- FAIL: TestReplayAfterCompletedPreviousVersionCleanup (0.25s) +=== RUN TestVersionedPipelineDuplicateReorderRetryAndCleanup +--- PASS: TestVersionedPipelineDuplicateReorderRetryAndCleanup (2.34s) +FAIL + +time="2026-11-15T20:28:03+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-db8c2cb4f0d4, mtuci-vp5-medvedev-api-run-5ed09d6abf58, mtuci-vp5-medvedev-worker-run-b5fa7057152c, mtuci-vp5-medvedev-api-run-7ccf113a8771, mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-postgres-1 Running + Container mtuci-vp5-medvedev-valkey-1 Running + Container mtuci-vp5-medvedev-s3-1 Running + Container mtuci-vp5-medvedev-postgres-1 Waiting + Container mtuci-vp5-medvedev-s3-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Healthy + Container mtuci-vp5-medvedev-migrate-1 Starting + Container mtuci-vp5-medvedev-s3-1 Healthy + Container mtuci-vp5-medvedev-bootstrap-1 Starting + Container mtuci-vp5-medvedev-migrate-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Exited + Container mtuci-vp5-medvedev-bootstrap-1 Exited + Container mtuci-vp5-medvedev-worker-1 Starting + Container mtuci-vp5-medvedev-worker-1 Started diff --git a/evidence/red-replay-version2.txt b/evidence/red-replay-version2.txt new file mode 100644 index 00000000..a246f1cc --- /dev/null +++ b/evidence/red-replay-version2.txt @@ -0,0 +1,8 @@ +time="2026-11-15T19:59:14+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-5a18781dd27e Creating + Container mtuci-vp5-medvedev-worker-run-5a18781dd27e Created +=== RUN TestReplayAfterCompletedPreviousVersionCleanup + pipeline_integration_test.go:109: ready version1 staging cleanup permanently blocked replay of failed version2 despite retained original +--- FAIL: TestReplayAfterCompletedPreviousVersionCleanup (0.31s) +FAIL + diff --git a/evidence/red-transform-infrastructure.txt b/evidence/red-transform-infrastructure.txt new file mode 100644 index 00000000..483c6589 --- /dev/null +++ b/evidence/red-transform-infrastructure.txt @@ -0,0 +1,6 @@ +=== RUN TestInfrastructureTransformFailureIsRetried + pipeline_retry_test.go:38: temporary transform infrastructure failure became permanent: failed=true code=invalid_image +--- FAIL: TestInfrastructureTransformFailureIsRetried (0.00s) +FAIL +FAIL backend/internal/application/usecase 0.003s +FAIL diff --git a/evidence/replay-version2-command.md b/evidence/replay-version2-command.md new file mode 100644 index 00000000..33f8586b --- /dev/null +++ b/evidence/replay-version2-command.md @@ -0,0 +1,22 @@ +# PostgreSQL replay regression + +Рабочая папка: `/mnt/data/home/bench-saves/mtuci-vp-remaining-20261115/lab5` на omarchy. + +Тестовый бинарный файл собран нативным Go1.27.2, затем исполнялся в отдельном одноразовом worker-контейнере с теми же защищёнными *_FILE входами, что у стенда. Нормальный worker был остановлен на время регрессии. Полный тест вызывает настоящий PostgreSQLStore и удаляет только созданный им fixture в порядке внешних ключей. + +```sh +cd backend +CGO_ENABLED=0 /home/rey/.local/share/mise/installs/go/1.27.2/bin/go test -c -o ../pipeline-db.test ./internal/adapter/repository/postgres +cd .. +./scripts/compose.sh run --rm --no-deps --entrypoint /pipeline-db.test -e RUN_PIPELINE_DB_TESTS=1 -v "$PWD/pipeline-db.test:/pipeline-db.test:ro" worker -test.v -test.run '^TestReplayAfterCompletedPreviousVersionCleanup$' +``` + +RED: `red-replay-version2.txt`, FAIL0.31s после реального ready v1→cleanup→advance v2→failed→replay: отказ при сохранённом original. GREEN: `green-replay-version2.txt`, PASS0.24s после исправления условий staging_expired и regenerated SQLC mapper. Это регрессия реального SQL/Store, не E2E worker version advancement. + +SHA256 проверенного GREEN среза: + +- db/sqlc/queries/pipeline.sql: `c2a207f6820c897ed9747221867bf8610fd40f2f1177d37d8249b0674f6f638b` +- internal/adapter/repository/postgres/queries/pipeline.sql.go: `08f4c3bd1b5b3cd09bb42f0105e17b3f0f5c4d3264bc250daf3225fdc20633f5` +- internal/adapter/repository/postgres/pipeline_integration_test.go: `ee1fcf19cf5b98a4167a416d911d27ea408d25dea4929c606d6c04873adfa2f0` + +Новых commit для решения пока нет. SHA относятся к исходникам на момент проверки, а не к skeleton commit. diff --git a/evidence/runtime-commit-publish-gap.txt b/evidence/runtime-commit-publish-gap.txt new file mode 100644 index 00000000..1ae5f356 --- /dev/null +++ b/evidence/runtime-commit-publish-gap.txt @@ -0,0 +1,2 @@ +Running as unit: run-p1196184-i22152379.scope; invocation ID: 6f85d08018534a5abf698f139cde0057 +{"checked_at":"2026-11-15T17:26:48.979Z","scenario":"real crash after_original, actual reconciler apply, lease recovery and durable cleanup","metadata_age_injected_seconds":120,"attachment_id":"f3d0b97d-2c3b-4b0d-abb1-019289ae67a6","worker_exit":88,"commit_publish_gap_restart":true,"status_after_reconciler":"processing","variants_after_reconciler":0,"thumbnail_before_recovery":409,"final_status":"ready","final_variants":3,"original_exact":true,"staging_cleaned":true} diff --git a/evidence/runtime-pipeline-faults-after_commit.json b/evidence/runtime-pipeline-faults-after_commit.json new file mode 100644 index 00000000..fe6a287f --- /dev/null +++ b/evidence/runtime-pipeline-faults-after_commit.json @@ -0,0 +1,14 @@ +{ + "checked_at": "2026-11-15T17:15:15.202Z", + "scenario": "real crash after_commit, actual reconciler apply, lease recovery and durable cleanup", + "metadata_age_injected_seconds": 120, + "attachment_id": "960da673-5397-4d02-8188-4c168757fe0d", + "worker_exit": 88, + "status_after_reconciler": "ready", + "variants_after_reconciler": 3, + "thumbnail_before_recovery": 200, + "final_status": "ready", + "final_variants": 3, + "original_exact": true, + "staging_cleaned": true +} diff --git a/evidence/runtime-pipeline-faults-after_objects.json b/evidence/runtime-pipeline-faults-after_objects.json new file mode 100644 index 00000000..ee4b28da --- /dev/null +++ b/evidence/runtime-pipeline-faults-after_objects.json @@ -0,0 +1,14 @@ +{ + "checked_at": "2026-11-15T17:16:01.349Z", + "scenario": "real crash after_objects, actual reconciler apply, lease recovery and durable cleanup", + "metadata_age_injected_seconds": 120, + "attachment_id": "5c98fde4-f4d9-4b99-9f99-2c08a603ad90", + "worker_exit": 88, + "status_after_reconciler": "processing", + "variants_after_reconciler": 0, + "thumbnail_before_recovery": 409, + "final_status": "ready", + "final_variants": 3, + "original_exact": true, + "staging_cleaned": true +} diff --git a/evidence/runtime-pipeline-faults-after_original-restart-gap.json b/evidence/runtime-pipeline-faults-after_original-restart-gap.json new file mode 100644 index 00000000..885ff692 --- /dev/null +++ b/evidence/runtime-pipeline-faults-after_original-restart-gap.json @@ -0,0 +1,15 @@ +{ + "checked_at": "2026-11-15T17:26:48.979Z", + "scenario": "real crash after_original, actual reconciler apply, lease recovery and durable cleanup", + "metadata_age_injected_seconds": 120, + "attachment_id": "f3d0b97d-2c3b-4b0d-abb1-019289ae67a6", + "worker_exit": 88, + "commit_publish_gap_restart": true, + "status_after_reconciler": "processing", + "variants_after_reconciler": 0, + "thumbnail_before_recovery": 409, + "final_status": "ready", + "final_variants": 3, + "original_exact": true, + "staging_cleaned": true +} diff --git a/evidence/runtime-pipeline-faults.json b/evidence/runtime-pipeline-faults.json new file mode 100644 index 00000000..531af1a0 --- /dev/null +++ b/evidence/runtime-pipeline-faults.json @@ -0,0 +1,14 @@ +{ + "checked_at": "2026-11-15T17:06:43.004Z", + "scenario": "real crash after_original, actual reconciler apply, lease recovery and durable cleanup", + "metadata_age_injected_seconds": 120, + "attachment_id": "6615ccea-cdbe-4ed7-b45f-34f9ba1fe71a", + "worker_exit": 88, + "status_after_reconciler": "processing", + "variants_after_reconciler": 0, + "thumbnail_before_recovery": 409, + "final_status": "ready", + "final_variants": 3, + "original_exact": true, + "staging_cleaned": true +} diff --git a/evidence/runtime-reconciler-red.json b/evidence/runtime-reconciler-red.json new file mode 100644 index 00000000..6cb1ab82 --- /dev/null +++ b/evidence/runtime-reconciler-red.json @@ -0,0 +1,10 @@ +{ + "checked_at": "2026-11-15T16:50:07.757Z", + "scenario": "real worker crash after_original, artificial metadata age, actual legacy CLI reconcile apply", + "attachment_id": "68c7e769-1ab5-41e1-80bf-42b651a5ed39", + "worker_exit": 88, + "status": "ready", + "variants": 0, + "thumbnail_http": 404, + "red_confirmed": true +} diff --git a/evidence/validate-dependencies.txt b/evidence/validate-dependencies.txt new file mode 100644 index 00000000..422dab06 --- /dev/null +++ b/evidence/validate-dependencies.txt @@ -0,0 +1,62 @@ +Collecting openapi-spec-validator==0.7.2 (from -r requirements-dev.txt (line 2)) + Using cached openapi_spec_validator-0.7.2-py3-none-any.whl.metadata (5.7 kB) +Collecting jsonschema<5.0.0,>=4.18.0 (from openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached jsonschema-4.26.0-py3-none-any.whl.metadata (7.6 kB) +Collecting jsonschema-path<0.4.0,>=0.3.1 (from openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached jsonschema_path-0.3.4-py3-none-any.whl.metadata (4.3 kB) +Collecting lazy-object-proxy<2.0.0,>=1.7.1 (from openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl.metadata (5.1 kB) +Collecting openapi-schema-validator<0.7.0,>=0.6.0 (from openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached openapi_schema_validator-0.6.3-py3-none-any.whl.metadata (5.4 kB) +Collecting attrs>=22.2.0 (from jsonschema<5.0.0,>=4.18.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached attrs-26.1.0-py3-none-any.whl.metadata (8.8 kB) +Collecting jsonschema-specifications>=2023.03.6 (from jsonschema<5.0.0,>=4.18.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB) +Collecting referencing>=0.28.4 (from jsonschema<5.0.0,>=4.18.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached referencing-0.37.0-py3-none-any.whl.metadata (2.8 kB) +Collecting rpds-py>=0.25.0 (from jsonschema<5.0.0,>=4.18.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB) +Collecting PyYAML>=5.1 (from jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB) +Collecting pathable<0.5.0,>=0.4.1 (from jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached pathable-0.4.4-py3-none-any.whl.metadata (1.8 kB) +Collecting referencing>=0.28.4 (from jsonschema<5.0.0,>=4.18.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached referencing-0.36.2-py3-none-any.whl.metadata (2.8 kB) +Collecting requests<3.0.0,>=2.31.0 (from jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached requests-2.34.2-py3-none-any.whl.metadata (4.8 kB) +Collecting rfc3339-validator (from openapi-schema-validator<0.7.0,>=0.6.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl.metadata (1.5 kB) +Collecting charset_normalizer<4,>=2 (from requests<3.0.0,>=2.31.0->jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (46 kB) +Collecting idna<4,>=2.5 (from requests<3.0.0,>=2.31.0->jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached idna-3.20-py3-none-any.whl.metadata (7.2 kB) +Collecting urllib3<3,>=1.26 (from requests<3.0.0,>=2.31.0->jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached urllib3-2.8.0-py3-none-any.whl.metadata (7.4 kB) +Collecting certifi>=2023.5.7 (from requests<3.0.0,>=2.31.0->jsonschema-path<0.4.0,>=0.3.1->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached certifi-2026.10.23-py3-none-any.whl.metadata (2.5 kB) +Collecting six (from rfc3339-validator->openapi-schema-validator<0.7.0,>=0.6.0->openapi-spec-validator==0.7.2->-r requirements-dev.txt (line 2)) + Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) +Using cached openapi_spec_validator-0.7.2-py3-none-any.whl (39 kB) +Using cached jsonschema-4.26.0-py3-none-any.whl (90 kB) +Using cached jsonschema_path-0.3.4-py3-none-any.whl (14 kB) +Using cached lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (70 kB) +Using cached openapi_schema_validator-0.6.3-py3-none-any.whl (8.8 kB) +Using cached pathable-0.4.4-py3-none-any.whl (9.6 kB) +Using cached referencing-0.36.2-py3-none-any.whl (26 kB) +Using cached requests-2.34.2-py3-none-any.whl (73 kB) +Using cached charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (255 kB) +Using cached idna-3.20-py3-none-any.whl (69 kB) +Using cached urllib3-2.8.0-py3-none-any.whl (135 kB) +Using cached attrs-26.1.0-py3-none-any.whl (67 kB) +Using cached certifi-2026.10.23-py3-none-any.whl (135 kB) +Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB) +Using cached pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (794 kB) +Using cached rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (371 kB) +Using cached rfc3339_validator-0.1.4-py2.py3-none-any.whl (3.5 kB) +Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Installing collected packages: urllib3, six, rpds-py, PyYAML, pathable, lazy-object-proxy, idna, charset_normalizer, certifi, attrs, rfc3339-validator, requests, referencing, jsonschema-specifications, jsonschema-path, jsonschema, openapi-schema-validator, openapi-spec-validator + +Successfully installed PyYAML-6.0.3 attrs-26.1.0 certifi-2026.10.23 charset_normalizer-3.5.2 idna-3.20 jsonschema-4.26.0 jsonschema-path-0.3.4 jsonschema-specifications-2025.9.1 lazy-object-proxy-1.13.0 openapi-schema-validator-0.6.3 openapi-spec-validator-0.7.2 pathable-0.4.4 referencing-0.36.2 requests-2.34.2 rfc3339-validator-0.1.4 rpds-py-2026.9.1 six-1.17.0 urllib3-2.8.0 + +[notice] A new release of pip is available: 26.2.1 -> 26.3 +[notice] To update, run: /mnt/data/home/bench-saves/mtuci-vp-remaining-20261115/lab5/.venv/bin/python -m pip install --upgrade pip diff --git a/evidence/versioned-fault-matrix.txt b/evidence/versioned-fault-matrix.txt new file mode 100644 index 00000000..fc153d85 --- /dev/null +++ b/evidence/versioned-fault-matrix.txt @@ -0,0 +1,29 @@ +Running as unit: run-p1182373-i22171722.scope; invocation ID: 50022225edb240eabfc5bf827389b15e + Container mtuci-vp5-medvedev-worker-1 Stopping + Container mtuci-vp5-medvedev-worker-1 Stopped +time="2026-11-15T20:24:55+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-worker-run-46e306aac98e Creating + Container mtuci-vp5-medvedev-worker-run-46e306aac98e Created +=== RUN TestReplayAfterCompletedPreviousVersionCleanup +--- PASS: TestReplayAfterCompletedPreviousVersionCleanup (0.25s) +=== RUN TestVersionedPipelineDuplicateReorderRetryAndCleanup +--- PASS: TestVersionedPipelineDuplicateReorderRetryAndCleanup (2.41s) +PASS +time="2026-11-15T20:25:00+03:00" level=warning msg="Found orphan containers (mtuci-vp5-medvedev-worker-run-d4f26bcfc148, mtuci-vp5-medvedev-worker-run-a2389dc840d1, mtuci-vp5-medvedev-worker-run-6c8f0bb601f5, mtuci-vp5-medvedev-worker-run-b9be14682b27) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up." + Container mtuci-vp5-medvedev-postgres-1 Running + Container mtuci-vp5-medvedev-valkey-1 Running + Container mtuci-vp5-medvedev-s3-1 Running + Container mtuci-vp5-medvedev-s3-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Waiting + Container mtuci-vp5-medvedev-postgres-1 Healthy + Container mtuci-vp5-medvedev-migrate-1 Starting + Container mtuci-vp5-medvedev-s3-1 Healthy + Container mtuci-vp5-medvedev-bootstrap-1 Starting + Container mtuci-vp5-medvedev-migrate-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Started + Container mtuci-vp5-medvedev-bootstrap-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Waiting + Container mtuci-vp5-medvedev-migrate-1 Exited + Container mtuci-vp5-medvedev-bootstrap-1 Exited + Container mtuci-vp5-medvedev-worker-1 Starting + Container mtuci-vp5-medvedev-worker-1 Started diff --git a/scripts/bootstrap.ts b/scripts/bootstrap.ts new file mode 100644 index 00000000..4b689287 --- /dev/null +++ b/scripts/bootstrap.ts @@ -0,0 +1,35 @@ +import { mkdir, writeFile, chmod, access } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +const root = resolve(import.meta.dir, '..'); +const local = resolve(root, '.local'); +const files = ['pg_password', 'database_url', 'cache_url', 'valkey.conf']; +const present = await Promise.all(files.map(async name => {try {await access(resolve(local, name));return true;} catch {return false;}})); +if (present.some(Boolean)) { + if (!present.every(Boolean)) throw new Error('Incomplete local configuration; preserve it and investigate before bootstrap'); + console.log('Local configuration exists'); +} else { + await mkdir(local, { recursive: true, mode: 0o700 }); + const pg = crypto.randomUUID().replaceAll('-', '') + crypto.randomUUID().replaceAll('-', ''); + const cache = crypto.randomUUID().replaceAll('-', ''); + const values = [pg, `postgres://messenger:${pg}@postgres:5432/messenger?sslmode=disable`, `redis://:${cache}@valkey:6379/0`, `bind 0.0.0.0\nprotected-mode yes\nrequirepass ${cache}\nappendonly yes\nappendfsync always\ndir /data\n`]; + for (let i = 0; i < files.length; i++) { + const destination = resolve(local, files[i]); + await writeFile(destination, values[i] + '\n', { mode: 0o600, flag: 'wx' }); + await chmod(destination, 0o644); // Compose secrets must be readable by nonroot containers; parent is 0700. + } + console.log('Created local synthetic stand credentials'); +} + +const s3Files=['s3_access','s3_secret','s3_root','s3_credentials']; +const existing=await Promise.all(s3Files.map(async name=>{try{await access(resolve(local,name));return true;}catch{return false;}})); +if(existing.some(Boolean)&&!existing.every(Boolean))throw new Error('Incomplete S3 configuration; preserve it before investigation'); +if(!existing.some(Boolean)){ + const accessKey='root_'+crypto.randomUUID().replaceAll('-',''); + const rootSecret=crypto.randomUUID().replaceAll('-','')+crypto.randomUUID().replaceAll('-',''); + const appSecret=crypto.randomUUID().replaceAll('-','')+crypto.randomUUID().replaceAll('-',''); + const common={Endpoint:'s3:9000',Bucket:'messenger'}; + const values=[accessKey,rootSecret,JSON.stringify({...common,Access:accessKey,Secret:rootSecret}),JSON.stringify({...common,Access:'messenger-api',Secret:appSecret})]; + for(let i=0;ihistory.some((m:any)=>m.id===x)))throw new Error('confirmed message missing');results.push({stopped_instance:alias,surviving_instance:surviving,session_preserved:true,confirmed_messages:confirmed.length,recovery_observed_ms:Math.round(performance.now()-started)}); +}finally{compose('up','-d','api');await ready()}} +const result={checked_at:new Date().toISOString(),results};await Bun.write(resolve(root,'evidence/both-replicas.json'),JSON.stringify(result,null,2)+'\n');console.log(JSON.stringify(result)); diff --git a/student-tests/pipeline-faults.ts b/student-tests/pipeline-faults.ts new file mode 100644 index 00000000..9562dbce --- /dev/null +++ b/student-tests/pipeline-faults.ts @@ -0,0 +1,34 @@ +import {resolve} from 'node:path'; +const root=resolve(import.meta.dir,'..');const base='http://127.0.0.1:28205'; +const point=Bun.argv.find(x=>x.startsWith('--point='))?.slice(8)??'after_original';if(!['after_original','after_objects','after_commit'].includes(point))throw new Error('unknown fault point'); +if(Bun.argv.includes('--dry-run')){console.log('Valid pipeline fault-test invocation; no resources or credentials read');process.exit(0);} +function run(args:string[]){const r=Bun.spawnSync(args,{cwd:root,stdout:'pipe',stderr:'pipe'});if(r.exitCode!==0){const file=resolve(root,'evidence/fault-command-error.txt');require('node:fs').writeFileSync(file,r.stderr,{mode:0o600});throw new Error('command failed; private stderr retained: '+file);}return new TextDecoder().decode(r.stdout).trim();} +function compose(...args:string[]){return run([resolve(root,'scripts/compose.sh'),...args]);} +const pg=compose('ps','-q','postgres'),api=compose('ps','-q','api').split('\n')[0]; +if(run(['docker','inspect',pg,'--format','{{index .Config.Labels "com.docker.compose.project"}}'])!=='mtuci-vp5-medvedev')throw new Error('wrong project'); +const database=new URL((await Bun.file(resolve(root,'.local/database_url')).text()).trim()).pathname.slice(1);if(!/^messenger(?:_stage_[a-f0-9]{8})?$/.test(database))throw new Error('unexpected test database'); +try{ +compose('stop','worker'); +const prefix='red_'+crypto.randomUUID().slice(0,8),password=crypto.randomUUID(); +async function req(path:string,method='GET',body?:unknown,token?:string,extra:Record={}){const headers:Record={...extra};if(token)headers.Authorization='Bearer '+token;if(body&&!(body instanceof Uint8Array))headers['Content-Type']='application/json';const r=await fetch(base+path,{method,headers,body:body instanceof Uint8Array?body:body===undefined?undefined:JSON.stringify(body),signal:AbortSignal.timeout(5000)});const text=await r.text();return {status:r.status,body:text&&r.headers.get('content-type')?.includes('application/json')?JSON.parse(text):null};} +let x=await req('/api/v1/users','POST',{username:prefix,display_name:'Synthetic RED',password});if(x.status!==201)throw new Error('registration failed'); +let s=await req('/api/v1/auth/sessions','POST',undefined,undefined,{Authorization:'Basic '+btoa(prefix+':'+password)});if(s.status!==201)throw new Error('session failed');const token=s.body.token; +let c=await req('/api/v1/chats','POST',{title:'Owned reconciler crash fixture',member_ids:[s.body.user.id]},token);if(c.status!==201)throw new Error('chat failed'); +let a=await req('/api/v1/chats/'+c.body.id+'/attachments?filename=sample.png','POST',new Uint8Array(await Bun.file(resolve(root,'tests/fixtures/sample.png')).arrayBuffer()),token,{'Content-Type':'image/png'});if(a.status!==202)throw new Error('upload failed');const id=a.body.id;if(!/^[a-f0-9-]{36}$/.test(id))throw new Error('bad id'); +const gap=Bun.argv.includes('--restart-gap'); +if(gap){const pending=await req('/api/v1/attachments/'+id,'GET',undefined,token);const unavailable=await req('/api/v1/attachments/'+id+'/content','GET',undefined,token);if(pending.body.status!=='queued'||unavailable.status!==409)throw new Error('stopped worker did not leave durable queued attachment');const message=await req('/api/v1/chats/'+c.body.id+'/messages','POST',{text:'Attachment accepted before publisher restart',attachment_ids:[id]},token,{'Idempotency-Key':crypto.randomUUID()});if(message.status!==201)throw new Error('messages blocked while worker stopped');const unpublished=run(['docker','exec',pg,'psql','-U','messenger','-d',database,'-At','-c',`SELECT count(*) FROM attachment_jobs WHERE attachment_id='${id}'::uuid AND published_at='epoch'`]);if(unpublished!=='1')throw new Error('fault not located inside COMMIT/publish gap');compose('restart','api','valkey');run([process.execPath,resolve(root,'student-tests/wait.ts')]);} +const fault=compose('run','-d','--no-deps','worker','-fault',point).split('\n').at(-1)!;let exit:number|null=null; +for(let i=0;i<200;i++){const state=JSON.parse(run(['docker','inspect',fault,'--format','{{json .State}}']));if(state.Status==='exited'){exit=state.ExitCode;break;}await Bun.sleep(200);}if(exit!==88)throw new Error('real worker crash not observed'); +const aged=run(['docker','exec',pg,'psql','-U','messenger','-d',database,'-At','-v','ON_ERROR_STOP=1','-c',`UPDATE attachments SET created_at=clock_timestamp()-interval '2 minutes' WHERE id='${id}'::uuid RETURNING id`]);if(!aged.includes(id))throw new Error('metadata age injection did not target the accepted attachment'); +run(['docker','exec',api,'/messenger','-reconcile','-apply','-grace','1m']); +const meta=await req('/api/v1/attachments/'+id,'GET',undefined,token);const thumb=await req('/api/v1/attachments/'+id+'/content?variant=thumbnail','GET',undefined,token); +const committed=point==='after_commit';if(meta.body.status!==(committed?'ready':'processing')||meta.body.variants.length!==(committed?3:0)||thumb.status!==(committed?200:409))throw new Error('legacy reconciliation changed the expected pipeline result'); +compose('up','-d','worker'); +let final=meta;for(let i=0;i<400;i++){final=await req('/api/v1/attachments/'+id,'GET',undefined,token);if(final.body.status==='ready')break;if(final.body.status==='failed')throw new Error('recovery permanently failed');await Bun.sleep(150);} +if(final.body.status!=='ready'||final.body.variants.length!==3)throw new Error('recovery did not publish all required variants'); +const original=await fetch(base+'/api/v1/attachments/'+id+'/content',{headers:{Authorization:'Bearer '+token}});const bytes=new Uint8Array(await original.arrayBuffer());const expected=new Uint8Array(await Bun.file(resolve(root,'tests/fixtures/sample.png')).arrayBuffer());if(original.status!==200||Buffer.compare(bytes,expected)!==0)throw new Error('original changed'); +let cleaned=false;for(let i=0;i<100;i++){cleaned=run(['docker','exec',pg,'psql','-U','messenger','-d',new URL((await Bun.file(resolve(root,'.local/database_url')).text()).trim()).pathname.slice(1),'-At','-c',`SELECT staging_cleaned FROM attachments WHERE id='${id}'::uuid`])==='t';if(cleaned)break;await Bun.sleep(100);} +if(!cleaned)throw new Error('durable staging cleanup did not finish'); +const result={checked_at:new Date().toISOString(),scenario:'real crash '+point+', actual reconciler apply, lease recovery and durable cleanup',metadata_age_injected_seconds:120,attachment_id:id,worker_exit:exit,commit_publish_gap_restart:gap,status_after_reconciler:meta.body.status,variants_after_reconciler:meta.body.variants.length,thumbnail_before_recovery:thumb.status,final_status:final.body.status,final_variants:final.body.variants.length,original_exact:true,staging_cleaned:cleaned}; +await Bun.write(resolve(root,'evidence/runtime-pipeline-faults-'+point+(gap?'-restart-gap':'')+'.json'),JSON.stringify(result,null,2)+'\n');console.log(JSON.stringify(result)); +}finally{compose('up','-d','worker');} diff --git a/student-tests/reconciler-red.ts b/student-tests/reconciler-red.ts new file mode 100644 index 00000000..6920e30d --- /dev/null +++ b/student-tests/reconciler-red.ts @@ -0,0 +1,20 @@ +import {resolve} from 'node:path'; +const root=resolve(import.meta.dir,'..');const base='http://127.0.0.1:28205'; +function run(args:string[]){const r=Bun.spawnSync(args,{cwd:root,stdout:'pipe',stderr:'pipe'});if(r.exitCode!==0)throw new Error('command failed: '+args.slice(0,3).join(' '));return new TextDecoder().decode(r.stdout).trim();} +function compose(...args:string[]){return run([resolve(root,'scripts/compose.sh'),...args]);} +const pg=compose('ps','-q','postgres'),api=compose('ps','-q','api').split('\n')[0]; +if(run(['docker','inspect',pg,'--format','{{index .Config.Labels "com.docker.compose.project"}}'])!=='mtuci-vp5-medvedev')throw new Error('wrong project'); +compose('stop','worker'); +const prefix='red_'+crypto.randomUUID().slice(0,8),password=crypto.randomUUID(); +async function req(path:string,method='GET',body?:unknown,token?:string,extra:Record={}){const headers:Record={...extra};if(token)headers.Authorization='Bearer '+token;if(body&&!(body instanceof Uint8Array))headers['Content-Type']='application/json';const r=await fetch(base+path,{method,headers,body:body instanceof Uint8Array?body:body===undefined?undefined:JSON.stringify(body),signal:AbortSignal.timeout(5000)});const text=await r.text();return {status:r.status,body:text?JSON.parse(text):null};} +let x=await req('/api/v1/users','POST',{username:prefix,display_name:'Synthetic RED',password});if(x.status!==201)throw new Error('registration failed'); +let s=await req('/api/v1/auth/sessions','POST',undefined,undefined,{Authorization:'Basic '+btoa(prefix+':'+password)});if(s.status!==201)throw new Error('session failed');const token=s.body.token; +let c=await req('/api/v1/chats','POST',{title:'Owned reconciler crash fixture',member_ids:[s.body.user.id]},token);if(c.status!==201)throw new Error('chat failed'); +let a=await req('/api/v1/chats/'+c.body.id+'/attachments?filename=sample.png','POST',new Uint8Array(await Bun.file(resolve(root,'tests/fixtures/sample.png')).arrayBuffer()),token,{'Content-Type':'image/png'});if(a.status!==202)throw new Error('upload failed');const id=a.body.id;if(!/^[a-f0-9-]{36}$/.test(id))throw new Error('bad id'); +const fault=compose('run','-d','--no-deps','worker','-fault','after_original').split('\n').at(-1)!;let exit:number|null=null; +for(let i=0;i<200;i++){const state=JSON.parse(run(['docker','inspect',fault,'--format','{{json .State}}']));if(state.Status==='exited'){exit=state.ExitCode;break;}await Bun.sleep(200);}if(exit!==88)throw new Error('real worker crash not observed'); +run(['docker','exec',pg,'psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',`UPDATE attachments SET created_at=clock_timestamp()-interval '2 minutes' WHERE id='${id}'::uuid`]); +run(['docker','exec',api,'/messenger','-reconcile','-apply','-grace','1m']); +const meta=await req('/api/v1/attachments/'+id,'GET',undefined,token);const thumb=await req('/api/v1/attachments/'+id+'/content?variant=thumbnail','GET',undefined,token); +const result={checked_at:new Date().toISOString(),scenario:'real worker crash after_original, artificial metadata age, actual legacy CLI reconcile apply',attachment_id:id,worker_exit:exit,status:meta.body.status,variants:meta.body.variants.length,thumbnail_http:thumb.status,red_confirmed:meta.body.status==='ready'&&meta.body.variants.length===0&&thumb.status===404}; +await Bun.write(resolve(root,'evidence/runtime-reconciler-red.json'),JSON.stringify(result,null,2)+'\n');console.log(JSON.stringify(result));if(!result.red_confirmed)throw new Error('expected regression not reproduced'); diff --git a/student-tests/run-db-matrix.sh b/student-tests/run-db-matrix.sh new file mode 100755 index 00000000..40dd2bcf --- /dev/null +++ b/student-tests/run-db-matrix.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +GO=${GO:-go} +cd "$ROOT/backend" +gofmt -w internal/adapter/repository/postgres/pipeline_integration_test.go +CGO_ENABLED=0 "$GO" test -c -o "$ROOT/pipeline-db.test" ./internal/adapter/repository/postgres +cd "$ROOT" +"$ROOT/scripts/compose.sh" stop worker +trap '"$ROOT/scripts/compose.sh" up -d worker' EXIT +"$ROOT/scripts/compose.sh" run --rm --no-deps --entrypoint /pipeline-db.test -e RUN_PIPELINE_DB_TESTS=1 -v "$ROOT/pipeline-db.test:/pipeline-db.test:ro" worker -test.v -test.run '^Test(ReplayAfterCompletedPreviousVersionCleanup|VersionedPipelineDuplicateReorderRetryAndCleanup)$' diff --git a/student-tests/wait.ts b/student-tests/wait.ts new file mode 100644 index 00000000..e2cb0cf0 --- /dev/null +++ b/student-tests/wait.ts @@ -0,0 +1,14 @@ +import {resolve} from 'node:path'; +const root=resolve(import.meta.dir,'..'); +const lab=(await Bun.file(resolve(root,'lab.json')).json()).number; +const base=process.env.BASE_URL??`http://127.0.0.1:${28200+lab}`; +const needed=lab>=3?Number(process.env.API_REPLICAS??2):1; +const instances=new Set(); +for(let attempt=0;attempt<600;attempt++){ + try{ + const r=await fetch(base+'/health/ready',{signal:AbortSignal.timeout(1000)});await r.arrayBuffer(); + if(r.status===200){instances.add(r.headers.get('X-Instance-Id')??'single');if(instances.size>=needed){console.log(`Ready: ${instances.size} API process(es)`);process.exit(0);}} + }catch{} + await Bun.sleep(150); +} +throw new Error('Stand readiness deadline expired'); -- 2.54.0