From 1f535f1b5b3e1ef79d948d77b94374ef51e41bda Mon Sep 17 00:00:00 2001 From: OledRood Date: Mon, 16 Nov 2026 02:38:13 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=205=20=E2=80=94=20=D0=B0=D1=81?= =?UTF-8?q?=D0=B8=D0=BD=D1=85=D1=80=D0=BE=D0=BD=D0=BD=D0=B0=D1=8F=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B0=20=D0=B8=20?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 7 +- .github/workflows/application.yml | 21 + Dockerfile | 45 ++ Dockerfile.migrate | 20 + Dockerfile.worker | 23 + REPORT.md | 156 ++++- cmd/api/main.go | 155 +++++ cmd/api/poolstats.go | 29 + cmd/attachment-cleanup/main.go | 164 +++++ cmd/jobs/main.go | 133 ++++ cmd/migrate/async_attachments_test.go | 254 ++++++++ cmd/migrate/main.go | 152 +++++ cmd/migrate/main_test.go | 111 ++++ cmd/migrate/postgres_test.go | 212 +++++++ cmd/migrate/roundtrip_test.go | 167 +++++ cmd/relay/main.go | 113 ++++ cmd/worker/main.go | 58 ++ compose.yaml | 298 +++++++++ db/migrations/00001_init_schema.sql | 79 +++ .../00002_add_created_at_defaults.sql | 20 + db/migrations/00003_drop_session_table.sql | 17 + db/migrations/00004_message_idempotency.sql | 23 + db/migrations/00005_attachments.sql | 54 ++ .../00006_allow_attachment_only_messages.sql | 18 + db/migrations/00007_async_attachments.sql | 144 +++++ db/migrations/00008_versioned_pipeline.sql | 27 + db/migrations/embed.go | 9 + db/queries/attachment_variants.sql | 11 + db/queries/attachments.sql | 69 +++ db/queries/attachments_jobs.sql | 197 ++++++ db/queries/chat_members.sql | 3 + db/queries/chats.sql | 34 ++ db/queries/message_idempotence.sql | 22 + db/queries/messages.sql | 19 + db/queries/messages_attachments.sql | 16 + db/queries/outbox.sql | 39 ++ db/queries/users.sql | 21 + db/queries/versioned_pipeline.sql | 30 + evidence/lab5/api-failover.jsonl | 7 + evidence/lab5/clean-run.md | 39 ++ evidence/lab5/crash-before-commit.jsonl | 1 + evidence/lab5/http-and-geometry.jsonl | 12 + evidence/lab5/pipeline-faults.jsonl | 64 ++ evidence/lab5/readiness.jsonl | 12 + .../lab5/track-2/concurrent-upgrade.jsonl | 1 + evidence/lab5/track-2/migration-live.json | 16 + evidence/lab5/track-2/regression.jsonl.gz | Bin 0 -> 4194 bytes evidence/lab5/track-2/summary.json | 77 +++ evidence/lab5/track-2/versioned.jsonl.gz | Bin 0 -> 32317 bytes evidence/lab5/track-2/versioned.txt | 17 + go.mod | 44 ++ go.sum | 107 ++++ infra/haproxy.cfg | 28 + infra/imagemagick/policy.xml | 16 + infra/s3/bootstrap.sh | 65 ++ internal/broker/rabbitmq.go | 119 ++++ internal/config/config.go | 180 ++++++ internal/config/pipeline.go | 77 +++ internal/database/postgres.go | 21 + internal/database/redis.go | 31 + .../database/sqlc/attachment_variants.sql.go | 88 +++ internal/database/sqlc/attachments.sql.go | 306 ++++++++++ .../database/sqlc/attachments_jobs.sql.go | 529 ++++++++++++++++ internal/database/sqlc/chat_members.sql.go | 27 + internal/database/sqlc/chats.sql.go | 118 ++++ internal/database/sqlc/db.go | 32 + .../database/sqlc/message_idempotence.sql.go | 99 +++ internal/database/sqlc/messages.sql.go | 91 +++ .../database/sqlc/messages_attachments.sql.go | 107 ++++ internal/database/sqlc/models.go | 124 ++++ internal/database/sqlc/outbox.sql.go | 111 ++++ internal/database/sqlc/users.sql.go | 77 +++ .../database/sqlc/versioned_pipeline.sql.go | 111 ++++ internal/fault/pause.go | 43 ++ internal/imageproc/processor.go | 164 +++++ internal/model/attachment.go | 31 + internal/model/attachment_job.go | 26 + internal/model/chat.go | 10 + internal/model/message.go | 12 + internal/model/session.go | 11 + internal/model/user.go | 11 + internal/pipeline/cleanup.go | 64 ++ internal/pipeline/worker.go | 353 +++++++++++ internal/repository/attachment.go | 36 ++ internal/repository/attachment_tx.go | 13 + internal/repository/chat.go | 18 + internal/repository/errors.go | 16 + internal/repository/message.go | 17 + internal/repository/postgres/attachment.go | 440 ++++++++++++++ internal/repository/postgres/chat.go | 163 +++++ internal/repository/postgres/errors.go | 46 ++ internal/repository/postgres/message.go | 191 ++++++ internal/repository/postgres/pipeline.go | 167 +++++ .../repository/postgres/pipeline_cleanup.go | 118 ++++ internal/repository/postgres/user.go | 85 +++ internal/repository/postgres/utils.go | 26 + .../repository/postgres/versioned_pipeline.go | 87 +++ internal/repository/redis/errors.go | 38 ++ internal/repository/redis/session.go | 127 ++++ internal/repository/session.go | 13 + internal/repository/user.go | 12 + internal/security/password.go | 143 +++++ internal/service/attachment.go | 207 +++++++ internal/service/attachment_cleanup.go | 98 +++ internal/service/attachment_reconcile.go | 162 +++++ internal/service/auth.go | 142 +++++ internal/service/chat.go | 194 ++++++ internal/service/errors.go | 63 ++ internal/service/helper.go | 63 ++ internal/service/message.go | 142 +++++ internal/service/user.go | 70 +++ internal/storage/errors.go | 9 + internal/storage/object_cleanup_storage.go | 16 + internal/storage/object_writer.go | 11 + internal/storage/s3/client.go | 160 +++++ internal/storage/s3/errors.go | 35 ++ internal/transport/http/handler/attachment.go | 159 +++++ .../http/handler/attachment_types.go | 54 ++ internal/transport/http/handler/auth.go | 94 +++ internal/transport/http/handler/auth_types.go | 15 + internal/transport/http/handler/chat.go | 102 ++++ internal/transport/http/handler/chat_types.go | 32 + internal/transport/http/handler/health.go | 86 +++ .../transport/http/handler/health_types.go | 5 + internal/transport/http/handler/message.go | 90 +++ .../transport/http/handler/message_types.go | 42 ++ internal/transport/http/handler/user.go | 60 ++ internal/transport/http/handler/user_types.go | 28 + internal/transport/http/httputils/helper.go | 207 +++++++ internal/transport/http/middleware/auth.go | 71 +++ internal/transport/http/middleware/context.go | 29 + internal/transport/http/middleware/logging.go | 39 ++ internal/transport/http/router.go | 57 ++ scripts/init-local.sh | 46 ++ scripts/restart.sh | 43 ++ scripts/stop-one.sh | 17 + sqlc.yaml | 11 + student-tests/auth_contract.py | 538 +++++++++++++++++ .../auth_tests/attachment_cleanup_test.go | 221 +++++++ .../auth_tests/attachment_tx_test.go | 340 +++++++++++ student-tests/auth_tests/auth_handler_test.go | 32 + student-tests/auth_tests/budgets_test.go | 209 +++++++ student-tests/auth_tests/errors_test.go | 273 +++++++++ .../auth_tests/logout_diagnostic_test.go | 202 +++++++ .../auth_tests/message_attachments_test.go | 570 ++++++++++++++++++ student-tests/auth_tests/password_test.go | 126 ++++ student-tests/auth_tests/redis_test.go | 455 ++++++++++++++ student-tests/auth_tests/restart_http_test.go | 121 ++++ student-tests/auth_tests/storage_test.go | 345 +++++++++++ student-tests/auth_tests/transport_test.go | 226 +++++++ student-tests/lab5/FAULT-HOOKS.md | 44 ++ student-tests/lab5/README.md | 91 +++ student-tests/lab5/VERSIONED.md | 53 ++ student-tests/lab5/acceptance.py | 217 +++++++ student-tests/lab5/checks.py | 279 +++++++++ student-tests/lab5/compose.fault.test.yaml | 40 ++ .../lab5/compose.readiness.test.yaml | 13 + student-tests/lab5/compose_action.py | 94 +++ student-tests/lab5/imageprobe/main.go | 115 ++++ student-tests/lab5/pipeline.py | 386 ++++++++++++ student-tests/lab5/pipeline_action.py | 477 +++++++++++++++ student-tests/lab5/stand.py | 457 ++++++++++++++ student-tests/lab5/versioned.py | 430 +++++++++++++ 163 files changed, 17194 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 Dockerfile.migrate create mode 100644 Dockerfile.worker create mode 100644 cmd/api/main.go create mode 100644 cmd/api/poolstats.go create mode 100644 cmd/attachment-cleanup/main.go create mode 100644 cmd/jobs/main.go create mode 100644 cmd/migrate/async_attachments_test.go create mode 100644 cmd/migrate/main.go create mode 100644 cmd/migrate/main_test.go create mode 100644 cmd/migrate/postgres_test.go create mode 100644 cmd/migrate/roundtrip_test.go create mode 100644 cmd/relay/main.go create mode 100644 cmd/worker/main.go create mode 100644 compose.yaml create mode 100644 db/migrations/00001_init_schema.sql create mode 100644 db/migrations/00002_add_created_at_defaults.sql create mode 100644 db/migrations/00003_drop_session_table.sql create mode 100644 db/migrations/00004_message_idempotency.sql create mode 100644 db/migrations/00005_attachments.sql create mode 100644 db/migrations/00006_allow_attachment_only_messages.sql create mode 100644 db/migrations/00007_async_attachments.sql create mode 100644 db/migrations/00008_versioned_pipeline.sql create mode 100644 db/migrations/embed.go create mode 100644 db/queries/attachment_variants.sql create mode 100644 db/queries/attachments.sql create mode 100644 db/queries/attachments_jobs.sql create mode 100644 db/queries/chat_members.sql create mode 100644 db/queries/chats.sql create mode 100644 db/queries/message_idempotence.sql create mode 100644 db/queries/messages.sql create mode 100644 db/queries/messages_attachments.sql create mode 100644 db/queries/outbox.sql create mode 100644 db/queries/users.sql create mode 100644 db/queries/versioned_pipeline.sql create mode 100644 evidence/lab5/api-failover.jsonl create mode 100644 evidence/lab5/clean-run.md create mode 100644 evidence/lab5/crash-before-commit.jsonl create mode 100644 evidence/lab5/http-and-geometry.jsonl create mode 100644 evidence/lab5/pipeline-faults.jsonl create mode 100644 evidence/lab5/readiness.jsonl create mode 100644 evidence/lab5/track-2/concurrent-upgrade.jsonl create mode 100644 evidence/lab5/track-2/migration-live.json create mode 100644 evidence/lab5/track-2/regression.jsonl.gz create mode 100644 evidence/lab5/track-2/summary.json create mode 100644 evidence/lab5/track-2/versioned.jsonl.gz create mode 100644 evidence/lab5/track-2/versioned.txt create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/haproxy.cfg create mode 100644 infra/imagemagick/policy.xml create mode 100755 infra/s3/bootstrap.sh create mode 100644 internal/broker/rabbitmq.go create mode 100644 internal/config/config.go create mode 100644 internal/config/pipeline.go create mode 100644 internal/database/postgres.go create mode 100644 internal/database/redis.go create mode 100644 internal/database/sqlc/attachment_variants.sql.go create mode 100644 internal/database/sqlc/attachments.sql.go create mode 100644 internal/database/sqlc/attachments_jobs.sql.go create mode 100644 internal/database/sqlc/chat_members.sql.go create mode 100644 internal/database/sqlc/chats.sql.go create mode 100644 internal/database/sqlc/db.go create mode 100644 internal/database/sqlc/message_idempotence.sql.go create mode 100644 internal/database/sqlc/messages.sql.go create mode 100644 internal/database/sqlc/messages_attachments.sql.go create mode 100644 internal/database/sqlc/models.go create mode 100644 internal/database/sqlc/outbox.sql.go create mode 100644 internal/database/sqlc/users.sql.go create mode 100644 internal/database/sqlc/versioned_pipeline.sql.go create mode 100644 internal/fault/pause.go create mode 100644 internal/imageproc/processor.go create mode 100644 internal/model/attachment.go create mode 100644 internal/model/attachment_job.go create mode 100644 internal/model/chat.go create mode 100644 internal/model/message.go create mode 100644 internal/model/session.go create mode 100644 internal/model/user.go create mode 100644 internal/pipeline/cleanup.go create mode 100644 internal/pipeline/worker.go create mode 100644 internal/repository/attachment.go create mode 100644 internal/repository/attachment_tx.go create mode 100644 internal/repository/chat.go create mode 100644 internal/repository/errors.go create mode 100644 internal/repository/message.go create mode 100644 internal/repository/postgres/attachment.go create mode 100644 internal/repository/postgres/chat.go create mode 100644 internal/repository/postgres/errors.go create mode 100644 internal/repository/postgres/message.go create mode 100644 internal/repository/postgres/pipeline.go create mode 100644 internal/repository/postgres/pipeline_cleanup.go create mode 100644 internal/repository/postgres/user.go create mode 100644 internal/repository/postgres/utils.go create mode 100644 internal/repository/postgres/versioned_pipeline.go create mode 100644 internal/repository/redis/errors.go create mode 100644 internal/repository/redis/session.go create mode 100644 internal/repository/session.go create mode 100644 internal/repository/user.go create mode 100644 internal/security/password.go create mode 100644 internal/service/attachment.go create mode 100644 internal/service/attachment_cleanup.go create mode 100644 internal/service/attachment_reconcile.go create mode 100644 internal/service/auth.go create mode 100644 internal/service/chat.go create mode 100644 internal/service/errors.go create mode 100644 internal/service/helper.go create mode 100644 internal/service/message.go create mode 100644 internal/service/user.go create mode 100644 internal/storage/errors.go create mode 100644 internal/storage/object_cleanup_storage.go create mode 100644 internal/storage/object_writer.go create mode 100644 internal/storage/s3/client.go create mode 100644 internal/storage/s3/errors.go create mode 100644 internal/transport/http/handler/attachment.go create mode 100644 internal/transport/http/handler/attachment_types.go create mode 100644 internal/transport/http/handler/auth.go create mode 100644 internal/transport/http/handler/auth_types.go create mode 100644 internal/transport/http/handler/chat.go create mode 100644 internal/transport/http/handler/chat_types.go create mode 100644 internal/transport/http/handler/health.go create mode 100644 internal/transport/http/handler/health_types.go create mode 100644 internal/transport/http/handler/message.go create mode 100644 internal/transport/http/handler/message_types.go create mode 100644 internal/transport/http/handler/user.go create mode 100644 internal/transport/http/handler/user_types.go create mode 100644 internal/transport/http/httputils/helper.go create mode 100644 internal/transport/http/middleware/auth.go create mode 100644 internal/transport/http/middleware/context.go create mode 100644 internal/transport/http/middleware/logging.go create mode 100644 internal/transport/http/router.go create mode 100755 scripts/init-local.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 sqlc.yaml create mode 100644 student-tests/auth_contract.py create mode 100644 student-tests/auth_tests/attachment_cleanup_test.go create mode 100644 student-tests/auth_tests/attachment_tx_test.go create mode 100644 student-tests/auth_tests/auth_handler_test.go create mode 100644 student-tests/auth_tests/budgets_test.go create mode 100644 student-tests/auth_tests/errors_test.go create mode 100644 student-tests/auth_tests/logout_diagnostic_test.go create mode 100644 student-tests/auth_tests/message_attachments_test.go create mode 100644 student-tests/auth_tests/password_test.go create mode 100644 student-tests/auth_tests/redis_test.go create mode 100644 student-tests/auth_tests/restart_http_test.go create mode 100644 student-tests/auth_tests/storage_test.go create mode 100644 student-tests/auth_tests/transport_test.go create mode 100644 student-tests/lab5/FAULT-HOOKS.md create mode 100644 student-tests/lab5/README.md create mode 100644 student-tests/lab5/VERSIONED.md create mode 100644 student-tests/lab5/acceptance.py create mode 100644 student-tests/lab5/checks.py create mode 100644 student-tests/lab5/compose.fault.test.yaml create mode 100644 student-tests/lab5/compose.readiness.test.yaml create mode 100755 student-tests/lab5/compose_action.py create mode 100644 student-tests/lab5/imageprobe/main.go create mode 100644 student-tests/lab5/pipeline.py create mode 100755 student-tests/lab5/pipeline_action.py create mode 100644 student-tests/lab5/stand.py create mode 100644 student-tests/lab5/versioned.py diff --git a/.env.example b/.env.example index 119384b5..057c8bd7 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,12 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +WORKER_CONCURRENCY=1 +WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=90 +JOB_RETRY_BASE_SECONDS=2 STAGING_RETENTION_SECONDS=86400 +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=5 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..3b5f877e --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,21 @@ +name: Lab 5 application +on: [push, pull_request] +permissions: + contents: read +jobs: + compose-smoke: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + COMPOSE_PROJECT_NAME: lab5-ci + steps: + - uses: actions/checkout@v4 + - name: Initialize disposable configuration + run: sh scripts/init-local.sh + - name: Build and start + run: docker compose --env-file .env up -d --build --wait --wait-timeout 180 + - name: Run contract smoke + run: make check test BASE_URL=http://127.0.0.1:8085 + - name: Remove the CI project + if: always() + run: docker compose --env-file .env down --volumes diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..930e0f1c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM golang:1.27.1-alpine AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd/api/ ./cmd/api/ +COPY cmd/jobs/ ./cmd/jobs/ +COPY cmd/relay/ ./cmd/relay/ +COPY internal/ ./internal/ + +RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \ + CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/server ./cmd/api && \ + CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/jobs ./cmd/jobs && \ + CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/relay ./cmd/relay + + +FROM alpine:3.22 AS runtime + +RUN addgroup -S app && adduser -S app -G app + +WORKDIR /app + +USER app + +FROM runtime AS cleanup + +COPY --from=builder /app/jobs ./jobs +ENTRYPOINT ["/app/jobs"] +CMD ["cleanup"] + +FROM runtime AS relay +COPY --from=builder /app/relay ./relay +CMD ["./relay"] + +FROM runtime AS api + +COPY --from=builder /app/server ./server +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1 + +CMD ["./server"] diff --git a/Dockerfile.migrate b/Dockerfile.migrate new file mode 100644 index 00000000..4badcd5a --- /dev/null +++ b/Dockerfile.migrate @@ -0,0 +1,20 @@ +FROM golang:1.27.1-alpine AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd/migrate/ ./cmd/migrate/ +COPY db/migrations/ ./db/migrations/ +RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \ + CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/migrate ./cmd/migrate + +FROM alpine:3.22 + +RUN addgroup -S app && adduser -S app -G app +WORKDIR /app +COPY --from=builder /app/migrate ./migrate + +USER app +ENTRYPOINT ["/app/migrate"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 00000000..93aa82b6 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,23 @@ +FROM golang:1.27.1-alpine AS builder + +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd/worker/ ./cmd/worker/ +COPY internal/ ./internal/ +RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \ + CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/worker ./cmd/worker + +FROM alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f + +RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp \ + && addgroup -S app && adduser -S -G app app + +COPY infra/imagemagick/policy.xml /etc/ImageMagick-7/policy.xml + +COPY --from=builder /app/worker /usr/local/bin/worker + +WORKDIR /tmp +USER app + +CMD ["/usr/local/bin/worker"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..4eecb267 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,147 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5. Очереди и асинхронная обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Олег Ремез, БПИ2304. +- Целевая оценка «5», трек 2 по идемпотентности версий pipeline. +- Ветка: solution/BPI2304-remez. База PR: main репозитория mtusi2026/mtusi2026lab5. +- Использованы Go, PostgreSQL/pgx/sqlc, RabbitMQ/amqp091-go, Redis, HAProxy, RustFS/AWS SDK и ImageMagick. Codex участвовал в проектировании, реализации и подготовке отчёта. Чат «Лаба 5 тесты» подготовил и выполнил независимые проверки. [Подтверждения доставки RabbitMQ](https://www.rabbitmq.com/docs/confirms), [политика ImageMagick](https://imagemagick.org/security-policy/), [параметры WebP](https://imagemagick.org/webp/). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose v2, OpenSSL и make. Тесты проверены с Python 3.14.6. На macOS с системным Python 3.9 укажите `PYTHON=python3.14` в командах make, так как парсер поставляемых тестов может отклонить допустимую дробную часть RFC3339 длиннее шести цифр. Go 1.27.1 и Alpine 3.22 используются при сборке. ImageMagick с поддержкой JPEG и WebP входит в образ worker. Версии RabbitMQ, HAProxy и RustFS и digest образов PostgreSQL/Redis заданы в compose.yaml. Все подключения для демонстрации используют синтетические данные и локальный HTTP. -## Архитектура и решения +```sh +sh scripts/init-local.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +make check +make test BASE_URL=http://127.0.0.1:8085 +make test-persistence BASE_URL=http://127.0.0.1:8085 ACTION_SCRIPT=scripts/restart.sh +make test-failover BASE_URL=http://127.0.0.1:8085 ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env up -d --no-deps api +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +init-local.sh создаёт локальный .env, пароли и URL подключений в secrets/. Запуск использует мигратор Goose и приватную настройку S3. Процессы API, worker и relay стартуют после успешной миграции. Compose подключает Redis для сессий. API readiness проверяет PostgreSQL, Redis и S3. При недоступном RabbitMQ API всё ещё может сохранить работу в outbox. Сменить локальные порты можно через LAB5_HTTP_PORT и LAB5_POSTGRES_PORT. Для безопасной остановки использовать docker compose --env-file .env stop. Команда down --volumes удаляет данные проекта и годится только для изолированного одноразового стенда. -## Доказательства критериев +## Что происходит с вложением -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Синхронный путь API принимает PNG/JPEG, проверяет размер, MIME, имя, доступ и размеры картинки, вычисляет SHA-256. Под блокировкой ключа сохраняет байты в приватный S3 staging, затем в одной транзакции записывает attachment, job и событие outbox. После COMMIT API выдаёт 202 со снимком queued. Процесс обработки ищет исходник по ключу из PostgreSQL. Пользователь может отправить сообщение с queued или processing-вложением. Для failed новое прикрепление даёт 409, а повтор уже принятого POST с тем же ключом возвращает сохранённое сообщение после проверки доступа. Скачивание доступно участникам чата после ready. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Relay выбирает готовое событие под блокировкой строки, публикует persistent JSON с mandatory в durable очередь RabbitMQ и отмечает sent_at после confirm и проверки маршрутизации. Если процесс убит между публикацией и коммитом, тот же job_id может приехать повторно. Содержимое файла и секреты в сообщении отсутствуют. -## Эксперименты и отказы +Worker захватывает lease и увеличивает счётчик начатых попыток. Он сверяет исходные байты с SHA-256, готовит варианты и записывает оригинал и результаты в S3 под отдельным префиксом поколения lease. PostgreSQL публикует оригинал, метаданные всех вариантов и статус ready одной транзакцией под проверкой поколения и срока lease. Затем worker подтверждает AMQP delivery. Доставка готовой job проверяет опубликованный результат и не создаёт новую попытку. Восстановитель пишет одно событие на каждое истёкшее поколение lease. Старое поколение не может записать поверх результата нового. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Варианты, лимиты и ошибки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Имя | Формат и параметры | +| --- | --- | +| original | Неизменённый PNG или JPEG с исходным SHA-256 | +| thumbnail | JPEG quality 85, вписан в 256×256, меньшие изображения сохраняют размер, результат округляется вниз, alpha на белом | +| crop | JPEG quality 85, центральный квадрат 128×128, допускается увеличение, alpha на белом | +| optimized | WebP quality 80, method 4, вписан в 1280×1280 без увеличения, прозрачность сохраняется | -## Ограничения и следующие шаги +Для вариантов применяется EXIF orientation перед геометрией, а затем удаляются метаданные через -strip. Все изменения относятся к вариантам, исходный файл скачивается побайтно. Не гарантируется, что оптимизированный вариант каждого маленького файла займёт меньше места. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Таблица описывает v1, которую получает обычный upload. В v2 JPEG кодируется с quality 90, WebP с quality 72. Геометрия, форматы и оригинальные байты сохраняются. Версия рецепта задаётся `pipeline_version`, для обеих версий формат сообщения брокера остаётся `attachment.process.v1`. + +HTTP принимает до 10 МиБ и 40 млн пикселей. Worker повторно проверяет размер, заголовок и SHA-256, ограничивает время отдельного Go-контекста и убивает дочерний ImageMagick при отмене. Compose задаёт worker 1,5 ГиБ RAM, 2 CPU и tmpfs /tmp 768 МиБ. Политика ImageMagick задаёт один поток, 256 МиБ памяти и mmap, 512 МиБ диска для pixel cache и 45 секунд на команду, разрешает JPEG, PNG и WebP. Граничный файл, исчерпавший ресурс конкретного стенда, получает ошибку обработки. Worker по умолчанию использует одну активную обработку и prefetch 1 на consumer. + +Изменяемые параметры worker: WORKER_CONCURRENCY, WORKER_PREFETCH, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS, JOB_MAX_ATTEMPTS, JOB_RETRY_BASE_SECONDS. По умолчанию timeout обработки 60 секунд, lease 90 секунд, максимум 5 начатых обработок. Повторы не расходуют попытку, пока заново не получен lease. Для временного сбоя задержка равномерно выбирается от 1 секунды до min(30 секунд, 2 секунды × 2^(номер попытки − 1)), с ограничением роста степени. Для постоянной ошибки, например несоответствия исходника, попытки прекращаются. Для исчерпания временных повторов код retry_exhausted. Историю попыток и операторские действия храним в PostgreSQL. Неразбираемые сообщения идут в очередь image.invalid. + +## Операции с failed и staging + +```sh +docker compose exec -T attachment-cleanup /app/jobs failed +request_id=$(python3 -c 'import uuid; print(uuid.uuid4())') +# Установите job_id в ID из вывода failed. +docker compose exec -T attachment-cleanup /app/jobs replay \ + -job-id "$job_id" -request-id "$request_id" -operator student \ + -reason 'исправлена причина ошибки' -extra-attempts 1 +docker compose exec -T attachment-cleanup /app/jobs cleanup -once -dry-run +``` + +Replay проверяет failed и наличие исходника, записывает оператора, причину и ограниченный бюджет в одной транзакции с новым outbox. Повтор того же request-id и replay уже готовой job не меняют результат. Уборка удаляет staging после ready, исходники failed — после STAGING_RETENTION_SECONDS, бесхозный staging и неопубликованные ключи попыток — после CLEANUP_GRACE_SECONDS. Учитываются блокировка загрузки, lease и ссылки на оригинал и варианты. По умолчанию retention 24 часа, grace 10 минут, проход каждые 5 секунд. Очередь сама по себе может ждать дольше retention, queued и processing исходники сохраняются. + +Оператор переключает конкретное вложение на второй рецепт по ID. + +```sh +docker compose exec -T attachment-cleanup /app/jobs upgrade \ + -attachment-id "$attachment_id" -to-version 2 +``` + +При первом принятом переходе команда возвращает `queued: true` и ID новой job. Повтор возвращает `queued: false` с тем же ID. Готовое v1-вложение можно переключить после удаления staging. Источник для v2 тогда берётся из опубликованного оригинала. Во время перехода тот же ID вложения снова проходит `queued` и `processing`, скачивание отвечает 409. После ready API показывает набор текущего рецепта. Ссылки сообщений и SHA-256 оригинала сохраняются. + +## Доказательства и сценарий защиты + +Для управляемых отказов запускайте отдельную копию Compose по [инструкции приёмочных тестов](student-tests/lab5/README.md). В ней изолированы порты, volumes и каталог маркеров. Файлы `.arm` включают паузу один раз. Файл `.hit` связывает окно отказа с конкретным UUID. В обычном Compose паузы выключены. [Описание точек отказа](student-tests/lab5/FAULT-HOOKS.md). + +На защите сначала запустите `make test`, затем воспроизведите `durable-broker-worker-stop-and-redelivery` и `crash-COMMIT-before-publish`. Покажите один и тот же ID до и после рестартов, 409 до готовности и скачивание вариантов после неё. В `crash-object-before-ack-lease` сравните историю попыток и опубликованные ключи. В `backoff-jitter-permanent-failed-replay` покажите интервалы, причину `failed` и единственную запись аудита replay. + +Publisher confirm показывает, что брокер принял публикацию. Consumer ack закрывает конкретную доставку после устойчивого результата или сохранённого решения о retry/failed. При 202 уже существуют staging и атомарно сохранённые attachment, job и outbox. При redelivery таблица jobs определяет текущий lease и состояние, а ключ `(attachment_id, pipeline_version, name)` определяет уникальный опубликованный вариант. При истечении lease восстановитель публикует одну новую запись outbox на поколение. Предыдущая попытка теряет право на COMMIT, даже если её процесс ещё работает. + +[Протокол чистого запуска профиля «4»](evidence/lab5/clean-run.md). На пустых volumes были применены все 7 миграций того снимка. `make test`, `make test-persistence` и `make test-failover` прошли. После сценариев отказа лог мигратора остался прежним. При failover было 111 успешных записей и одна переходная ошибка, запись восстановилась за 3,04 с. Отдельно прошли `make check`, полная валидация контрактов, `sqlc compile`, `go vet` и `go test`. + +На чистом стенде проверены ограничение загрузки по байтам и пикселям, ошибки для повреждённых данных, доступ к вложениям и результат запроса через две реплики. Дополнительные PNG 1600×800 и JPEG с EXIF orientation 6 дали WebP 1280×640 и 640×1280. Размеры проверены декодером по скачанным байтам. Оригиналы совпали побайтно, контрольная строка метаданных и EXIF отсутствовали во всех вариантах. [Выдержки HTTP-проверок](evidence/lab5/http-and-geometry.jsonl). + +Отдельный SIGKILL API после записи staging остановил загрузку до 202. Изолированный тест подтвердил наличие объекта в S3, ноль строк attachment/job/outbox, удаление бесхозного объекта уборщиком и 404 для того же ID после подъёма API. [Результат аварии перед COMMIT](evidence/lab5/crash-before-commit.jsonl). + +### Независимая приёмка очереди и worker + +| Сценарий | Наблюдавшийся результат | +| --- | --- | +| Остановленный worker, рестарты API и RabbitMQ | Upload вернул 202, сообщение с вложением сохранилось, скачивание до ready вернуло 409. У durable очереди до и после рестарта было одно ожидающее сообщение и ноль consumers. Relay во время рестарта был остановлен. После запуска worker то же вложение стало ready, дубль не начал новую попытку. | +| SIGKILL relay после COMMIT до publish | Перед отказом существовал один outbox со статусом ожидания, опубликованных событий было ноль. После рестарта та же job завершилась первой попыткой, опубликовала четыре объекта, staging удалён. | +| SIGKILL relay после confirm до отметки outbox | Повторная публикация завершилась одним набором результатов и пустой очередью. | +| SIGKILL worker после записи объектов до COMMIT результата | У первой попытки было четыре объекта в S3 и статус processing. Истёк lease поколения 1, поколение 2 стало ready. После уборки остались четыре опубликованных объекта второй попытки. | +| SIGKILL worker после ready до ack | Redelivery сохранила поколение и историю единственной успешной попытки. RabbitMQ подтвердил отсутствие ожидающих и неподтверждённых доставок. | +| Дедлайн и лимиты | При test timeout 14 с и lease 18 с первая попытка закончилась `processing_timeout`, вторая стала ready. Наблюдались один consumer с manual ack и prefetch 1, 1,5 ГиБ лимита worker, ImageMagick 256 МиБ RAM, 256 МиБ mmap, 512 МиБ disk, один поток и 45 с на команду. | +| Временные и постоянные ошибки, failed и replay | В двух опытах с недоступным S3 задержки составили 1806/1252 и 1705/2627 мс, затем `retry_exhausted` на третьей попытке. Повреждённый checksum привёл к одному `invalid_image`. Операторский replay после устранения причин дал ready и одну запись аудита на job. Replay готовой job сохранил результат и число попыток. | +| Некорректные задания в брокере | Три сообщения с несовпадающими идентификаторами или версией попали в `image.invalid`. Готовый результат и история попыток сохранились. | + +[Наблюдения из PostgreSQL, RabbitMQ и S3](evidence/lab5/pipeline-faults.jsonl). После аварийной серии мигратор по-прежнему имел ровно один успешный запуск и по одной записи каждой версии. При отключении доступа к хранилищам одна API продолжала отвечать `live=200`, сообщала `ready=503`, HAProxy направил все 29 запросов контрольного окна на вторую API. После восстановления трафик снова шёл на обе. [Наблюдения проверки readiness](evidence/lab5/readiness.jsonl). + +Интеграционные Go-тесты с отдельными PostgreSQL и Redis прошли 53 сценария, включая откат транзакций, проверку доступа при replay и совместимость старых сообщений. Три дополнительных Redis-теста требуют собственных хуков рестарта и были пропущены. Отдельные сквозные сценарии подтвердили сохранность сессии, данных и ключа идемпотентности при общем рестарте. Дополнительный сценарий rolling update ожидает хука замены реплики. [Воспроизведение независимого прогона](student-tests/lab5/README.md). + +На основном стенде профиля «4» прямой запрос в RustFS без подписи на существующий объект вернул 403. + +### Нагрузка на API при отказе реплик + +Независимый прогон выполнил 6 параллельных потоков, каждый чередовал POST сообщения и GET профиля. Прогрев длился 2 с. В окне 28,37 с скрипт выполнил 15 861 попытку. Успешных было 15 843, или 558,37 запроса/с. Ошибок было 18 (0,1135%). Для успешных запросов p50/p95/p99 составили 5,516 / 21,228 / 47,711 мс. Сохранились все 7 926 подтверждённых записей. После SIGTERM запись и чтение восстановились за 3,104 с и 3,301 с для двух реплик. + +Снимок под нагрузкой показал 36,80% CPU и 15,35 МиБ у первой API, 37,26% и 12,92 МиБ у второй, 32,09% и 77,19 МиБ у HAProxy. Лимиты одной API — 2 CPU и 512 МиБ. Замер сделан на Apple M2 с 8 ГиБ RAM, Docker VM получила 8 CPU и 4 ГБ RAM. Прикладная нагрузка и отказ измерены на синтетическом чате. Каждый клиент ждал ответа предыдущего запроса. Перцентили учитывают успешные ответы, а пропускная способность относится к одному этому профилю. [Сырые метрики](evidence/lab5/api-failover.jsonl). + +## Трек 2. Идемпотентность версий pipeline + +**Гипотеза.** Если публикация проверяет и текущую версию, и поколение действующего lease, а физические ключи S3 включают обе величины, перестановка сообщений и запоздавшие попытки сохранят готовый результат v2. После падения worker принятая работа текущей версии продолжится с новым поколением. + +1. PostgreSQL хранит одно задание на `(attachment_id, pipeline_version)` и один вариант каждого имени на эту пару. Для переключения оператор берёт ту же блокировку строки v1, которую используют финализация, retry и очистка. У смены рецепта и попытки v1 есть один порядок фиксации. +2. В одной транзакции переход закрывает старый lease и откладываемый retry, завершает активную попытку как `superseded`, меняет версию вложения и записывает v2 job с outbox. Идентификатор вложения и строки сообщений сохраняются. Завершённая v1 остаётся в истории. +3. Worker может записать файлы только под префиксом своей версии и поколения. Публикация в PostgreSQL проходит под проверкой актуальной версии, поколения, срока lease и наличия метаданных всех трёх вариантов. В той же транзакции фиксируются оригинал версии, итоговый указатель и ready. GET читает согласованный набор в одной snapshot-транзакции. +4. Дубль текущей ready job подтверждается после проверки сохранённых объектов. Валидная доставка v1 после переключения подтверждается по уже зафиксированному переходу. Непубликованные файлы старых попыток удаляются по grace, опубликованные наборы обеих версий защищены ссылками. История увеличивает объём хранения на один набор при переходе с готовой v1. + +В каждом сценарии загружается новый экземпляр одного PNG. Скрипт фиксирует реальное сообщение и состояние PostgreSQL до и после вмешательства. Он скачивает исходник и варианты по HTTP, перечитывает объекты по сохранённым ключам S3, сравнивает SHA-256, размеры и историю попыток. Он также проверяет отсутствие лишних ключей после уборки. Для перестановки скрипт откладывает исходный outbox v1 и разрешает его публикацию после ready v2. Для падений используется SIGKILL живого worker в точках до первого S3 PUT и после записи всех объектов. Для гонки используются два процесса worker. [Методика и команда запуска](student-tests/lab5/VERSIONED.md). + +15 ноября 2026 года тестовик выполнил три полные серии из пяти сценариев на изолированном Docker-проекте со схемой лабораторной v8, lease 36 с, deadline 30 с и grace очистки 1 с. Получено **15 из 15 успешных опытов**. Время приведено как минимум, медиана и максимум. Для падений оно отсчитывается от отправки SIGKILL, для остальных сценариев охватывает ожидание ready при переходе или после запуска relay. + +| Сценарий | Успех | До ready, с | Наблюдение | +| --- | ---: | ---: | --- | +| Дубль v2 и запоздавшая доставка v1 | 3/3 | 0,142 / 0,161 / 0,162 | По одной успешной попытке на версию. Переотправленные валидные envelopes прошли RabbitMQ, ready и SHA v2 сохранились, DLQ не выросла. | +| Перестановка v2 → v1 | 3/3 | 0,377 / 0,432 / 0,960 | Исходный outbox v1 прошёл брокер после ready v2. У v1 ноль попыток, у v2 одна. | +| SIGKILL до первого S3 PUT | 3/3 | 31,613 / 31,802 / 32,085 | Перед падением ноль объектов нового поколения. Первая попытка v2 истекла, вторая завершила ту же job. | +| SIGKILL после S3 PUT | 3/3 | 31,915 / 33,189 / 33,353 | Перед падением четыре объекта первой попытки. Следующее поколение завершило job, старая попытка очищена. | +| Возобновление старого worker v1 после ready v2 | 3/3 | 0,161 / 0,419 / 0,873 | Живой процесс получил отказ в финализации за 19,52–21,47 с до прежнего дедлайна. Итоговые ключи v2 сохранились, сироты v1 удалены. | + +Все 15 v2 получили одинаковые три SHA-256 на одном исходнике. Для готовой v1 контрольные SHA отличались по каждому из трёх вариантов, оригинальные байты совпадали. Два конкурентных вызова `jobs upgrade` после очистки staging отдельно подтвердили один новый job ID, один outbox и ровно один ответ с `queued: true`. [Агрегаты](evidence/lab5/track-2/summary.json), [сырые наблюдения](evidence/lab5/track-2/versioned.jsonl.gz), [параллельный upgrade](evidence/lab5/track-2/concurrent-upgrade.jsonl). + +После перехода на схему v8 снова прошли чистый bootstrap, `make test`, проверки сохранности и failover, лимиты HTTP и скачивание всех вариантов. Контракт и публичные тесты не изменились. [Регрессионные проверки после v8](evidence/lab5/track-2/regression.jsonl.gz). + +При обновлении основного стенда 16 ноября мигратор применил только `00008`. До обновления в PostgreSQL было шесть готовых вложений, после перехода у каждого сохранилась ссылка на опубликованный оригинал. На `127.0.0.1:8085` прошли smoke, readiness и проверки со штатными скриптами рестарта и отказа. При failover запись восстановилась за 2,599 с, лог мигратора после рестартов содержал тот же успешный запуск. [Переход основного стенда v7 → v8](evidence/lab5/track-2/migration-live.json). + +RabbitMQ обеспечивает доставку at least once. Одна и та же job появлялась в очереди повторно, а после падения worker повторно вычислял и записывал объекты под другим поколением. Гарантия приложения относится к единственной зафиксированной публикации результата для текущей версии. Эти наблюдения совместимы с повторной доставкой и выполнением. Успешное восстановление зависит от сохранности PostgreSQL, S3, тома RabbitMQ и наличия рабочего relay и worker. + +## Ограничения + +Проверяем один Docker-хост. Две реплики дают проверяемый отказ одной API, а данные broker переживают рестарт с существующим volume. Потеря самого хоста или всех volumes остаётся общей точкой отказа. Очередь даёт доставку at least once. Идемпотентность реализована приложением на уровне итогового набора объектов и транзакции, поэтому повтор можно наблюдать, хотя результат один. Уборка проходит по страницам S3, её стоимость растёт с объёмом bucket. + +Проверки трека 2 выполнены на одном образе ImageMagick, одном PNG и одном Docker-хосте. Число повторов позволяет воспроизвести инварианты в указанных окнах отказа. Интервалы восстановления относятся к заданному профилю lease и локальному стенду. + +[Сценарии для защиты трека 2](student-tests/lab5/VERSIONED.md). diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 00000000..6ebe29a6 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "errors" + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository/postgres" + "highload/internal/repository/redis" + "highload/internal/service" + "highload/internal/storage/s3" + httpdeliver "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + + cfg, err := config.Load() + + if err != nil { + slog.Error("invalid configuration", "error", err) + os.Exit(1) + } + + sessionStore := cfg.Session.Store + + if sessionStore != "redis" { + slog.Error("session store is "+sessionStore+"must redis", "error", nil) + os.Exit(1) + } + + databaseURL := cfg.Database.PostgresURL + pool, err := database.NewPostgres(context.Background(), databaseURL) + + if err != nil { + slog.Error("postgres error", "error", err) + os.Exit(1) + } + defer pool.Close() + if os.Getenv("PG_POOL_DIAGNOSTICS") == "1" { + diagnosticsCtx, stopDiagnostics := context.WithCancel(context.Background()) + defer stopDiagnostics() + go logPoolStats(diagnosticsCtx, pool) + } + + queries := sqlc.New(pool) + + redisAddr := cfg.Database.RedisAddr + + redisClient, err := database.NewRedis(context.Background(), redisAddr) + + if err != nil { + slog.Error("redis error", "error", err) + os.Exit(1) + } + + defer redisClient.Close() + + instanceID, err := os.Hostname() + if err != nil || instanceID == "" { + slog.Error("cannot get instance ID", "error", err) + os.Exit(1) + } + + s3Storage, err := s3.NewStorage(context.Background(), &cfg.S3) + + if err != nil { + slog.Error("cannot create new storage", "error", err) + os.Exit(1) + } + + userRepo := postgres.NewUserRepository(queries) + chatRepo := postgres.NewChatRepository(queries, pool) + messageRepo := postgres.NewMessageRepository(queries, pool) + sessionRepo := redis.NewSessionRepository(redisClient) + attachmentRepo := postgres.NewAttachmentRepository(queries, pool, int32(cfg.JobMaxAttempts)) + + authService := service.NewAuthService(userRepo, sessionRepo, cfg.Session.TTL) + userService := service.NewUserService(userRepo) + chatService := service.NewChatService(chatRepo, userRepo, userService) + messageService := service.NewMessageService(messageRepo, chatService, userService) + attachmentService := service.NewAttachmentService(attachmentRepo, chatService, s3Storage) + + healthHandler := handler.NewHealthHandler(pool, redisClient, s3Storage) + authHandler := handler.NewAuthHandler(authService) + userHandler := handler.NewUserHandler(userService) + chatHandler := handler.NewChatHandler(chatService) + messageHandler := handler.NewMessageHandler(messageService) + attachmentHandler := handler.NewAttachmentHandler(attachmentService) + + mid := middleware.NewMiddleware(authService, instanceID) + router := httpdeliver.NewRouter(mid, authHandler, healthHandler, chatHandler, userHandler, messageHandler, attachmentHandler) + + server := &http.Server{ + Addr: net.JoinHostPort(cfg.HTTP.Host, cfg.HTTP.Port), + Handler: router.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + } + + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + slog.Error("server failed to start", "error", err) + return + } + + slog.Info("server started", "addr", server.Addr) + + serverErr := make(chan error, 1) + go func() { + err := server.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + serverErr <- err + slog.Error("server failed", "error", err) + } + }() + + ctx, stop := signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGTERM, + ) + defer stop() + + select { + case <-ctx.Done(): + slog.Info("shutdown signal received") + + case err := <-serverErr: + slog.Error("server failed", "error", err) + return + } + + shutdownCtx, cancel := context.WithTimeout( + context.Background(), + 15*time.Second, + ) + defer cancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + slog.Error("shutdown failed", "error", err) + } + +} diff --git a/cmd/api/poolstats.go b/cmd/api/poolstats.go new file mode 100644 index 00000000..b57a70c6 --- /dev/null +++ b/cmd/api/poolstats.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// logPoolStats is opt-in diagnostics for the isolated scaling experiment. +// It contains only aggregate counters, never connection strings or SQL values. +func logPoolStats(ctx context.Context, pool *pgxpool.Pool) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s := pool.Stat() + slog.Info("pg_pool_stats", "max_conns", s.MaxConns(), + "acquired", s.AcquiredConns(), "idle", s.IdleConns(), + "acquire_count", s.AcquireCount(), "empty_count", s.EmptyAcquireCount(), + "empty_wait_ns", s.EmptyAcquireWaitTime().Nanoseconds(), + "canceled_count", s.CanceledAcquireCount()) + } + } +} diff --git a/cmd/attachment-cleanup/main.go b/cmd/attachment-cleanup/main.go new file mode 100644 index 00000000..79f791f6 --- /dev/null +++ b/cmd/attachment-cleanup/main.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage/s3" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +type cleanupConfig struct { + once bool + dryRun bool + gracePeriod time.Duration + interval time.Duration + passTimeout time.Duration + reconcile bool + restoreID string + backupFile string +} + +func main() { + var cfg cleanupConfig + + flag.BoolVar(&cfg.once, "once", false, "Run once and exit") + flag.BoolVar(&cfg.dryRun, "dry-run", true, "Do not actually delete files") + flag.DurationVar(&cfg.gracePeriod, "grace-period", 10*time.Minute, "Grace period for old files") + flag.DurationVar(&cfg.interval, "interval", 5*time.Minute, "Interval between runs") + flag.DurationVar(&cfg.passTimeout, "pass-timeout", 2*time.Minute, "Timeout for a single pass") + flag.BoolVar(&cfg.reconcile, "reconcile", false, "Scan both stores once and print JSON observations") + flag.StringVar(&cfg.restoreID, "restore-id", "", "Restore the missing original for this attachment ID") + flag.StringVar(&cfg.backupFile, "backup-file", "", "Path to the original backup used for restore") + flag.Parse() + + if cfg.gracePeriod <= 0 || cfg.interval <= 0 || cfg.passTimeout <= 0 { + slog.Error("all duration flags must be positive") + os.Exit(1) + } + if (cfg.restoreID == "") != (cfg.backupFile == "") || (cfg.reconcile && cfg.restoreID != "") { + slog.Error("use -reconcile alone, or supply both -restore-id and -backup-file") + os.Exit(1) + } + if err := run(cfg); err != nil { + slog.Error("attachment cleanup failed", "error", err) + os.Exit(1) + } +} + +func run(cfg cleanupConfig) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + databaseURL, s3Config, err := config.LoadAttachmentCleanup() + if err != nil { + return fmt.Errorf("load cleanup configuration: %w", err) + } + + startupCtx, cancelStartup := context.WithTimeout(ctx, 10*time.Second) + defer cancelStartup() + + pool, err := database.NewPostgres(startupCtx, databaseURL) + if err != nil { + var configErr *pgconn.ParseConfigError + if errors.As(err, &configErr) { + return errors.New("invalid PostgreSQL connection string") + } + return fmt.Errorf("connect to PostgreSQL: %w", err) + } + defer pool.Close() + + s3Storage, err := s3.NewStorage(startupCtx, &s3Config) + if err != nil { + return fmt.Errorf("create S3 client: %w", err) + } + cancelStartup() + + attachmentRepo := postgres.NewAttachmentRepository(sqlc.New(pool), pool) + options := &service.AttachmentCleanupOptions{ + GracePeriod: cfg.gracePeriod, + DryRun: cfg.dryRun, + ListTimeout: 5 * time.Second, + } + attachmentCleanupService := service.NewAttachmentCleanupService(attachmentRepo, s3Storage, options) + + if cfg.reconcile || cfg.restoreID != "" { + passCtx, cancel := context.WithTimeout(ctx, cfg.passTimeout) + defer cancel() + reconciler := service.NewAttachmentReconciler(attachmentRepo, s3Storage, *options) + start := time.Now() + report := struct { + DryRun bool `json:"dry_run"` + DurationMS int64 `json:"duration_ms"` + Complete bool `json:"complete"` + Orphans *service.OrphanReport `json:"orphan_scan,omitempty"` + Missing *service.MissingReport `json:"missing_scan,omitempty"` + Restore string `json:"restore,omitempty"` + }{DryRun: cfg.dryRun} + var actionErr error + if cfg.reconcile { + missing, err := reconciler.ScanMissing(passCtx) + report.Missing = &missing + actionErr = err + if err == nil { + orphans, err := attachmentCleanupService.ScanOrphans(passCtx) + report.Orphans = &orphans + actionErr = err + } + } else { + file, err := os.Open(cfg.backupFile) + if err != nil { + return fmt.Errorf("open backup original: %w", err) + } + defer file.Close() + report.Restore, actionErr = reconciler.Restore(passCtx, cfg.restoreID, file) + } + report.DurationMS = time.Since(start).Milliseconds() + report.Complete = actionErr == nil + return errors.Join(actionErr, json.NewEncoder(os.Stdout).Encode(report)) + } + + runPass := func() error { + passCtx, cancelPass := context.WithTimeout(ctx, cfg.passTimeout) + defer cancelPass() + return attachmentCleanupService.RunOnce(passCtx) + } + + if cfg.once { + return runPass() + } + + if err := runPass(); err != nil && ctx.Err() == nil { + slog.Error("attachment cleanup pass failed", "error", err) + } + + ticker := time.NewTicker(cfg.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if ctx.Err() != nil { + return nil + } + if err := runPass(); err != nil && ctx.Err() == nil { + slog.Error("attachment cleanup pass failed", "error", err) + } + } + } +} diff --git a/cmd/jobs/main.go b/cmd/jobs/main.go new file mode 100644 index 00000000..c2dbb63c --- /dev/null +++ b/cmd/jobs/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/pipeline" + "highload/internal/repository/postgres" + "highload/internal/storage/s3" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Args[1:]); err != nil { + slog.Error("attachment jobs command failed", "error", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) error { + const usage = "usage: jobs failed | replay | upgrade | cleanup" + if len(args) == 0 { + return fmt.Errorf("%s", usage) + } + flags := flag.NewFlagSet(args[0], flag.ContinueOnError) + job := flags.String("job-id", "", "failed job ID") + request := flags.String("request-id", "", "unique operator request UUID") + operator := flags.String("operator", "", "operator ID") + reason := flags.String("reason", "", "replay reason") + extra := flags.Int("extra-attempts", 1, "budget for replay (1–5)") + attachment := flags.String("attachment-id", "", "attachment ID to upgrade") + version := flags.Int("to-version", 0, "target pipeline version (2)") + dry := flags.Bool("dry-run", false, "report eligible objects without deleting") + once := flags.Bool("once", false, "execute one cleanup pass") + if err := flags.Parse(args[1:]); err != nil { + return err + } + command := args[0] + if command != "failed" && command != "cleanup" && command != "replay" && command != "upgrade" { + return fmt.Errorf("%s", usage) + } + if command == "replay" && (*job == "" || *request == "" || strings.TrimSpace(*operator) == "" || + strings.TrimSpace(*reason) == "" || *extra < 1 || *extra > 5) { + return fmt.Errorf("replay needs job-id, request-id, operator, reason and 1–5 extra-attempts") + } + if command == "upgrade" && (*attachment == "" || *version != 2) { + return fmt.Errorf("upgrade needs attachment-id and to-version=2") + } + attemptLimit := 5 + if command == "upgrade" { + var err error + attemptLimit, err = config.EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20) + if err != nil { + return err + } + } + url, storageConfig, err := config.LoadAttachmentCleanup() + if err != nil { + return err + } + pool, err := database.NewPostgres(ctx, url) + if err != nil { + return fmt.Errorf("PostgreSQL unavailable") + } + defer pool.Close() + repo := postgres.NewAttachmentRepository(sqlc.New(pool), pool, int32(attemptLimit)) + if command == "failed" { + dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + rows, err := repo.ListFailedJobs(dbCtx) + if err != nil { + return err + } + type failed struct { + JobID string `json:"job_id"` + AttachmentID string `json:"attachment_id"` + ErrorCode string `json:"error_code"` + Attempts int32 `json:"attempts"` + UpdatedAt time.Time `json:"updated_at"` + } + result := make([]failed, 0, len(rows)) + for _, row := range rows { + result = append(result, failed{row.ID.String(), row.AttachmentID.String(), row.ErrorCode.String, row.AttemptsStarted, row.UpdatedAt.Time}) + } + return json.NewEncoder(os.Stdout).Encode(result) + } + objects, err := s3.NewStorage(ctx, &storageConfig) + if err != nil { + return err + } + if command == "replay" { + changed, err := repo.ReplayFailed(ctx, *job, *request, *operator, *reason, int32(*extra), objects.Head) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"job_id": *job, "replayed": changed}) + } + if command == "upgrade" { + jobID, queued, err := repo.UpgradePipeline(ctx, *attachment, objects.Head) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{ + "attachment_id": *attachment, "pipeline_version": *version, "job_id": jobID, "queued": queued, + }) + } + cfg, err := config.LoadPipeline() + if err != nil { + return err + } + cleanup := pipeline.Cleanup{Repo: repo, Objects: objects, Grace: cfg.CleanupGrace, Retention: cfg.Retention, DryRun: *dry} + if *once { + count, err := cleanup.RunOnce(ctx) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"objects": count, "dry_run": *dry}) + } + cleanup.Serve(ctx, cfg.CleanupPeriod) + return nil +} diff --git a/cmd/migrate/async_attachments_test.go b/cmd/migrate/async_attachments_test.go new file mode 100644 index 00000000..65632644 --- /dev/null +++ b/cmd/migrate/async_attachments_test.go @@ -0,0 +1,254 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "strings" + "testing" + "time" + + "highload/db/migrations" + "highload/internal/database/sqlc" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/pressly/goose/v3" +) + +func TestAsyncAttachmentsMigrationAndJobTransitions(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files, + goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil)))) + if err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 6); err != nil { + t.Fatal(err) + } + + const original = "attachments/step-two-original" + _, err = db.ExecContext(ctx, ` + INSERT INTO users (id, username, display_name, password_hash) VALUES + ('00000000-0000-4000-8000-000000000001', 'step_two', 'Step two', 'synthetic'); + INSERT INTO chats (id, title) VALUES ('00000000-0000-4000-8000-000000000002', 'Step two'); + INSERT INTO chat_members VALUES + ('00000000-0000-4000-8000-000000000002', '00000000-0000-4000-8000-000000000001'); + INSERT INTO messages (id, chat_id, sender_id, text) VALUES + ('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000001', 'Old message'); + INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) VALUES + ('00000000-0000-4000-8000-000000000004', '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000001', 'old.png', 'attachments/step-two-original', + 'image/png', 123, repeat('a', 64)); + INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) VALUES + ('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000004', + '00000000-0000-4000-8000-000000000002', 0); + `) + if err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 7); err != nil { + t.Fatal(err) + } + if _, err := provider.Down(ctx); err != nil { + t.Fatal(err) + } + var preservedKey string + if err := db.QueryRowContext(ctx, "SELECT object_key FROM attachments JOIN messages_attachments ON attachment_id = id").Scan(&preservedKey); err != nil || preservedKey != original { + t.Fatalf("rollback lost the legacy attachment reference: %v", err) + } + if _, err := provider.UpTo(ctx, 7); err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, lab5MigrationCount); err != nil { + t.Fatal(err) + } + + dsn, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE")) + if err != nil { + t.Fatal("cannot read test connection configuration") + } + pool, err := pgxpool.New(ctx, string(dsn)) + if err != nil { + t.Fatal("cannot configure test pool") + } + defer pool.Close() + q := sqlc.New(pool) + + var jobID, attachmentID pgtype.UUID + var payload []byte + err = pool.QueryRow(ctx, ` + SELECT j.id, a.id, o.payload + FROM messages_attachments AS m + JOIN attachments AS a ON a.id = m.attachment_id + JOIN attachment_jobs AS j ON j.attachment_id = a.id + JOIN outbox AS o ON o.job_id = j.id + WHERE a.status = 'queued' AND a.object_key = $1 AND a.staging_key = $1 + AND o.dedup_key = 'initial/' || j.id + `, original).Scan(&jobID, &attachmentID, &payload) + if err != nil { + t.Fatalf("legacy attachment was not queued with its initial event: %v", err) + } + var event struct { + JobID string `json:"job_id"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + } + if err := json.Unmarshal(payload, &event); err != nil || event.JobID != jobID.String() || + event.Attempt != 1 || event.CreatedAt.IsZero() { + t.Fatalf("invalid backfill payload: %v", err) + } + + start := make(chan struct{}) + results := make(chan error, 2) + claim := sqlc.ClaimJobParams{ + JobID: jobID, NewLeaseUntil: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true}, + } + for range 2 { + go func() { + <-start + _, err := q.ClaimJob(ctx, claim) + results <- err + }() + } + close(start) + winners := 0 + for range 2 { + switch err := <-results; { + case err == nil: + winners++ + case errors.Is(err, pgx.ErrNoRows): + default: + t.Fatalf("concurrent claim failed: %v", err) + } + } + if winners != 1 { + t.Fatalf("gave the lease to %d workers", winners) + } + + exec := func(query string, args ...any) { + t.Helper() + if _, err := pool.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID) + for _, want := range []int{1, 0} { + recovery, err := q.EnqueueExpiredLeases(ctx, 10) + if err != nil || len(recovery) != want { + t.Fatalf("recovery events = %d, want %d: %v", len(recovery), want, err) + } + if want == 1 { + exec("UPDATE outbox SET sent_at = now() WHERE id = $1", recovery[0].ID) + } + } + row, err := q.ClaimJob(ctx, claim) + if err != nil || row.LeaseGeneration != 2 || row.AttemptsStarted != 2 { + t.Fatalf("expired lease was not replaced correctly: %v", err) + } + fail := sqlc.FailJobParams{JobID: jobID, LeaseGeneration: 1, ErrorCode: "invalid_image"} + if n, err := q.FailJob(ctx, fail); err != nil || n != 0 { + t.Fatalf("stale worker could fail attachment: rows=%d, err=%v", n, err) + } + retry := sqlc.ScheduleRetryParams{ + JobID: jobID, LeaseGeneration: 2, ErrorCode: "s3_unavailable", + AvailableAt: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true}, + } + if n, err := q.ScheduleRetry(ctx, retry); err != nil || n != 1 { + t.Fatalf("retry could not be scheduled: rows=%d, err=%v", n, err) + } + if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("worker bypassed the retry delay: %v", err) + } + exec("UPDATE attachment_jobs SET next_attempt_at = now(), max_attempts = 3 WHERE id = $1", jobID) + row, err = q.ClaimJob(ctx, claim) + if err != nil || row.LeaseGeneration != 3 { + t.Fatalf("due retry could not claim job: %v", err) + } + exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID) + if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("worker bypassed the attempt limit: %v", err) + } + fail.LeaseGeneration = 3 + fail.ErrorCode = "retry_exhausted" + if n, err := q.FailJob(ctx, fail); err != nil || n != 1 { + t.Fatalf("exhausted job could not be failed: rows=%d, err=%v", n, err) + } + var requestID pgtype.UUID + if err := requestID.Scan("00000000-0000-4000-8000-000000000005"); err != nil { + t.Fatal(err) + } + replay := sqlc.ReplayFailedParams{ + JobID: jobID, RequestID: requestID, OperatorID: "operator", Reason: "test replay", ExtraAttempts: 1, + } + for _, want := range []int64{1, 0} { + if n, err := q.ReplayFailed(ctx, replay); err != nil || n != want { + t.Fatalf("replay changed %d rows, want %d: %v", n, want, err) + } + } + row, err = q.ClaimJob(ctx, claim) + if err != nil || row.AttemptsStarted != 4 || row.LeaseGeneration != 4 { + t.Fatalf("replay did not preserve attempt and lease counters: %v", err) + } + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + locked := q.WithTx(tx) + if _, err := locked.LockActiveAttachmentJob(ctx, sqlc.LockActiveAttachmentJobParams{JobID: jobID, LeaseGeneration: 4}); err != nil { + t.Fatal(err) + } + complete := sqlc.CompleteJobParams{JobID: jobID, LeaseGeneration: 4, ObjectKey: "results/original"} + if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 { + t.Fatalf("published result without variants: rows=%d, err=%v", n, err) + } + variant := sqlc.CreateAttachmentVariantParams{ + AttachmentID: attachmentID, PipelineVersion: 1, Name: "thumbnail", ObjectKey: "results/thumb", + ContentType: "image/jpeg", SizeBytes: 123, Width: 16, Height: 10, Sha256: strings.Repeat("a", 64), + } + if err := locked.CreateAttachmentVariant(ctx, variant); err != nil { + t.Fatal(err) + } + if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 { + t.Fatalf("published result without crop and optimized: rows=%d, err=%v", n, err) + } + variant.Name, variant.ObjectKey = "crop", "results/crop" + if err := locked.CreateAttachmentVariant(ctx, variant); err != nil { + t.Fatal(err) + } + variant.Name, variant.ObjectKey, variant.ContentType = "optimized", "results/webp", "image/webp" + if err := locked.CreateAttachmentVariant(ctx, variant); err != nil { + t.Fatal(err) + } + if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 1 { + t.Fatalf("current owner could not publish result: rows=%d, err=%v", n, err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("worker claimed a ready job: %v", err) + } + published, err := q.GetPublishedOriginalKey(ctx, sqlc.GetPublishedOriginalKeyParams{ + AttachmentID: attachmentID, PipelineVersion: 1, + }) + if err != nil || published.String != complete.ObjectKey { + t.Fatalf("published original not recorded for its version: %v", err) + } + + var attempts, events, replays int + err = pool.QueryRow(ctx, ` + SELECT (SELECT count(*) FROM attachment_attempts WHERE job_id = $1 AND finished_at IS NOT NULL), + (SELECT count(*) FROM outbox WHERE job_id = $1), + (SELECT count(*) FROM attachment_replays WHERE job_id = $1) + `, jobID).Scan(&attempts, &events, &replays) + if err != nil || attempts != 4 || events != 4 || replays != 1 { + t.Fatalf("unexpected history: attempts=%d, events=%d, replays=%d: %v", attempts, events, replays, err) + } +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..cbe59e16 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "flag" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "highload/db/migrations" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/lock" +) + +// Migration errors retain their cause for classification, but deliberately do +// not print connection strings, PostgreSQL DETAIL fields or raw SQL. +type migrationError struct { + stage string + cause error +} + +func (e *migrationError) Error() string { return e.stage } +func (e *migrationError) Unwrap() error { return e.cause } + +type boundedSessionLocker struct { + lock.SessionLocker +} + +func (l boundedSessionLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error { + // Goose unlocks with a detached context; bound this cleanup separately. + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return l.SessionLocker.SessionUnlock(ctx, conn) +} + +func main() { + command := flag.String("command", "up", "migration action: up or status") + timeout := flag.Duration("timeout", 2*time.Minute, "total migration/connection/lock budget") + flag.Parse() + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if flag.NArg() != 0 || (*command != "up" && *command != "status") || *timeout < time.Second { + logger.Error("invalid migration arguments", "allowed_commands", "up,status", + "minimum_timeout", "1s") + os.Exit(1) + } + + signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(signalCtx, *timeout) + defer cancel() + if err := run(ctx, *command, *timeout, logger); err != nil { + fields := []any{"stage", err.Error(), "timeout", errors.Is(err, context.DeadlineExceeded), + "cancelled", errors.Is(err, context.Canceled)} + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + fields = append(fields, "sqlstate", pgErr.Code) + } + var partial *goose.PartialError + if errors.As(err, &partial) && partial.Failed != nil && partial.Failed.Source != nil { + fields = append(fields, "migration", partial.Failed.Source.Path, + "version", partial.Failed.Source.Version) + } + logger.Error("migration failed", fields...) + os.Exit(1) + } +} + +func run(ctx context.Context, command string, timeout time.Duration, logger *slog.Logger) error { + fail := func(stage string, err error) error { + return &migrationError{stage: stage, cause: err} + } + path := os.Getenv("DATABASE_URL_FILE") + if path == "" { + path = "/run/secrets/database_url" + } + data, err := os.ReadFile(path) + if err != nil { + return fail("read database secret file", err) + } + url := strings.TrimSpace(string(data)) + if url == "" { + return fail("database secret file is empty", errors.New("empty database URL")) + } + cfg, err := pgx.ParseConfig(url) + if err != nil { + return fail("parse database secret", err) + } + cfg.ConnectTimeout = 5 * time.Second + cfg.RuntimeParams["application_name"] = "highload-migrate" + cfg.RuntimeParams["lock_timeout"] = "10000" + cfg.RuntimeParams["statement_timeout"] = strconv.FormatInt(min(timeout.Milliseconds(), 2147483647), 10) + db := stdlib.OpenDB(*cfg) + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(2) + db.SetConnMaxLifetime(5 * time.Minute) + defer db.Close() + if err := db.PingContext(ctx); err != nil { + return fail("connect to PostgreSQL", err) + } + + locker, err := lock.NewPostgresSessionLocker( + lock.WithLockTimeout(1, 120), + lock.WithUnlockTimeout(1, 5), + ) + if err != nil { + return fail("configure migration lock", err) + } + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files, + goose.WithSessionLocker(boundedSessionLocker{locker}), + goose.WithDisableGlobalRegistry(true), + goose.WithSlog(logger), + // Goose verbose logging includes SQL statements. Log only selected + // migration metadata ourselves, even when a migration fails. + goose.WithVerbose(false), + ) + if err != nil { + return fail("load embedded migrations", err) + } + if command == "status" { + statuses, err := provider.Status(ctx) + if err != nil { + return fail("read migration status", err) + } + for _, status := range statuses { + logger.Info("migration status", "file", status.Source.Path, + "version", status.Source.Version, "state", status.State) + } + return nil + } + started := time.Now() + results, err := provider.Up(ctx) + if err != nil { + return fail("apply pending migrations", err) + } + for _, result := range results { + logger.Info("migration applied", "file", result.Source.Path, + "version", result.Source.Version, "duration_seconds", result.Duration.Seconds()) + } + logger.Info("migrations completed", "applied", len(results), + "duration_seconds", time.Since(started).Seconds()) + return nil +} diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go new file mode 100644 index 00000000..745a8605 --- /dev/null +++ b/cmd/migrate/main_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "highload/db/migrations" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +func TestConnectionSecretErrorsAreSafe(t *testing.T) { + for _, test := range []struct { + name string + content string + missing bool + }{ + {name: "missing", missing: true}, + {name: "empty", content: " \n\t"}, + {name: "malformed", content: "postgres://synthetic:fixture-password@localhost/%zz"}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "database_url") + if !test.missing { + if err := os.WriteFile(path, []byte(test.content), 0600); err != nil { + t.Fatal("cannot create test secret") + } + } + t.Setenv("DATABASE_URL_FILE", path) + logger := slog.New(slog.NewJSONHandler(io.Discard, nil)) + err := run(context.Background(), "up", time.Second, logger) + if err == nil { + t.Fatal("invalid connection secret was accepted") + } + if strings.Contains(err.Error(), "fixture-password") || + strings.Contains(err.Error(), "postgres://") { + t.Fatal("migration error exposed a connection secret") + } + var classified *migrationError + if !errors.As(err, &classified) || classified.Unwrap() == nil { + t.Fatal("migration error lost its classification/cause") + } + }) + } +} + +func TestEmbeddedMigrationsAreVersionedAndCollectable(t *testing.T) { + cfg, err := pgx.ParseConfig("postgres://synthetic@localhost/synthetic?sslmode=disable") + if err != nil { + t.Fatal("cannot configure test driver") + } + db := stdlib.OpenDB(*cfg) + defer db.Close() + // NewProvider collects sources without connecting to PostgreSQL. + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files) + if err != nil { + t.Fatal("embedded migration names/versions cannot be collected by Goose") + } + sources := provider.ListSources() + if len(sources) == 0 { + t.Fatal("migrator build contains no SQL") + } + var previous int64 + for _, source := range sources { + if source.Version <= previous { + t.Fatal("embedded versions are not unique and increasing") + } + previous = source.Version + data, err := migrations.Files.ReadFile(source.Path) + if err != nil || !strings.Contains(string(data), "-- +goose Up") { + t.Fatal("embedded migration is unavailable or has no Up section") + } + } +} + +type deadlineProbe struct { + deadline time.Time + cancel error +} + +func (*deadlineProbe) SessionLock(context.Context, *sql.Conn) error { return nil } +func (p *deadlineProbe) SessionUnlock(ctx context.Context, _ *sql.Conn) error { + p.deadline, _ = ctx.Deadline() + p.cancel = ctx.Err() + return nil +} + +func TestUnlockHasBudgetEvenWithDetachedContext(t *testing.T) { + original, cancel := context.WithCancel(context.Background()) + cancel() + probe := &deadlineProbe{} + locker := boundedSessionLocker{probe} + started := time.Now() + if err := locker.SessionUnlock(context.WithoutCancel(original), nil); err != nil { + t.Fatal("unlock delegation failed") + } + if probe.cancel != nil || probe.deadline.IsZero() || + probe.deadline.Before(started) || probe.deadline.After(started.Add(6*time.Second)) { + t.Fatal("detached cleanup did not receive a bounded, live context") + } +} diff --git a/cmd/migrate/postgres_test.go b/cmd/migrate/postgres_test.go new file mode 100644 index 00000000..18110d8c --- /dev/null +++ b/cmd/migrate/postgres_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "log/slog" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/stdlib" +) + +// These tests need an explicitly supplied disposable PostgreSQL database. +// Each test creates and drops only its own randomly named schema. +func migrationPostgresFixture(t *testing.T) (*sql.DB, context.Context) { + t.Helper() + connectionURL := os.Getenv("MIGRATE_TEST_DATABASE_URL") + if connectionURL == "" { + t.Skip("set MIGRATE_TEST_DATABASE_URL to run PostgreSQL integration tests") + } + cfg, err := pgx.ParseConfig(connectionURL) + if err != nil { + t.Fatal("invalid test database configuration") + } + cfg.ConnectTimeout = 5 * time.Second + admin := stdlib.OpenDB(*cfg) + t.Cleanup(func() { admin.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + schema := "migration_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal("cannot create isolated migration test schema") + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := admin.ExecContext(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error("cannot remove isolated migration test schema") + } + }) + cfg.RuntimeParams["search_path"] = schema + // ConnString retains the original parsed URL; changing RuntimeParams alone + // would not pass search_path to the separate migrator connection. + scopedURL, err := url.Parse(connectionURL) + if err != nil || (scopedURL.Scheme != "postgres" && scopedURL.Scheme != "postgresql") { + t.Fatal("MIGRATE_TEST_DATABASE_URL must be a PostgreSQL URL") + } + query := scopedURL.Query() + query.Set("search_path", schema) + scopedURL.RawQuery = query.Encode() + path := filepath.Join(t.TempDir(), "database_url") + if err := os.WriteFile(path, []byte(scopedURL.String()), 0600); err != nil { + t.Fatal("cannot write test connection secret") + } + t.Setenv("DATABASE_URL_FILE", path) + db := stdlib.OpenDB(*cfg) + t.Cleanup(func() { db.Close() }) + return db, ctx +} + +func migrationLogRecords(t *testing.T, output *bytes.Buffer) []map[string]any { + t.Helper() + var records []map[string]any + for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) { + if len(line) == 0 { + continue + } + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal("migrator emitted a non-JSON log record") + } + for key := range record { + switch key { + case "time", "level", "msg", "file", "version", "state", "applied", "duration_seconds": + default: + t.Fatalf("unexpected log field %q; SQL or connection details must not be logged", key) + } + } + if strings.Contains(string(line), "CREATE TABLE") || strings.Contains(string(line), "postgres://") { + t.Fatal("migration log contains SQL or a connection URL") + } + records = append(records, record) + } + return records +} + +func completedMigrationCount(t *testing.T, output *bytes.Buffer) int { + t.Helper() + for _, record := range migrationLogRecords(t, output) { + if record["msg"] == "migrations completed" { + count, ok := record["applied"].(float64) + if !ok { + t.Fatal("migration completion log has no numeric applied count") + } + return int(count) + } + } + t.Fatal("successful migration run has no completion log") + return 0 +} + +const lab5MigrationCount = 8 + +func TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + if err := run(ctx, "up", time.Minute, logger); err != nil { + t.Fatal("migration of empty schema failed") + } + if count := completedMigrationCount(t, &output); count != lab5MigrationCount { + t.Fatalf("empty schema: applied %d migrations, want %d", count, lab5MigrationCount) + } + if _, err := db.ExecContext(ctx, "INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3)", + "migration_fixture", "Migration fixture", "synthetic-test-only-hash"); err != nil { + t.Fatal("cannot insert persistent migration fixture") + } + output.Reset() + if err := run(ctx, "up", time.Minute, logger); err != nil { + t.Fatal("repeat migration failed") + } + if count := completedMigrationCount(t, &output); count != 0 { + t.Fatalf("repeat run: applied %d migrations, want 0", count) + } + var users, versions int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM users WHERE username = $1", "migration_fixture").Scan(&users); err != nil || users != 1 { + t.Fatal("repeat migration lost or duplicated existing data") + } + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != lab5MigrationCount { + t.Fatal("repeat migration changed version history") + } + output.Reset() + if err := run(ctx, "status", time.Minute, logger); err != nil { + t.Fatal("migration status failed") + } + records := migrationLogRecords(t, &output) + if len(records) != lab5MigrationCount { + t.Fatal("status did not report all seven migrations") + } + for _, record := range records { + if record["state"] != "applied" { + t.Fatal("status did not report applied migrations") + } + } +} + +func TestPostgresConcurrentMigratorsApplyEachVersionOnce(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + var outputs [2]bytes.Buffer + var runErrors [2]error + var workers sync.WaitGroup + start := make(chan struct{}) + for i := range outputs { + workers.Add(1) + go func(i int) { + defer workers.Done() + <-start + runErrors[i] = run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&outputs[i], nil))) + }(i) + } + close(start) + workers.Wait() + var applied int + for i, err := range runErrors { + if err != nil { + t.Fatal("concurrent migration run failed") + } + applied += completedMigrationCount(t, &outputs[i]) + } + if applied != lab5MigrationCount { + t.Fatalf("concurrent runners applied %d migrations in total, want %d", applied, lab5MigrationCount) + } + var versions, distinctVersions int + if err := db.QueryRowContext(ctx, "SELECT count(*), count(DISTINCT version_id) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions, &distinctVersions); err != nil || versions != lab5MigrationCount || distinctVersions != lab5MigrationCount { + t.Fatal("concurrent runners duplicated or lost version history") + } +} + +func TestPostgresMigrationFailureRollsBackAndPreservesCause(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + // The first migration creates users, then conflicts with this chats table. + if _, err := db.ExecContext(ctx, "CREATE TABLE chats (marker text)"); err != nil { + t.Fatal("cannot create conflicting test table") + } + var output bytes.Buffer + err := run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&output, nil))) + var pgErr *pgconn.PgError + if err == nil || err.Error() != "apply pending migrations" || !errors.As(err, &pgErr) || pgErr.Code != "42P07" { + t.Fatal("migration failure did not retain a safely reportable duplicate-table error") + } + migrationLogRecords(t, &output) + var usersAbsent bool + if err := db.QueryRowContext(ctx, "SELECT to_regclass('users') IS NULL").Scan(&usersAbsent); err != nil || !usersAbsent { + t.Fatal("failed migration left a partially applied users table") + } + var versions int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != 0 { + t.Fatal("failed migration was recorded as applied") + } +} diff --git a/cmd/migrate/roundtrip_test.go b/cmd/migrate/roundtrip_test.go new file mode 100644 index 00000000..23caa15b --- /dev/null +++ b/cmd/migrate/roundtrip_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "database/sql" + "io" + "log/slog" + "testing" + + "github.com/pressly/goose/v3" + "highload/db/migrations" +) + +func sessionSchema(t *testing.T, db *sql.DB, ctx context.Context) []string { + t.Helper() + rows, err := db.QueryContext(ctx, ` + SELECT 'column:'||attname||':'||format_type(atttypid,atttypmod)||':'||attnotnull||':'|| + COALESCE(pg_get_expr(d.adbin,d.adrelid),'') + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid='sessions'::regclass AND a.attnum>0 AND NOT a.attisdropped + UNION ALL SELECT 'constraint:'||conname||':'||pg_get_constraintdef(oid) + FROM pg_constraint WHERE conrelid='sessions'::regclass + UNION ALL SELECT 'index:'||indexname||':'||indexdef + FROM pg_indexes WHERE schemaname=current_schema() AND tablename='sessions' + ORDER BY 1`) + if err != nil { + t.Fatal("cannot inspect restored session columns/defaults/constraints/indexes") + } + defer rows.Close() + var result []string + for rows.Next() { + var s string + if rows.Scan(&s) != nil { + t.Fatal("schema scan failed") + } + result = append(result, s) + } + if rows.Err() != nil || len(result) < 10 { + t.Fatal("incomplete session structure snapshot") + } + return result +} + +func businessData(t *testing.T, db *sql.DB, ctx context.Context) []string { + t.Helper() + result := []string{} + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + rows, err := db.QueryContext(ctx, "SELECT row_to_json(t)::text FROM "+table+" t ORDER BY row_to_json(t)::text") + if err != nil { + t.Fatal("business table disappeared during migration") + } + count := 0 + for rows.Next() { + var value string + if rows.Scan(&value) != nil { + t.Fatal("data scan failed") + } + result = append(result, table+value) + count++ + } + err = rows.Err() + rows.Close() + if err != nil || count == 0 { + t.Fatal("business fixture missing during migration") + } + } + return result +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestPostgresMigrationThreeDownTwoDownOneAndReapply(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files, + goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil)))) + if err != nil { + t.Fatal("cannot load actual embedded migrations") + } + if _, err = provider.UpTo(ctx, 1); err != nil { + t.Fatal("cannot apply version 1") + } + v1 := sessionSchema(t, db, ctx) + if _, err = db.ExecContext(ctx, ` + INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES + ('00000000-0000-4000-8000-000000000001','roundtrip_user','Roundtrip','synthetic-hash','2026-01-01T00:00:00Z'); + INSERT INTO chats(id,title,created_at) VALUES + ('00000000-0000-4000-8000-000000000002','Roundtrip','2026-01-01T00:00:00Z'); + INSERT INTO chat_members VALUES('00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001'); + INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES + ('00000000-0000-4000-8000-000000000003','00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001','Keep this message','2026-01-01T00:00:00Z'); + INSERT INTO sessions VALUES(decode(repeat('ab',32),'hex'),'00000000-0000-4000-8000-000000000001','2026-01-01T00:00:00Z','2026-01-02T00:00:00Z',NULL); + `); err != nil { + t.Fatal("cannot create migration preservation fixtures") + } + original := businessData(t, db, ctx) + if _, err = provider.UpTo(ctx, 2); err != nil { + t.Fatal("cannot apply version 2") + } + v2 := sessionSchema(t, db, ctx) + assertVersion := func(want int64) { + t.Helper() + got, e := provider.GetDBVersion(ctx) + if e != nil || got != want { + t.Fatalf("want migration version %d, got %d", want, got) + } + if !equalStrings(original, businessData(t, db, ctx)) { + t.Fatal("migration changed unrelated users/chats/members/messages") + } + } + assertAbsent := func() { + t.Helper() + var absent bool + if db.QueryRowContext(ctx, "SELECT to_regclass('sessions') IS NULL").Scan(&absent) != nil || !absent { + t.Fatal("version 3 must remove SQL sessions") + } + } + if _, err = provider.Up(ctx); err != nil { + t.Fatal("cannot migrate to latest version") + } + assertVersion(lab5MigrationCount) + assertAbsent() + if _, err = provider.DownTo(ctx, 2); err != nil { + t.Fatal("latest -> 2 failed") + } + assertVersion(2) + if !equalStrings(v2, sessionSchema(t, db, ctx)) { + t.Fatal("3 -> 2 did not restore exact version-2 session columns, defaults, keys, FK, CHECK and indexes") + } + var count int + if db.QueryRowContext(ctx, "SELECT count(*) FROM sessions").Scan(&count) != nil || count != 0 { + t.Fatal("rollback must restore an empty sessions table; deleted credentials cannot be resurrected") + } + if _, err = db.ExecContext(ctx, `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`); err != nil { + t.Fatal("restored table/default/FK cannot accept a valid session") + } + for _, statement := range []string{ + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000099',now()+interval '1 hour')`, + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000001',now()-interval '1 hour')`, + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`, + } { + if _, err = db.ExecContext(ctx, statement); err == nil { + t.Fatal("restored table lost FK, expiry CHECK or primary key") + } + } + if _, err = provider.DownTo(ctx, 1); err != nil { + t.Fatal("2 -> 1 failed") + } + assertVersion(1) + if !equalStrings(v1, sessionSchema(t, db, ctx)) { + t.Fatal("2 -> 1 did not restore original sessions structure without created_at default") + } + if _, err = provider.Up(ctx); err != nil { + t.Fatal("1 -> latest reapplication failed") + } + assertVersion(lab5MigrationCount) + assertAbsent() +} diff --git a/cmd/relay/main.go b/cmd/relay/main.go new file mode 100644 index 00000000..a8861741 --- /dev/null +++ b/cmd/relay/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "highload/internal/broker" + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/fault" + "highload/internal/repository/postgres" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx); err != nil { + slog.Error("relay startup failed", "error", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + cfg, err := config.LoadPipeline() + if err != nil { + return err + } + if cfg.BrokerURL == "" { + return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL") + } + url, err := config.LoadPostgresURL() + if err != nil { + return err + } + pool, err := database.NewPostgres(ctx, url) + if err != nil { + return fmt.Errorf("PostgreSQL unavailable") + } + defer pool.Close() + repo := postgres.NewAttachmentRepository(sqlc.New(pool), pool) + + recovered := make(chan struct{}) + go func() { + defer close(recovered) + for ctx.Err() == nil { + if n, err := repo.EnqueueExpiredLeases(ctx, 32); err != nil && ctx.Err() == nil { + slog.Warn("lease recovery failed", "error", err) + } else if n > 0 { + slog.Info("expired leases enqueued", "count", n) + } + select { + case <-ctx.Done(): + case <-time.After(250 * time.Millisecond): + } + } + }() + for ctx.Err() == nil { + if err := publish(ctx, repo, cfg.BrokerURL); err != nil && ctx.Err() == nil { + slog.Warn("relay reconnecting", "error", err) + } + select { + case <-ctx.Done(): + case <-time.After(500 * time.Millisecond): + } + } + <-recovered + return nil +} + +func publish(ctx context.Context, repo *postgres.AttachmentRepository, url string) error { + conn, ch, err := broker.Connect(url) + if err != nil { + return err + } + defer broker.Close(conn) + publisher, err := broker.NewPublisher(ch) + if err != nil { + return err + } + for ctx.Err() == nil { + sent, err := repo.PublishPending(ctx, func(txCtx context.Context, event sqlc.Outbox) error { + if err := fault.Pause(txCtx, "before_publish", event.JobID.String()); err != nil { + return err + } + confirmCtx, cancel := context.WithTimeout(txCtx, 5*time.Second) + err := publisher.Publish(confirmCtx, event.ID.String(), event.Payload) + cancel() + if err != nil { + return err + } + return fault.Pause(txCtx, "after_publish", event.JobID.String()) + }) + if err != nil { + return err + } + if !sent { + if conn.IsClosed() { + return fmt.Errorf("RabbitMQ connection closed") + } + select { + case <-ctx.Done(): + case <-time.After(150 * time.Millisecond): + } + } + } + return nil +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..235ae37e --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "sync" + "syscall" + + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/pipeline" + "highload/internal/repository/postgres" + "highload/internal/storage/s3" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx); err != nil { + slog.Error("worker startup failed", "error", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + cfg, err := config.LoadPipeline() + if err != nil { + return err + } + if cfg.BrokerURL == "" { + return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL") + } + url, storageConfig, err := config.LoadAttachmentCleanup() + if err != nil { + return err + } + pool, err := database.NewPostgres(ctx, url) + if err != nil { + return fmt.Errorf("PostgreSQL unavailable") + } + defer pool.Close() + objects, err := s3.NewStorage(ctx, &storageConfig) + if err != nil { + return fmt.Errorf("S3 configuration invalid") + } + worker := pipeline.Worker{Repo: postgres.NewAttachmentRepository(sqlc.New(pool), pool), Objects: objects, Config: cfg} + slog.Info("worker started", "concurrency", cfg.Concurrency, "prefetch_per_consumer", cfg.Prefetch) + var lanes sync.WaitGroup + for range cfg.Concurrency { + lanes.Go(func() { worker.Serve(ctx) }) + } + lanes.Wait() + return nil +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..3c90ce45 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,298 @@ +name: highload-lab5 + +services: + api: + build: + context: . + dockerfile: Dockerfile + scale: 2 + cpus: "2.0" + mem_limit: 512m + expose: + - "8080" + env_file: + - .env + environment: + SESSION_STORE: redis + REDIS_ADDR: redis:6379 + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=32m,mode=1777 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + attachment-cleanup: + build: + context: . + dockerfile: Dockerfile + target: cleanup + command: ["cleanup", "-dry-run=${LAB5_CLEANUP_DRY_RUN:-false}"] + cpus: "0.5" + mem_limit: 128m + env_file: + - .env + environment: + BROKER_URL_FILE: "" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + relay: + build: + context: . + dockerfile: Dockerfile + target: relay + env_file: [.env] + environment: + BROKER_URL_FILE: /run/secrets/broker_url + secrets: [database_url, broker_url] + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + cpus: "0.5" + mem_limit: 128m + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Dockerfile.worker + env_file: [.env] + environment: + BROKER_URL_FILE: /run/secrets/broker_url + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + cpus: "2.0" + mem_limit: 1536m + read_only: true + tmpfs: + - /tmp:size=768m,mode=1777 + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + restart: unless-stopped + stop_grace_period: 15s + + rabbitmq: + image: rabbitmq:4.2.5-alpine@sha256:d4a3806070111cc69130a6542aa4049b612789852ef11fc3862abc94631b2572 + hostname: rabbitmq + entrypoint: ["/bin/sh", "-ec"] + command: + - | + export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/rabbitmq_password)" + exec docker-entrypoint.sh rabbitmq-server + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_VHOST: messenger + secrets: [rabbitmq_password] + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 5s + timeout: 5s + retries: 12 + mem_limit: 768m + cpus: "1.0" + restart: unless-stopped + stop_grace_period: 30s + + lb: + image: haproxy:3.2.24-alpine@sha256:1aca32ee932b96d2a75f1882909325839dd23c138ba9c6fd26e4d88179f30f13 + cpus: "1.0" + mem_limit: 256m + ports: + - "127.0.0.1:${LAB5_HTTP_PORT:-8085}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + restart: unless-stopped + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + migrate: + build: + context: . + dockerfile: Dockerfile.migrate + cpus: "1.0" + mem_limit: 256m + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + stop_grace_period: 20s + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + postgres: + image: postgres:18@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722 + cpus: "2.0" + mem_limit: 512m + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + ports: + - "127.0.0.1:${LAB5_POSTGRES_PORT:-15485}:5432" + volumes: + - postgres_data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + stop_grace_period: 60s + + redis: + image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 + cpus: "0.5" + mem_limit: 128m + restart: unless-stopped + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + s3: + image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + cpus: "1.0" + mem_limit: 512m + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_root_access_key + RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_root_secret_key + RUSTFS_CONSOLE_ENABLE: "false" + secrets: + - rustfs_root_access_key + - rustfs_root_secret_key + volumes: + - rustfs_data:/data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 30s + restart: unless-stopped + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + s3-init: + image: rustfs/rc:v0.1.30@sha256:1296510ec27800668d26a8bdca6251e587086187a9368c7b80134a169e8a3bb8 + cpus: "0.5" + mem_limit: 128m + entrypoint: ["/bin/sh", "/bootstrap.sh"] + environment: + HOME: /tmp + S3_BUCKET: messenger-attachments + S3_REGION: us-east-1 + secrets: + - rustfs_root_access_key + - rustfs_root_secret_key + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/s3/bootstrap.sh:/bootstrap.sh:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + read_only: true + tmpfs: + - /tmp:uid=100,gid=101,mode=0700 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + +secrets: + broker_url: + file: ./secrets/broker_url + rabbitmq_password: + file: ./secrets/rabbitmq_password + database_url: + file: ./secrets/database_url + postgres_password: + file: ./secrets/postgres_password + rustfs_root_access_key: + file: ./secrets/rustfs_root_access_key + rustfs_root_secret_key: + file: ./secrets/rustfs_root_secret_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + rabbitmq_data: + postgres_data: + redis_data: + rustfs_data: diff --git a/db/migrations/00001_init_schema.sql b/db/migrations/00001_init_schema.sql new file mode 100644 index 00000000..1edaef66 --- /dev/null +++ b/db/migrations/00001_init_schema.sql @@ -0,0 +1,79 @@ +-- +goose Up +CREATE TABLE users +( + id uuid PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT users_username_format_check CHECK (username COLLATE "C" ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length_check CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_display_name_not_blank_check CHECK (char_length(btrim(display_name)) > 0), + CONSTRAINT users_password_hash_not_empty_check CHECK (char_length(password_hash) > 0) + +); + + + +CREATE TABLE chats +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT chats_title_length_check CHECK (char_length(title) BETWEEN 1 AND 100 ), + CONSTRAINT chats_title_not_blank_check CHECK (char_length(btrim(title)) > 0) + +); + +CREATE TABLE chat_members +( + chat_id UUID NOT NULL REFERENCES chats (id), + user_id UUID NOT NULL REFERENCES users (id), + + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX idx_user_chat_id ON chat_members (user_id, chat_id); + + +CREATE TABLE messages +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + + CONSTRAINT messages_text_length_check CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_text_not_blank_check CHECK (char_length(btrim(text)) > 0) +); + +CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id); + + +CREATE TABLE sessions +( + token_hash BYTEA PRIMARY KEY NOT NULL, + user_id UUID NOT NULL REFERENCES users (id), + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ NULL, + + CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at) +); + +CREATE INDEX idx_session_user_id ON sessions (user_id); + + +-- +goose Down + +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS chat_members; +DROP TABLE IF EXISTS chats; +DROP TABLE IF EXISTS users; diff --git a/db/migrations/00002_add_created_at_defaults.sql b/db/migrations/00002_add_created_at_defaults.sql new file mode 100644 index 00000000..e6c9b1df --- /dev/null +++ b/db/migrations/00002_add_created_at_defaults.sql @@ -0,0 +1,20 @@ +-- +goose Up +ALTER TABLE users + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE chats + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE messages + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE sessions + ALTER COLUMN created_at SET DEFAULT now(); + +-- +goose Down + +ALTER TABLE sessions + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE messages + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE chats + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE users + ALTER COLUMN created_at DROP DEFAULT; diff --git a/db/migrations/00003_drop_session_table.sql b/db/migrations/00003_drop_session_table.sql new file mode 100644 index 00000000..e392cd89 --- /dev/null +++ b/db/migrations/00003_drop_session_table.sql @@ -0,0 +1,17 @@ +-- +goose Up +DROP TABLE IF EXISTS sessions; + +-- +goose Down +-- Restore the schema at version 2; deleted session data cannot be recovered. +CREATE TABLE sessions +( + token_hash BYTEA PRIMARY KEY NOT NULL, + user_id UUID NOT NULL REFERENCES users (id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ NULL, + + CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at) +); + +CREATE INDEX idx_session_user_id ON sessions (user_id); diff --git a/db/migrations/00004_message_idempotency.sql b/db/migrations/00004_message_idempotency.sql new file mode 100644 index 00000000..54fdfe2a --- /dev/null +++ b/db/migrations/00004_message_idempotency.sql @@ -0,0 +1,23 @@ +-- +goose Up + +CREATE TABLE message_idempotency +( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + idempotency_key VARCHAR(128) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'), + + PRIMARY KEY (user_id, chat_id, idempotency_key), + UNIQUE (message_id), + + CONSTRAINT message_idempotency_key_format_check + CHECK (idempotency_key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_expiry_check + CHECK (expires_at > created_at) +); + +-- +goose Down + +DROP TABLE IF EXISTS message_idempotency; diff --git a/db/migrations/00005_attachments.sql b/db/migrations/00005_attachments.sql new file mode 100644 index 00000000..0e8a790f --- /dev/null +++ b/db/migrations/00005_attachments.sql @@ -0,0 +1,54 @@ +-- +goose Up +CREATE TABLE attachments +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename text NOT NULL, + object_key text UNIQUE NOT NULL, + content_type VARCHAR(10) NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT fk_attachments_chat_member FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id), + CONSTRAINT attachments_sha256_format_check CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'), + CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_filename_check CHECK ( + char_length(filename) BETWEEN 1 AND 255 + AND filename !~ '[/\\]' + ), + CONSTRAINT attachments_size_bytes_check CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_length_check CHECK (char_length(object_key) > 0) + +); + +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE messages_attachments +( + message_id UUID NOT NULL , + attachment_id UUID NOT NULL , + chat_id UUID NOT NULL, + position SMALLINT NOT NULL, + + + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9), + + CONSTRAINT message_attachments_message_chat_fk + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id), + CONSTRAINT message_attachments_attachment_chat_fk + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + + +-- +goose Down +DROP TABLE IF EXISTS messages_attachments; +DROP TABLE IF EXISTS attachments; +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_unique; diff --git a/db/migrations/00006_allow_attachment_only_messages.sql b/db/migrations/00006_allow_attachment_only_messages.sql new file mode 100644 index 00000000..75b967ba --- /dev/null +++ b/db/migrations/00006_allow_attachment_only_messages.sql @@ -0,0 +1,18 @@ +-- +goose Up +ALTER TABLE messages + DROP CONSTRAINT messages_text_length_check, + DROP CONSTRAINT messages_text_not_blank_check; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length_check + CHECK (char_length(text) <= 4000); + +-- +goose Down +ALTER TABLE messages + DROP CONSTRAINT messages_text_length_check; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length_check + CHECK (char_length(text) BETWEEN 1 AND 4000), + ADD CONSTRAINT messages_text_not_blank_check + CHECK (char_length(btrim(text)) > 0); \ No newline at end of file diff --git a/db/migrations/00007_async_attachments.sql b/db/migrations/00007_async_attachments.sql new file mode 100644 index 00000000..5740d797 --- /dev/null +++ b/db/migrations/00007_async_attachments.sql @@ -0,0 +1,144 @@ +-- +goose Up +ALTER TABLE attachments + ADD COLUMN status TEXT NOT NULL DEFAULT 'queued', + ADD COLUMN error_code TEXT, + ADD COLUMN staging_key TEXT, + ADD COLUMN pipeline_version INT NOT NULL DEFAULT 1, + ALTER COLUMN object_key DROP NOT NULL; + +-- Retain the original download path while the copied synchronous API is in use. +UPDATE attachments SET staging_key = object_key; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD CONSTRAINT attachments_pipeline_check CHECK (pipeline_version > 0), + ADD CONSTRAINT attachments_staging_key_check CHECK (length(staging_key) > 0), + ADD CONSTRAINT attachments_object_state_check CHECK ( + (status NOT IN ('queued', 'processing') OR staging_key IS NOT NULL) + AND (status <> 'ready' OR object_key IS NOT NULL) + ), + ADD CONSTRAINT attachments_error_check CHECK ( + (status = 'failed') = (error_code IS NOT NULL) + AND (error_code IS NULL OR error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$') + ); + +CREATE UNIQUE INDEX attachments_staging_key_unique ON attachments (staging_key) + WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INT NOT NULL CHECK (pipeline_version > 0), + attempts_started INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 5, + next_attempt_at TIMESTAMPTZ DEFAULT now(), + lease_generation INT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_until TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version), + CHECK (attempts_started >= 0 AND max_attempts > 0 AND attempts_started <= max_attempts), + CHECK (lease_until IS NULL OR next_attempt_at IS NULL) +); + +CREATE INDEX attachment_jobs_expired_lease_idx ON attachment_jobs (lease_until, id) + WHERE lease_until IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL, + pipeline_version INT NOT NULL, + name TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE CHECK (length(object_key) > 0), + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + width INT NOT NULL CHECK (width > 0), + height INT NOT NULL CHECK (height > 0), + sha256 TEXT NOT NULL CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (attachment_id, pipeline_version, name), + FOREIGN KEY (attachment_id, pipeline_version) + REFERENCES attachment_jobs(attachment_id, pipeline_version), + CHECK ( + (name IN ('thumbnail', 'crop') AND content_type = 'image/jpeg') + OR (name = 'optimized' AND content_type = 'image/webp') + ) +); + +CREATE TABLE attachment_attempts ( + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + lease_generation INT NOT NULL CHECK (lease_generation > 0), + attempt INT NOT NULL CHECK (attempt > 0), + started_at TIMESTAMPTZ NOT NULL, + finished_at TIMESTAMPTZ, + outcome TEXT CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired')), + error_code TEXT CHECK (error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$'), + PRIMARY KEY (job_id, lease_generation), + UNIQUE (job_id, attempt), + CHECK ((outcome IS NULL) = (finished_at IS NULL)) +); + +CREATE TABLE attachment_replays ( + request_id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + operator_id TEXT NOT NULL CHECK (length(btrim(operator_id)) > 0), + reason TEXT NOT NULL CHECK (length(btrim(reason)) > 0), + extra_attempts INT NOT NULL CHECK (extra_attempts BETWEEN 1 AND 5), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE outbox ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + payload JSONB NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + sent_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + dedup_key TEXT UNIQUE NOT NULL CHECK (length(dedup_key) > 0) +); + +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE sent_at IS NULL; + +-- +goose StatementBegin +CREATE FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz) +RETURNS jsonb LANGUAGE sql STABLE +AS $$ + SELECT jsonb_build_object( + 'schema_version', 1, + 'type', 'attachment.process.v1', + 'job_id', $1, + 'attachment_id', $2, + 'pipeline_version', $3, + 'attempt', $4, + 'created_at', to_char($5 AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') + ); +$$; +-- +goose StatementEnd + +WITH jobs AS ( + INSERT INTO attachment_jobs (attachment_id, pipeline_version) + SELECT id, pipeline_version FROM attachments + RETURNING * +) +INSERT INTO outbox (job_id, payload, dedup_key) +SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at), + 'initial/' || id +FROM jobs; + +-- +goose Down +-- Restore the legacy reference when the staging original is the only copy. +-- SET NOT NULL aborts the rollback if an original no longer exists. +UPDATE attachments SET object_key = staging_key WHERE object_key IS NULL; +ALTER TABLE attachments ALTER COLUMN object_key SET NOT NULL; + +DROP FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz); +DROP TABLE outbox; +DROP TABLE attachment_replays; +DROP TABLE attachment_attempts; +DROP TABLE attachment_variants; +DROP TABLE attachment_jobs; + +ALTER TABLE attachments + DROP COLUMN status, + DROP COLUMN error_code, + DROP COLUMN staging_key, + DROP COLUMN pipeline_version; diff --git a/db/migrations/00008_versioned_pipeline.sql b/db/migrations/00008_versioned_pipeline.sql new file mode 100644 index 00000000..e5eff1d3 --- /dev/null +++ b/db/migrations/00008_versioned_pipeline.sql @@ -0,0 +1,27 @@ +-- +goose Up +ALTER TABLE attachment_jobs ADD COLUMN published_original_key TEXT + CHECK (length(published_original_key) > 0); + +UPDATE attachment_jobs AS j +SET published_original_key = a.object_key +FROM attachments AS a +WHERE a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + AND a.status = 'ready'; + +CREATE INDEX attachment_jobs_published_original_idx ON attachment_jobs (published_original_key) + WHERE published_original_key IS NOT NULL; + +ALTER TABLE attachment_attempts + DROP CONSTRAINT attachment_attempts_outcome_check, + ADD CONSTRAINT attachment_attempts_outcome_check + CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired', 'superseded')); + +-- +goose Down +UPDATE attachment_attempts SET outcome = 'failed' WHERE outcome = 'superseded'; + +ALTER TABLE attachment_attempts + DROP CONSTRAINT attachment_attempts_outcome_check, + ADD CONSTRAINT attachment_attempts_outcome_check + CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired')); + +ALTER TABLE attachment_jobs DROP COLUMN published_original_key; diff --git a/db/migrations/embed.go b/db/migrations/embed.go new file mode 100644 index 00000000..ee488c7d --- /dev/null +++ b/db/migrations/embed.go @@ -0,0 +1,9 @@ +package migrations + +import "embed" + +// Files contains the SQL shipped with this exact migrator build. +// Add schema changes as new numbered files; do not edit applied migrations. +// +//go:embed *.sql +var Files embed.FS diff --git a/db/queries/attachment_variants.sql b/db/queries/attachment_variants.sql new file mode 100644 index 00000000..2ed8489b --- /dev/null +++ b/db/queries/attachment_variants.sql @@ -0,0 +1,11 @@ +-- name: CreateAttachmentVariant :exec +INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9 +); + +-- name: ListAttachmentVariants :many +SELECT * FROM attachment_variants +WHERE attachment_id = $1 AND pipeline_version = $2 +ORDER BY name; diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql new file mode 100644 index 00000000..6daba6a8 --- /dev/null +++ b/db/queries/attachments.sql @@ -0,0 +1,69 @@ +-- name: CreateAttachment :one +-- Compatibility for the synchronous API until CreateQueuedAttachment is wired in. +WITH legacy AS ( + INSERT INTO attachments + (chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256) + VALUES (sqlc.arg(chat_id), sqlc.arg(owner_id), sqlc.arg(filename), + sqlc.arg(object_key)::text, sqlc.arg(object_key)::text, + sqlc.arg(content_type), sqlc.arg(size_bytes), sqlc.arg(sha256)) + RETURNING * +), job AS ( + INSERT INTO attachment_jobs (attachment_id, pipeline_version) + SELECT id, pipeline_version FROM legacy + RETURNING * +), event AS ( + INSERT INTO outbox (job_id, payload, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at), + 'initial/' || id + FROM job + RETURNING job_id +) +SELECT legacy.* FROM legacy CROSS JOIN event; + +-- name: GetAttachmentForUser :one +SELECT a.* +FROM attachments AS a +JOIN chat_members AS cm ON cm.chat_id = a.chat_id +WHERE a.id = sqlc.arg(attachment_id) + AND cm.user_id = sqlc.arg(user_id); + + +-- name: GetAttachmentsForMessages :many +SELECT message_id, attachment_id, position +FROM messages_attachments +WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[]) +ORDER BY message_id, position; + + +-- name: AttachmentExistsByObjectKey :one +SELECT EXISTS ( + SELECT 1 FROM attachments + WHERE object_key = sqlc.arg(object_key)::text OR staging_key = sqlc.arg(object_key)::text + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = sqlc.arg(object_key)::text + UNION ALL + SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text +); + +-- name: TryLockAttachmentObjectKey :one +SELECT pg_try_advisory_xact_lock( + hashtextextended(sqlc.arg(object_key)::text, 0) +)::boolean AS locked; + +-- name: ListAttachmentKeys :many +SELECT id, object_key, created_at +FROM attachments +WHERE object_key > sqlc.arg(after_key) +ORDER BY object_key +LIMIT sqlc.arg(page_size); + +-- name: GetAttachmentByID :one +SELECT * FROM attachments WHERE id = sqlc.arg(attachment_id); + + +-- name: CreateQueuedAttachment :one +INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, 'queued' +) RETURNING *; diff --git a/db/queries/attachments_jobs.sql b/db/queries/attachments_jobs.sql new file mode 100644 index 00000000..86cc3a99 --- /dev/null +++ b/db/queries/attachments_jobs.sql @@ -0,0 +1,197 @@ +-- name: CreateAttachmentJob :one +INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts) +VALUES (sqlc.arg(attachment_id), sqlc.arg(pipeline_version), + sqlc.arg(available_at)::timestamptz, sqlc.arg(max_attempts)) +RETURNING *; + +-- name: ClaimJob :one +WITH claimed AS ( + UPDATE attachment_jobs AS j + SET lease_generation = j.lease_generation + 1, + lease_until = sqlc.arg(new_lease_until)::timestamptz, + attempts_started = j.attempts_started + 1, + next_attempt_at = NULL, + updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = sqlc.arg(job_id) + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + AND j.attempts_started < j.max_attempts + AND sqlc.arg(new_lease_until)::timestamptz > clock_timestamp() + AND ( + (a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp()) + OR (a.status = 'processing' AND j.lease_until <= clock_timestamp()) + ) + RETURNING j.* +), expired AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at + FROM claimed + WHERE attempt.job_id = claimed.id + AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL +), recorded AS ( + INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at) + SELECT id, lease_generation, attempts_started, updated_at FROM claimed + RETURNING job_id +), processing AS ( + UPDATE attachments AS a + SET status = 'processing', updated_at = claimed.updated_at + FROM claimed + WHERE a.id = claimed.attachment_id + RETURNING a.id +) +SELECT claimed.* FROM claimed JOIN recorded ON recorded.job_id = claimed.id +JOIN processing ON processing.id = claimed.attachment_id; + +-- name: LockActiveAttachmentJob :one +-- Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion. +SELECT j.* +FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id +WHERE j.id = sqlc.arg(job_id) + AND j.lease_generation = sqlc.arg(lease_generation) + AND j.lease_until > clock_timestamp() + AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version +FOR UPDATE OF j; + +-- name: CompleteJob :execrows +WITH completed AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(), + published_original_key = sqlc.arg(object_key)::text + FROM attachments AS a + WHERE j.id = sqlc.arg(job_id) + AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp() + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + AND (SELECT count(*) FROM attachment_variants AS v + WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3 + RETURNING j.* +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'ready', finished_at = completed.updated_at + FROM completed + WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation +) +UPDATE attachments AS a +SET status = 'ready', object_key = sqlc.arg(object_key)::text, updated_at = completed.updated_at +FROM completed +WHERE a.id = completed.attachment_id; + +-- name: ScheduleRetry :execrows +WITH scheduled AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = sqlc.arg(available_at)::timestamptz, + updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = sqlc.arg(job_id) + AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp() + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + AND j.attempts_started < j.max_attempts + AND sqlc.arg(available_at)::timestamptz > clock_timestamp() + RETURNING j.* +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'retry', error_code = sqlc.arg(error_code)::text, finished_at = scheduled.updated_at + FROM scheduled + WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation +), event AS ( + INSERT INTO outbox (job_id, payload, available_at, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at), + next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1) + FROM scheduled +) +UPDATE attachments AS a +SET status = 'queued', updated_at = scheduled.updated_at +FROM scheduled WHERE a.id = scheduled.attachment_id; + +-- name: FailJob :execrows +WITH failed AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = sqlc.arg(job_id) + AND j.lease_generation = sqlc.arg(lease_generation) + AND ( + j.lease_until > clock_timestamp() + OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts + AND sqlc.arg(error_code)::text = 'retry_exhausted') + ) + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + RETURNING j.* +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'failed', error_code = sqlc.arg(error_code)::text, finished_at = failed.updated_at + FROM failed + WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation +) +UPDATE attachments AS a +SET status = 'failed', error_code = sqlc.arg(error_code)::text, updated_at = failed.updated_at +FROM failed WHERE a.id = failed.attachment_id; + +-- name: ReplayFailed :execrows +WITH replayed AS ( + UPDATE attachment_jobs AS j + SET max_attempts = j.attempts_started + sqlc.arg(extra_attempts)::int, + next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = sqlc.arg(job_id) + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + AND a.status = 'failed' AND a.staging_key IS NOT NULL + AND j.lease_until IS NULL AND j.next_attempt_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM attachment_replays AS r WHERE r.request_id = sqlc.arg(request_id)::uuid + ) + RETURNING j.* +), recorded AS ( + INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts) + SELECT sqlc.arg(request_id)::uuid, id, sqlc.arg(operator_id), sqlc.arg(reason), sqlc.arg(extra_attempts) + FROM replayed +), event AS ( + INSERT INTO outbox (job_id, payload, available_at, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at), + next_attempt_at, 'replay/' || id || '/' || sqlc.arg(request_id)::uuid + FROM replayed +) +UPDATE attachments AS a +SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at +FROM replayed WHERE a.id = replayed.attachment_id; + +-- name: GetAttachmentJobState :one +SELECT + j.id, j.attachment_id, j.pipeline_version, + CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status, + j.lease_generation, + j.lease_until, + coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired, + j.attempts_started, + j.max_attempts, + j.next_attempt_at +FROM attachment_jobs AS j +JOIN attachments AS a ON a.id = j.attachment_id +WHERE j.id = sqlc.arg(job_id)::uuid; + +-- name: ListFailedAttachmentJobs :many +SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at +FROM attachment_jobs j +JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version +WHERE a.status = 'failed' +ORDER BY a.updated_at DESC, j.id +LIMIT sqlc.arg(batch_size); + +-- name: LockAttachmentJobForPipeline :one +SELECT j.* +FROM attachment_jobs j +WHERE j.attachment_id = $1 AND j.pipeline_version = $2 +FOR UPDATE; + +-- name: LockAttachmentJobForReplay :one +SELECT j.* +FROM attachment_jobs j WHERE j.id = $1 +FOR UPDATE; + +-- name: ClearStagingReference :execrows +UPDATE attachments +SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END, + staging_key = NULL +WHERE id = sqlc.arg(id) AND staging_key = sqlc.arg(staging_key); + +-- name: GetAttachmentByStagingKey :one +SELECT * FROM attachments WHERE staging_key = sqlc.arg(staging_key)::text; diff --git a/db/queries/chat_members.sql b/db/queries/chat_members.sql new file mode 100644 index 00000000..811fdd8b --- /dev/null +++ b/db/queries/chat_members.sql @@ -0,0 +1,3 @@ +-- name: CreateChatMember :exec +INSERT INTO chat_members(chat_id, user_id) +VALUES ($1, $2); diff --git a/db/queries/chats.sql b/db/queries/chats.sql new file mode 100644 index 00000000..2bfe8e0f --- /dev/null +++ b/db/queries/chats.sql @@ -0,0 +1,34 @@ +-- name: CreateChat :one +INSERT INTO chats (title) +VALUES ($1) +RETURNING id, title, created_at; + + +-- name: GetChatByID :one + +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE c.id = $1 +GROUP BY c.id; + + +-- name: GetChatsByUserID :many +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE EXISTS (SELECT 1 + FROM chat_members membership + WHERE membership.chat_id = c.id + AND membership.user_id = sqlc.arg(user_id)) + AND ( + sqlc.narg(after_created_at)::timestamptz IS NULL + OR (c.created_at, c.id) > + ( + sqlc.narg(after_created_at)::timestamptz, + sqlc.narg(after_id)::uuid + ) + ) +GROUP BY c.id +ORDER BY c.created_at ASC, c.id ASC +LIMIT sqlc.arg(limit_count); \ No newline at end of file diff --git a/db/queries/message_idempotence.sql b/db/queries/message_idempotence.sql new file mode 100644 index 00000000..89088f01 --- /dev/null +++ b/db/queries/message_idempotence.sql @@ -0,0 +1,22 @@ +-- name: InsertMessageIdempotency :execrows +INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id) +VALUES ($1, $2, $3, $4) +ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING; + + +-- name: GetIdempotentMessage :one +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at +FROM message_idempotency AS i +JOIN messages AS m ON m.id = i.message_id +WHERE i.user_id = $1 + AND i.chat_id = $2 + AND i.idempotency_key = $3; + +-- name: LockMessageRequest :exec +SELECT pg_advisory_xact_lock(hashtextextended( + 'message/' || (sqlc.arg(user_id)::uuid)::text || '/' || + (sqlc.arg(chat_id)::uuid)::text || '/' || sqlc.arg(idempotency_key)::text, 0)); + +-- name: DeleteExpiredMessageIdempotency :exec +DELETE FROM message_idempotency +WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now(); diff --git a/db/queries/messages.sql b/db/queries/messages.sql new file mode 100644 index 00000000..f77fb037 --- /dev/null +++ b/db/queries/messages.sql @@ -0,0 +1,19 @@ +-- name: CreateMessage :one +INSERT INTO messages (chat_id, sender_id, text) +VALUES ($1, $2, $3) +RETURNING *; + +-- name: GetMessagesByChatID :many + +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = sqlc.arg(chat_id) + AND ( + sqlc.narg(after_created_at)::timestamptz IS NULL + OR (created_at, id) > + (sqlc.narg(after_created_at)::timestamptz, + sqlc.narg(after_id)::uuid + ) + ) +ORDER BY created_at ASC, id ASC +LIMIT sqlc.arg(limit_count); diff --git a/db/queries/messages_attachments.sql b/db/queries/messages_attachments.sql new file mode 100644 index 00000000..9a05d518 --- /dev/null +++ b/db/queries/messages_attachments.sql @@ -0,0 +1,16 @@ +-- name: CreateMessagesAttachments :execrows +INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) +SELECT $1, $2, $3, $4 +FROM attachments +WHERE id = $2 AND chat_id = $3; + +-- name: GetMessagesAttachments :many +SELECT attachment_id, position +FROM messages_attachments +WHERE message_id = sqlc.arg(message_id) +ORDER BY position ASC; + +-- name: LockAttachmentsForMessage :many +SELECT id, status FROM attachments +WHERE chat_id = sqlc.arg(chat_id) AND id = ANY(sqlc.arg(attachment_ids)::uuid[]) +ORDER BY id FOR SHARE; diff --git a/db/queries/outbox.sql b/db/queries/outbox.sql new file mode 100644 index 00000000..22081366 --- /dev/null +++ b/db/queries/outbox.sql @@ -0,0 +1,39 @@ +-- name: CreateInitialOutboxEvent :exec +INSERT INTO outbox (job_id, payload, available_at, dedup_key) +SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at), + next_attempt_at, 'initial/' || id +FROM attachment_jobs WHERE id = sqlc.arg(job_id)::uuid; + + +-- name: EnqueueExpiredLeases :many +WITH expired AS ( + SELECT j.*, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key + FROM attachment_jobs AS j + JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp() + AND NOT EXISTS ( + SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation + ) + ORDER BY j.lease_until, j.id + LIMIT sqlc.arg(batch_size) + FOR UPDATE OF j SKIP LOCKED +) +INSERT INTO outbox (job_id, payload, dedup_key) +SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, + LEAST(attempts_started + 1, max_attempts), clock_timestamp()), + recovery_key +FROM expired +ON CONFLICT (dedup_key) DO NOTHING +RETURNING *; + +-- name: NextPendingOutboxEvent :one +SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key +FROM outbox +WHERE sent_at IS NULL AND available_at <= statement_timestamp() +ORDER BY available_at, id +LIMIT 1 +FOR UPDATE SKIP LOCKED; + +-- name: MarkOutboxEventSent :execrows +UPDATE outbox SET sent_at = clock_timestamp() +WHERE id = sqlc.arg(id) AND sent_at IS NULL; diff --git a/db/queries/users.sql b/db/queries/users.sql new file mode 100644 index 00000000..730dfe25 --- /dev/null +++ b/db/queries/users.sql @@ -0,0 +1,21 @@ + +-- name: CreateUser :one + +INSERT INTO users (username, display_name, password_hash) +VALUES ($1, $2, $3) +RETURNING id, username, display_name, password_hash, created_at; + + + +-- name: GetUserByID :one + +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE id = $1; + + + +-- name: GetUserByUsername :one +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE username = $1; diff --git a/db/queries/versioned_pipeline.sql b/db/queries/versioned_pipeline.sql new file mode 100644 index 00000000..f1d5bb05 --- /dev/null +++ b/db/queries/versioned_pipeline.sql @@ -0,0 +1,30 @@ +-- name: SupersedeAttachmentJob :exec +WITH superseded AS ( + UPDATE attachment_jobs + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + WHERE id = sqlc.arg(job_id)::uuid + RETURNING id, updated_at +) +UPDATE attachment_attempts AS attempt +SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at +FROM superseded +WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL; + +-- name: QueueAttachmentUpgrade :one +UPDATE attachments +SET pipeline_version = 2, status = 'queued', staging_key = sqlc.arg(source_key)::text, + object_key = NULL, error_code = NULL, updated_at = clock_timestamp() +WHERE id = sqlc.arg(id)::uuid AND pipeline_version = 1 +RETURNING *; + +-- name: GetPipelineJobID :one +SELECT id FROM attachment_jobs +WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int; + +-- name: GetPublishedOriginalKey :one +SELECT published_original_key FROM attachment_jobs +WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int + AND published_original_key IS NOT NULL; + +-- name: IsPublishedOriginalKey :one +SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text); diff --git a/evidence/lab5/api-failover.jsonl b/evidence/lab5/api-failover.jsonl new file mode 100644 index 00000000..af4df7c3 --- /dev/null +++ b/evidence/lab5/api-failover.jsonl @@ -0,0 +1,7 @@ +{"event": "resource_sample", "t_s": 16.705083, "phase": "each-SIGTERM-read-write-with-load", "samples": ["966a4329e0d5\t36.80%\t15.35MiB / 512MiB", "33e9426ff450\t37.26%\t12.92MiB / 512MiB", "03a23dff1ff8\t32.09%\t77.19MiB / 256MiB"]} +{"event": "recovery", "t_s": 19.895726, "phase": "each-SIGTERM-read-write-with-load", "target": "966a4329e0d5", "seconds": 3.10449383300147} +{"event": "load_summary", "t_s": 25.620087, "phase": "each-SIGTERM-read-write-with-load", "window": "target 966a4329e0d5", "elapsed_s": 8.823298916002386, "attempts": 5984, "successes": 5978, "successful_rps": 677.5243655361141, "failures": 6, "error_ratio": 0.001002673796791444, "statuses": {"201": 2992, "200": 2986, "None": 6}, "p50_ms": 5.04, "p95_ms": 15.011, "p99_ms": 34.484} +{"event": "recovery", "t_s": 32.90963, "phase": "each-SIGTERM-read-write-with-load", "target": "33e9426ff450", "seconds": 3.301144083001418} +{"event": "load_summary", "t_s": 38.676268, "phase": "each-SIGTERM-read-write-with-load", "window": "target 33e9426ff450", "elapsed_s": 9.06493366601353, "attempts": 4430, "successes": 4424, "successful_rps": 488.0344592687499, "failures": 6, "error_ratio": 0.0013544018058690745, "statuses": {"201": 2215, "200": 2209, "None": 6}, "p50_ms": 5.677, "p95_ms": 24.386, "p99_ms": 65.408} +{"event": "load_summary", "t_s": 39.564574, "phase": "each-SIGTERM-read-write-with-load", "window": "full", "elapsed_s": 28.373479084009887, "attempts": 15861, "successes": 15843, "successful_rps": 558.3735414712838, "failures": 18, "error_ratio": 0.0011348590883298658, "statuses": {"201": 7926, "200": 7917, "None": 18}, "p50_ms": 5.516, "p95_ms": 21.228, "p99_ms": 47.711} +{"event": "load_history", "t_s": 40.011926, "phase": "each-SIGTERM-read-write-with-load", "observed": 7929, "confirmed": 7926, "offered": 7933} diff --git a/evidence/lab5/clean-run.md b/evidence/lab5/clean-run.md new file mode 100644 index 00000000..4d06acdf --- /dev/null +++ b/evidence/lab5/clean-run.md @@ -0,0 +1,39 @@ +# Чистый запуск и публичные проверки ЛР 5 + +15 ноября 2026 года. Apple M2, 8 ГиБ RAM. Docker Desktop 28.4.0, Linux ARM64, 8 доступных CPU и 4 ГБ RAM для VM. Go 1.27.1, sqlc 1.31.1. Чистый каталог исходников без `.env`, `secrets/`, Git и существующих volumes. Проект `lab5-final-vugkv4`, LB `127.0.0.1:18085`. + +```sh +export COMPOSE_PROJECT_NAME=lab5-final-vugkv4 +export LAB5_HTTP_PORT=18085 LAB5_POSTGRES_PORT=15486 +sh scripts/init-local.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +docker compose --env-file .env logs migrate +make test BASE_URL=http://127.0.0.1:18085 +make test-persistence BASE_URL=http://127.0.0.1:18085 ACTION_SCRIPT=scripts/restart.sh +make test-failover BASE_URL=http://127.0.0.1:18085 ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env up -d --no-deps api +``` + +Выдержка из отчёта Docker и скриптов проверки. + +```text +lab5-final-vugkv4-api-1 Healthy +lab5-final-vugkv4-api-2 Healthy +lab5-final-vugkv4-rabbitmq-1 Healthy +lab5-final-vugkv4-worker-1 Healthy +lab5-final-vugkv4-relay-1 Healthy +lab5-final-vugkv4-attachment-cleanup-1 Healthy +lab5-final-vugkv4-lb-1 Healthy +lab5-final-vugkv4-migrate-1 Exited +lab5-final-vugkv4-s3-init-1 Exited +{"msg":"migrations completed","applied":7,"duration_seconds":0.487786833} +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +{"observed_instances_before":2,"successful_writes":111,"failed_attempts":1,"first_recovery_seconds":3.04} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +Публичный failover остановил обслужившую запрос реплику API на 15-секундном окне нагрузки. После прогона реплика снова запущена. Переходный запрос покрылся повтором, подтверждённые записи сохранились. Лог мигратора до и после persistence/failover совпал побайтно. Штатный restart перезапускает существующие контейнеры по ID. [scripts/restart.sh](../../scripts/restart.sh). + +Локально выполнены `make check`, `make validate` с зависимостями из `requirements-dev.txt`, `sqlc compile`, `go vet ./...` и `go test -count=1 ./...`. Полный инфраструктурный прогон Go выполняется отдельно на стенде тестовика. `git diff --exit-code HEAD -- README.md COURSE.md contracts tests` подтвердил целостность задания и публичных тестов. diff --git a/evidence/lab5/crash-before-commit.jsonl b/evidence/lab5/crash-before-commit.jsonl new file mode 100644 index 00000000..eb537781 --- /dev/null +++ b/evidence/lab5/crash-before-commit.jsonl @@ -0,0 +1 @@ +{"scenario": "crash-staging-before-commit", "attachment_id": "75bc5374-82f0-4ecb-9428-bed7b411b798", "upload_acknowledged": false, "staging_present_after_crash": true, "postgres_rows": [0, 0, 0], "staging_removed": true, "metadata_after_restart": 404, "status": "PASS"} diff --git a/evidence/lab5/http-and-geometry.jsonl b/evidence/lab5/http-and-geometry.jsonl new file mode 100644 index 00000000..757504a1 --- /dev/null +++ b/evidence/lab5/http-and-geometry.jsonl @@ -0,0 +1,12 @@ +{"event": "result", "t_s": 0.049496, "phase": "health", "status": "PASS", "note": ""} +{"event": "result", "t_s": 1.596437, "phase": "upload-byte-pixel-media-limits", "status": "PASS", "note": ""} +{"event": "variant", "t_s": 2.910612, "phase": "async-output-variants", "name": "thumbnail", "width": 256, "height": 160, "bytes": 4654, "sha256": "d7cc662e82b52806da6245e82ad3c0c699ef7b4c6107cb2ee451e05930d3eabb"} +{"event": "variant", "t_s": 2.915311, "phase": "async-output-variants", "name": "crop", "width": 128, "height": 128, "bytes": 1808, "sha256": "507e48dc62a637e590ee652440576280e0d7e1de4bc3287010dc8ce104dc1697"} +{"event": "variant", "t_s": 2.919665, "phase": "async-output-variants", "name": "optimized", "width": 320, "height": 200, "bytes": 2462, "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb"} +{"event": "result", "t_s": 2.923246, "phase": "async-output-variants", "status": "PASS", "note": ""} +{"event": "result", "t_s": 6.717249, "phase": "idempotency-across-LB", "status": "PASS", "note": ""} +{"event": "result", "t_s": 13.465561, "phase": "CPU-RAM-ports", "status": "PASS", "note": ""} +{"event": "migrations", "t_s": 16.944801, "phase": "single-migration-start", "container": {"id": "082e76176ac67bbb25f0679db1c221f7cb21cd3c79277a1ea07309a36986c2a1", "alias": "082e76176ac6", "image": "sha256:7ea2786bc3979d268be58e18ce5015f48a24528e70ef5a77919fb15e275c5195", "state": "exited", "started_at": "2026-11-15T18:07:15.954914964Z", "finished_at": "2026-11-15T18:07:17.136262798Z", "exit_code": 0, "oom": false, "nano_cpus": 1000000000, "cpu_quota": 0, "cpu_period": 0, "memory": 268435456, "ports": {}, "bindings": {}, "network": "lab5-final-vugkv4_default", "publish_all": false}, "applied": [7], "versions": ["0|t|1", "1|t|1", "2|t|1", "3|t|1", "4|t|1", "5|t|1", "6|t|1", "7|t|1"]} +{"event": "result", "t_s": 16.944906, "phase": "single-migration-start", "status": "PASS", "note": ""} +{"event": "summary", "t_s": 16.944968, "phase": "single-migration-start", "PASS": 6, "FAIL": 0, "PENDING": 0} +{"scenario": "orientation-and-metadata", "status": "PASS", "fixtures": [{"filename": "wide.png", "attachment_id": "d9ac344a-26e1-4ac8-999b-0756abe247eb", "variants": {"crop": [128, 128], "optimized": [1280, 640], "thumbnail": [256, 128]}, "original_unchanged": true, "metadata_removed": true}, {"filename": "orientation6.jpg", "attachment_id": "16bec890-61b4-45ab-b690-87754329108b", "variants": {"crop": [128, 128], "optimized": [640, 1280], "thumbnail": [128, 256]}, "original_unchanged": true, "metadata_removed": true}]} diff --git a/evidence/lab5/pipeline-faults.jsonl b/evidence/lab5/pipeline-faults.jsonl new file mode 100644 index 00000000..751e0a08 --- /dev/null +++ b/evidence/lab5/pipeline-faults.jsonl @@ -0,0 +1,64 @@ +{"run": "pipeline-verified", "event": "pipeline_state", "t_s": 4.372676, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "job_id": "19d77db0-0b79-4597-a1aa-75331cfa0cd8", "jobs": 1, "outbox": 1, "staging_exists": true, "variant_objects": [], "failed_registered": false, "attempts": [], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 0, "result_count": 0, "original_exists": false, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline-verified", "event": "pipeline_state", "t_s": 8.012036, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "job_id": "19d77db0-0b79-4597-a1aa-75331cfa0cd8", "jobs": 1, "outbox": 1, "staging_exists": true, "variant_objects": [], "failed_registered": false, "attempts": [], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 0, "result_count": 0, "original_exists": false, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline-verified", "event": "restart-apis", "t_s": 10.380454, "phase": "durable-broker-worker-stop-and-redelivery", "evidence": {"started_before": {"98b96c5ce7f444e511e24dc0beb5e47ac7d8b99cccf3a21f523dc086705cd158": "2026-11-15T18:11:14.407616171Z", "93e9bf9769bc45e7b2de2939335b31bf91fb38bb9a270c4268ded3bbe27d0861": "2026-11-15T18:11:19.325341674Z"}, "started_after": {"98b96c5ce7f444e511e24dc0beb5e47ac7d8b99cccf3a21f523dc086705cd158": "2026-11-15T18:18:30.280046304Z", "93e9bf9769bc45e7b2de2939335b31bf91fb38bb9a270c4268ded3bbe27d0861": "2026-11-15T18:18:31.126341596Z"}}} +{"run": "pipeline-verified", "event": "broker-restart", "t_s": 22.193264, "phase": "durable-broker-worker-stop-and-redelivery", "evidence": {"broker_before": {"name": "image.jobs", "durable": true, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}, "broker_after": {"name": "image.jobs", "durable": true, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}}} +{"run": "pipeline-verified", "event": "attachment_state", "t_s": 23.346459, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "status": "ready", "seconds": 0.5566292910079937} +{"run": "pipeline-verified", "event": "pipeline_state", "t_s": 25.236213, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "job_id": "19d77db0-0b79-4597-a1aa-75331cfa0cd8", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline-verified", "event": "attachment_state", "t_s": 29.29045, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "status": "ready", "seconds": 0.006781375006539747} +{"run": "pipeline-verified", "event": "pipeline_state", "t_s": 31.997314, "phase": "durable-broker-worker-stop-and-redelivery", "attachment": "f4d0ec9c-b2f7-46ef-aedf-bf8663e7b1be", "job_id": "19d77db0-0b79-4597-a1aa-75331cfa0cd8", "jobs": 1, "outbox": 2, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 2, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline-verified", "event": "result", "t_s": 32.922155, "phase": "durable-broker-worker-stop-and-redelivery", "status": "PASS", "note": ""} +{"run": "pipeline-verified", "event": "fault_reached", "t_s": 37.994659, "phase": "crash-COMMIT-before-publish", "evidence": {"point": "commit-before-publish", "attachment_id": "df24f2a0-6db9-4f78-9208-ec5eb8b79480", "job_id": "05677c5f-2e2c-448a-a959-7b483b4e3d85", "container": "57f78f4109d706a21c3ee56c12080c8b44440722bc38912692043dc126ebd743", "signal": "SIGKILL", "status": "queued", "generation": 0, "attempts": [], "outbox": 1, "published": 0, "result_count": null, "kill_elapsed_s": 0.2771513330080779}} +{"run": "pipeline-verified", "event": "attachment_state", "t_s": 38.271831, "phase": "crash-COMMIT-before-publish", "attachment": "df24f2a0-6db9-4f78-9208-ec5eb8b79480", "status": "ready", "seconds": 0.27709124999819323} +{"run": "pipeline-verified", "event": "pipeline_state", "t_s": 41.528209, "phase": "crash-COMMIT-before-publish", "attachment": "df24f2a0-6db9-4f78-9208-ec5eb8b79480", "job_id": "05677c5f-2e2c-448a-a959-7b483b4e3d85", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline-verified", "event": "drain", "t_s": 43.080636, "phase": "crash-COMMIT-before-publish", "evidence": {"name": "image.jobs", "durable": true, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 1}} +{"run": "pipeline-verified", "event": "result", "t_s": 43.958164, "phase": "crash-COMMIT-before-publish", "status": "PASS", "note": ""} +{"run": "pipeline-verified", "event": "result", "t_s": 45.899941, "phase": "single-migration-start", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "fault_reached", "t_s": 142.343631, "phase": "crash-relay-after-confirm-before-mark", "evidence": {"point": "publish-before-mark", "attachment_id": "3a50ae05-e281-4ab5-91a4-4da3e4503b25", "job_id": "a7c0bdf7-b813-4458-a2f1-ec7deee48fb4", "container": "57f78f4109d706a21c3ee56c12080c8b44440722bc38912692043dc126ebd743", "signal": "SIGKILL", "status": "queued", "generation": 0, "attempts": [], "outbox": 1, "published": 0, "result_count": null, "kill_elapsed_s": 0.4695620830025291}} +{"run": "pipeline", "event": "attachment_state", "t_s": 143.814662, "phase": "crash-relay-after-confirm-before-mark", "attachment": "3a50ae05-e281-4ab5-91a4-4da3e4503b25", "status": "ready", "seconds": 0.5884555420052493} +{"run": "pipeline", "event": "pipeline_state", "t_s": 147.804815, "phase": "crash-relay-after-confirm-before-mark", "attachment": "3a50ae05-e281-4ab5-91a4-4da3e4503b25", "job_id": "a7c0bdf7-b813-4458-a2f1-ec7deee48fb4", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "drain", "t_s": 154.443119, "phase": "crash-relay-after-confirm-before-mark", "evidence": {"name": "image.jobs", "durable": true, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 1}} +{"run": "pipeline", "event": "result", "t_s": 157.460635, "phase": "crash-relay-after-confirm-before-mark", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "fault_reached", "t_s": 169.008007, "phase": "crash-object-before-ack-lease", "evidence": {"point": "object-before-ack", "attachment_id": "1265ca47-355e-446a-83e6-6785d69769ca", "job_id": "52417b87-c9a4-45b8-8919-58df3d7cfc53", "container": "78d995bc352b12d71068d5ad1763a00663a03399c3713b0c3198e708463d7cd8", "signal": "SIGKILL", "status": "processing", "generation": 1, "attempts": [{"attempt": 1, "kind": "processing", "delay_seconds": null, "error_code": null}], "outbox": 1, "published": 1, "result_count": 4, "kill_elapsed_s": 0.3759508750081295}} +{"run": "pipeline", "event": "attachment_state", "t_s": 180.985656, "phase": "crash-object-before-ack-lease", "attachment": "1265ca47-355e-446a-83e6-6785d69769ca", "status": "ready", "seconds": 11.97748087499349} +{"run": "pipeline", "event": "pipeline_state", "t_s": 184.385681, "phase": "crash-object-before-ack-lease", "attachment": "1265ca47-355e-446a-83e6-6785d69769ca", "job_id": "52417b87-c9a4-45b8-8919-58df3d7cfc53", "jobs": 1, "outbox": 2, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "lease_expired", "delay_seconds": null, "error_code": "lease_expired"}, {"attempt": 2, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 2, "generation": 2, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "drain", "t_s": 189.315926, "phase": "crash-object-before-ack-lease", "evidence": {"name": "image.jobs", "durable": true, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 1}} +{"run": "pipeline", "event": "result", "t_s": 190.309213, "phase": "crash-object-before-ack-lease", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "fault_reached", "t_s": 194.170989, "phase": "crash-ready-before-ack", "evidence": {"point": "ready-before-ack", "attachment_id": "f494dfcc-7346-4cb7-9905-6bee04e82d91", "job_id": "225feb85-a9d4-4970-86e0-891c595417b6", "container": "78d995bc352b12d71068d5ad1763a00663a03399c3713b0c3198e708463d7cd8", "signal": "SIGKILL", "status": "ready", "generation": 1, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "outbox": 1, "published": 1, "result_count": null, "kill_elapsed_s": 0.257462082998245}} +{"run": "pipeline", "event": "attachment_state", "t_s": 194.225224, "phase": "crash-ready-before-ack", "attachment": "f494dfcc-7346-4cb7-9905-6bee04e82d91", "status": "ready", "seconds": 0.05395666600088589} +{"run": "pipeline", "event": "pipeline_state", "t_s": 199.530009, "phase": "crash-ready-before-ack", "attachment": "f494dfcc-7346-4cb7-9905-6bee04e82d91", "job_id": "225feb85-a9d4-4970-86e0-891c595417b6", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "drain", "t_s": 201.403392, "phase": "crash-ready-before-ack", "evidence": {"name": "image.jobs", "durable": true, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 1}} +{"run": "pipeline", "event": "result", "t_s": 202.729892, "phase": "crash-ready-before-ack", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "worker-limits", "t_s": 205.694563, "phase": "bounded-processing-timeout-and-prefetch", "evidence": {"worker_memory_bytes": 1610612736, "concurrency": 1, "prefetch": 1, "lease_seconds": 18, "job_timeout_seconds": 14, "retry_base_seconds": 2, "image_limits": {"Memory": "256MiB", "Map": "256MiB", "Disk": "512MiB", "Thread": "1", "Time": "45 seconds"}, "consumers": [{"prefetch": 1, "ack_required": true, "active": true}]}} +{"run": "pipeline", "event": "attachment_state", "t_s": 222.229049, "phase": "bounded-processing-timeout-and-prefetch", "attachment": "902c9560-0551-4daf-b54c-c1ac3a7de21b", "status": "ready", "seconds": 16.136552875002963} +{"run": "pipeline", "event": "pipeline_state", "t_s": 226.208075, "phase": "bounded-processing-timeout-and-prefetch", "attachment": "902c9560-0551-4daf-b54c-c1ac3a7de21b", "job_id": "2f11304e-ea5c-46e3-926e-e8bc3d63fac3", "jobs": 1, "outbox": 2, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.765518, "error_code": "processing_timeout"}, {"attempt": 2, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 2, "generation": 2, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": [1773]} +{"run": "pipeline", "event": "result", "t_s": 227.389709, "phase": "bounded-processing-timeout-and-prefetch", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "attachment_state", "t_s": 229.395054, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "status": "ready", "seconds": 0.27098874999501277} +{"run": "pipeline", "event": "pipeline_state", "t_s": 233.627999, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "job_id": "ef05368b-aff6-4c34-bd87-3cf53bff52db", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "attachment_state", "t_s": 233.708161, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "status": "ready", "seconds": 0.08003708299656864} +{"run": "pipeline", "event": "pipeline_state", "t_s": 239.325047, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "job_id": "ef05368b-aff6-4c34-bd87-3cf53bff52db", "jobs": 1, "outbox": 1, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "invalid-job", "t_s": 245.076344, "phase": "mismatched-broker-envelope-DLQ", "evidence": {"dead_letters_before": 0, "dead_letters_after": 3}} +{"run": "pipeline", "event": "attachment_state", "t_s": 245.087, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "status": "ready", "seconds": 0.010581125010503456} +{"run": "pipeline", "event": "pipeline_state", "t_s": 249.616821, "phase": "mismatched-broker-envelope-DLQ", "attachment": "300be646-6ff7-42ce-a339-938cc6b0a04b", "job_id": "ef05368b-aff6-4c34-bd87-3cf53bff52db", "jobs": 1, "outbox": 4, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 4, "generation": 1, "result_count": 4, "original_exists": true, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "result", "t_s": 252.266017, "phase": "mismatched-broker-envelope-DLQ", "status": "PASS", "note": ""} +{"run": "pipeline", "event": "attachment_state", "t_s": 272.908274, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "ac55549d-9641-4928-bfc5-fc489649be9f", "status": "failed", "seconds": 17.926805250011967} +{"run": "pipeline", "event": "pipeline_state", "t_s": 275.288526, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "ac55549d-9641-4928-bfc5-fc489649be9f", "job_id": "3cc4c841-a7b6-455c-a698-46898ca96023", "jobs": 1, "outbox": 3, "staging_exists": true, "variant_objects": [], "failed_registered": true, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.8016, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 1.250712, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 3, "generation": 3, "result_count": 0, "original_exists": false, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": [1806, 1252]} +{"run": "pipeline", "event": "replay", "t_s": 276.677482, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": true, "request_id": "c814bdc8-10b6-4e7c-a5df-e02c3689233a", "job_id": "3cc4c841-a7b6-455c-a698-46898ca96023"}} +{"run": "pipeline", "event": "attachment_state", "t_s": 276.948792, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "ac55549d-9641-4928-bfc5-fc489649be9f", "status": "ready", "seconds": 0.27123812500212807} +{"run": "pipeline", "event": "pipeline_state", "t_s": 280.188617, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "ac55549d-9641-4928-bfc5-fc489649be9f", "job_id": "3cc4c841-a7b6-455c-a698-46898ca96023", "jobs": 1, "outbox": 4, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.8016, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 1.250712, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}, {"attempt": 4, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 4, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 4, "generation": 4, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": [1806, 1252]} +{"run": "pipeline", "event": "replay", "t_s": 281.19427, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": false, "request_id": "e76941b0-52f4-4491-aed4-07f9262518ed", "job_id": "3cc4c841-a7b6-455c-a698-46898ca96023"}} +{"run": "pipeline", "event": "pipeline_state", "t_s": 284.774071, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "ac55549d-9641-4928-bfc5-fc489649be9f", "job_id": "3cc4c841-a7b6-455c-a698-46898ca96023", "jobs": 1, "outbox": 4, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.8016, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 1.250712, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}, {"attempt": 4, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 4, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 4, "generation": 4, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": [1806, 1252]} +{"run": "pipeline", "event": "attachment_state", "t_s": 304.056015, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "78ad94a2-3f17-4a26-8aff-80850ce69cb1", "status": "failed", "seconds": 18.656614292005543} +{"run": "pipeline", "event": "pipeline_state", "t_s": 309.228063, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "78ad94a2-3f17-4a26-8aff-80850ce69cb1", "job_id": "5d86f479-1974-4c78-9fca-87c2c2b28c10", "jobs": 1, "outbox": 3, "staging_exists": true, "variant_objects": [], "failed_registered": true, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.700484, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 2.625588, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 3, "generation": 3, "result_count": 0, "original_exists": false, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": [1705, 2627]} +{"run": "pipeline", "event": "replay", "t_s": 311.389768, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": true, "request_id": "a2a8b35a-cd21-4506-b5de-94d17cb253ec", "job_id": "5d86f479-1974-4c78-9fca-87c2c2b28c10"}} +{"run": "pipeline", "event": "attachment_state", "t_s": 311.653564, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "78ad94a2-3f17-4a26-8aff-80850ce69cb1", "status": "ready", "seconds": 0.2637419590028003} +{"run": "pipeline", "event": "pipeline_state", "t_s": 314.848283, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "78ad94a2-3f17-4a26-8aff-80850ce69cb1", "job_id": "5d86f479-1974-4c78-9fca-87c2c2b28c10", "jobs": 1, "outbox": 4, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.700484, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 2.625588, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}, {"attempt": 4, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 4, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 4, "generation": 4, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": [1705, 2627]} +{"run": "pipeline", "event": "replay", "t_s": 315.459565, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": false, "request_id": "dfbc8483-47ad-4374-a7ec-0f2e47881594", "job_id": "5d86f479-1974-4c78-9fca-87c2c2b28c10"}} +{"run": "pipeline", "event": "pipeline_state", "t_s": 317.862922, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "78ad94a2-3f17-4a26-8aff-80850ce69cb1", "job_id": "5d86f479-1974-4c78-9fca-87c2c2b28c10", "jobs": 1, "outbox": 4, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "temporary", "delay_seconds": 1.700484, "error_code": "s3_unavailable"}, {"attempt": 2, "kind": "temporary", "delay_seconds": 2.625588, "error_code": "s3_unavailable"}, {"attempt": 3, "kind": "temporary", "delay_seconds": null, "error_code": "retry_exhausted"}, {"attempt": 4, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 4, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 4, "generation": 4, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": [1705, 2627]} +{"run": "pipeline", "event": "attachment_state", "t_s": 321.023425, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "0531a347-d57a-4dd7-aec8-7ab7f3311213", "status": "failed", "seconds": 0.46231995899870526} +{"run": "pipeline", "event": "pipeline_state", "t_s": 329.469513, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "0531a347-d57a-4dd7-aec8-7ab7f3311213", "job_id": "7fe94350-23b9-4722-a9ba-c1b28e89f6b0", "jobs": 1, "outbox": 1, "staging_exists": true, "variant_objects": [], "failed_registered": true, "attempts": [{"attempt": 1, "kind": "permanent", "delay_seconds": null, "error_code": "invalid_image"}], "max_attempts": 3, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 1, "generation": 1, "result_count": 0, "original_exists": false, "audit_replays": 0, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "replay", "t_s": 332.977678, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": true, "request_id": "8c8724a8-4ef9-4daf-a947-b9469609f7f9", "job_id": "7fe94350-23b9-4722-a9ba-c1b28e89f6b0"}} +{"run": "pipeline", "event": "attachment_state", "t_s": 333.242295, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "0531a347-d57a-4dd7-aec8-7ab7f3311213", "status": "ready", "seconds": 0.2645675000094343} +{"run": "pipeline", "event": "pipeline_state", "t_s": 337.745274, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "0531a347-d57a-4dd7-aec8-7ab7f3311213", "job_id": "7fe94350-23b9-4722-a9ba-c1b28e89f6b0", "jobs": 1, "outbox": 2, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "permanent", "delay_seconds": null, "error_code": "invalid_image"}, {"attempt": 2, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 2, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 2, "generation": 2, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "replay", "t_s": 338.728727, "phase": "backoff-jitter-permanent-failed-replay", "evidence": {"accepted": false, "request_id": "872a20d3-0396-4a76-9afa-44987d21135f", "job_id": "7fe94350-23b9-4722-a9ba-c1b28e89f6b0"}} +{"run": "pipeline", "event": "pipeline_state", "t_s": 340.942477, "phase": "backoff-jitter-permanent-failed-replay", "attachment": "0531a347-d57a-4dd7-aec8-7ab7f3311213", "job_id": "7fe94350-23b9-4722-a9ba-c1b28e89f6b0", "jobs": 1, "outbox": 2, "staging_exists": false, "variant_objects": ["crop", "optimized", "thumbnail"], "failed_registered": false, "attempts": [{"attempt": 1, "kind": "permanent", "delay_seconds": null, "error_code": "invalid_image"}, {"attempt": 2, "kind": "success", "delay_seconds": null, "error_code": null}], "max_attempts": 2, "lease_seconds": 18, "prefetch": 1, "concurrency": 1, "job_timeout_seconds": 14, "worker_memory_bytes": 1610612736, "retry_base_seconds": 2, "published": 2, "generation": 2, "result_count": 4, "original_exists": true, "audit_replays": 1, "envelope_valid": true, "scheduled_delays_ms": []} +{"run": "pipeline", "event": "result", "t_s": 342.330574, "phase": "backoff-jitter-permanent-failed-replay", "status": "PASS", "note": ""} diff --git a/evidence/lab5/readiness.jsonl b/evidence/lab5/readiness.jsonl new file mode 100644 index 00000000..be2dc9b2 --- /dev/null +++ b/evidence/lab5/readiness.jsonl @@ -0,0 +1,12 @@ +{"event": "docker_environment", "t_s": 0.509421, "phase": "LB-excludes-unready-API", "version": "28.4.0", "os": "linux", "architecture": "aarch64", "logical_cpus": 8, "memory_bytes": 4109803520} +{"event": "routing", "t_s": 5.547801, "phase": "LB-excludes-unready-API", "wanted": ["33e9426ff450", "966a4329e0d5"], "observed": ["33e9426ff450", "966a4329e0d5"], "excluded": null, "attempts": 2} +{"event": "direct_health", "t_s": 7.986095, "phase": "LB-excludes-unready-API", "endpoint": "live", "status": 200, "instance": "966a4329e0d5"} +{"event": "direct_health", "t_s": 11.310146, "phase": "LB-excludes-unready-API", "endpoint": "ready", "status": 200, "instance": "966a4329e0d5"} +{"event": "direct_health", "t_s": 17.483132, "phase": "LB-excludes-unready-API", "endpoint": "ready", "status": 503, "instance": "966a4329e0d5"} +{"event": "direct_health", "t_s": 22.457978, "phase": "LB-excludes-unready-API", "endpoint": "live", "status": 200, "instance": "966a4329e0d5"} +{"event": "routing", "t_s": 25.540731, "phase": "LB-excludes-unready-API", "wanted": ["33e9426ff450"], "observed": ["33e9426ff450"], "excluded": "966a4329e0d5", "attempts": 29} +{"event": "direct_health", "t_s": 34.334113, "phase": "LB-excludes-unready-API", "endpoint": "live", "status": 200, "instance": "966a4329e0d5"} +{"event": "direct_health", "t_s": 39.591823, "phase": "LB-excludes-unready-API", "endpoint": "ready", "status": 503, "instance": "966a4329e0d5"} +{"event": "direct_health", "t_s": 44.769431, "phase": "LB-excludes-unready-API", "endpoint": "ready", "status": 200, "instance": "966a4329e0d5"} +{"event": "routing", "t_s": 44.931393, "phase": "LB-excludes-unready-API", "wanted": ["33e9426ff450", "966a4329e0d5"], "observed": ["33e9426ff450", "966a4329e0d5"], "excluded": null, "attempts": 2} +{"event": "result", "t_s": 44.931548, "phase": "LB-excludes-unready-API", "status": "PASS", "note": ""} diff --git a/evidence/lab5/track-2/concurrent-upgrade.jsonl b/evidence/lab5/track-2/concurrent-upgrade.jsonl new file mode 100644 index 00000000..a518205a --- /dev/null +++ b/evidence/lab5/track-2/concurrent-upgrade.jsonl @@ -0,0 +1 @@ +{"scenario": "concurrent-upgrade-after-staging-cleanup", "status": "PASS", "attachment_id": "d75fde0c-bac1-43a8-ba01-ec6e54de182d", "upgrades": [{"attachment_id": "d75fde0c-bac1-43a8-ba01-ec6e54de182d", "job_id": "3c54e884-c76b-4458-8406-8321bf0c61db", "pipeline_version": 2, "queued": true}, {"attachment_id": "d75fde0c-bac1-43a8-ba01-ec6e54de182d", "job_id": "3c54e884-c76b-4458-8406-8321bf0c61db", "pipeline_version": 2, "queued": false}], "variant_sha256": {"crop": {"v1": "507e48dc62a637e590ee652440576280e0d7e1de4bc3287010dc8ce104dc1697", "v2": "708fde25e98b0564fdc6d3d8da1ffd46c625becc1f09aaae7b5785ba928f37b5"}, "optimized": {"v1": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "v2": "0dbde6e7be396cb48efb5ad3ab4df33836d7b1c5def672d3474c04223823e48c"}, "thumbnail": {"v1": "d7cc662e82b52806da6245e82ad3c0c699ef7b4c6107cb2ee451e05930d3eabb", "v2": "cb02ee0bf2718ebd074dfb9bcb7e2779482c034d0889412b5154265e1d85fdc5"}}, "sql": ["1|1|results/d75fde0c-bac1-43a8-ba01-ec6e54de182d/v1/1/original|1", "2|1|results/d75fde0c-bac1-43a8-ba01-ec6e54de182d/v2/1/original|1"], "original_unchanged": true} diff --git a/evidence/lab5/track-2/migration-live.json b/evidence/lab5/track-2/migration-live.json new file mode 100644 index 00000000..cdd99f1f --- /dev/null +++ b/evidence/lab5/track-2/migration-live.json @@ -0,0 +1,16 @@ +{ + "date": "2026-11-16", + "project": "highload-lab5", + "schema_before": 7, + "schema_after": 8, + "migrations_applied": 1, + "ready_attachments_before": 6, + "ready_original_references_after": 6, + "public_smoke": "PASS", + "public_persistence": "PASS", + "public_failover": "PASS", + "failover_successful_writes": 110, + "failover_failed_attempts": 1, + "failover_first_recovery_seconds": 2.599, + "readiness": "ok" +} diff --git a/evidence/lab5/track-2/regression.jsonl.gz b/evidence/lab5/track-2/regression.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..8c671be24a01a7e36eed27bcd7b0a37924b8de36 GIT binary patch literal 4194 zcmV-o5S{NIiwFP!00002|Lt7caw9jAec!K8=y^fmj)dMOc5aT?n018xv<`;?*CuBu zl0z2T?)CWJ=b%V+i)wWhOOn^}jMVKmSu7S32M3uafrNg#Ebq&zyS%=*EUWj6?Rs_F z-u~ysWp}f~RX*dKOKosrGY`AcZrqh_ziGF`+RplwzX$l%ZuBx-nTF58OQq?_RH?(hs7$d zf51i6z7V#zu;Kbbwnjd5^YyB=p=LtQY-O}^--j)p)7~%F?-nU6 zZ_;MpKB1rgSmpaLIN_{uCon8l@0Nw#F5YcJw^*;xr?Bmz3#q;D+gm4e z>>|7m3v_$2Ty!7W4P_ZNyONKOw%%(1rWEJKaNqh@eq3}nX`Nd`{KT}}Kk?=Zg9R&{ z_vcyp9*7^!lr#g>pI}-pR{J0Oo@~>6(P0Gl+x}^xT{r;x5I|hIEw}6Khnx7JEA5gN z+!I$u@-c8d7vA`(;QrV8cC+4yWIirdyA4M3ahtd_!bpP&&y6DzZn8~Y zYSJ%klhQ_TExC|N2qPwso)p?h`Hb;f#z%%v3X&dwIOeMnZn{+pga72=d(0#*$dmF0 z^WS+QF9uEO)|)kE@P`LBIj7FF34?Wg@!e0Ci`;HBWR$Twq-3SHA}7q}j1hAbWZ^Pc z9VULLP)JK!05gd~@i3BiQ< zR?xK)*9y@Eqb{Xa|J!b_i`8N`|9C%UjbQ!XZ#lp2|Ms_?j6w6T>s!34VYw@Kw+gHE zoeP!x?gt;>+Rfkh>n*4}T*cCpUK3kPS5H%3`>*NdLx>uuK__0uH+3cu=7i;=7S za`|%$*NC9#YX5nAzbah|yhFkHCYL_?_OW~)m+0_KST656fT!FsKybrds3N&&fw||^ zm*i!lQnl1FVBjm9olr^X$X&7`28Rz4d~;0BoiO13!(I~uQw6~{!!Jt&eV?;bNhpfJCtI_$MdudVd{w*(^+YDB)wX zv}cF@kA@vSTyHO+BzBK!H%VUg!kZiGot>&* z0Gv}+^)hjifmUnPE3KDJ4nQt~WAz6L<(b^)fo17gq&qObr zv49=ik!!+&NNZyTjr!Qo?txwA+)0_%t1c{7W!pQ$L%ee$`zW2Yig>ERm}+$?vMN&$ zdqPQMRYC0WRz(kI$c2q5`C7d$LZ);Q@9sbBv3SS9i{uiz>w|U1SRP1-g8UXrCt|8) z?`6!%6ZCdWB*|4tDd)lq^mjOkcMYkM&k;UiMug;9i${geV?s$N%h9+Lzhd0`#Y8XC zpUrYnnib#C_n+2MS}WgO@_%&y=;NTcdm-;$sJj>X?uEH~VeekJyBE~IIE{DPvO8LK zpT6eI{I-pJkMiZ$zx(E!c4f6bP~q~VmA0UF(uU_4$@^{WH$8G~sfTSAKb9rCEqM{x za&fyjc-q_2&HHh>{L>%4?V$|ayuH1em$2;SSIfowa$=P1#~c&B3#(Kf%LKRO@B6ar zZjMQ3N+K2=nKY_0D9g%%4UvVQi}A@smHGzOZM3`XzI^UKxW%77XNH!8qLXF>2Zh}F zg>l%HkUxw7@h~Ae1wzOmq!gYNrwr4W?2LToELx;l$c3~DVhjky6?p*%PtF>v(g zxF-2j9KQaaZ%%=LZ@+pkuJ*gO27C&Hn8XM;J#M6MOmPDsxc4m7x(3Nz$i6 zqz3U&B(%`4j~7uwZE>lXuUa93B>2p#1v5FRfYc+(Ffu6 zQhH{Ch2PZ8Z<<1wBs>}!fP+&2!3OAlj;e(!U50SJ73(K*|g)B!-+L zGGE~QimpOL2xpV(Rp`2q=50Ii6ge%_BnE|4i{@x2)u@?P_{SF$7zCt+s*pZX9rG+i zQui4E0u*^W0-Q$<{+j@9m~xd<%+^!{rf5MqkPi_mGo^BjXtnAb#waH}5}>xFh>%Ur z4ErL7(5~gHM9xeLW|=AyfW)dUnQ4fB7H}t&4#Ja2N+n@Nk<-Sz=Cm+rjj%L=oKn{O zCXgHEoO1=d;+#SK2e^CU%tyzW^EJ3=FlUn=Lrw^1dpC#dK`HIrV7l6arzd8TPs>_c zQ(C!`w!-eiDzWvx+w76(hwUP)UM^W3W~ndmfD{EQ)gf7hQ~*J%WKv}eky9>Z8pE0c z<2JoL#Hb{$^+@-fIxSpydiT%CQHQw(yOodVzp`Mr&cd)Lv`3uC3Ua}w7--zTDX4JQ za#SrLx`z8t@Pelm5@+=PIa+fhGLlZFqVL}22(9Vh;B%G?LL&+CrY1Y8H7!82hYAfY zhn*Ax06z_k&*=RWg*i-vU=sH#6jri@%Yox%o}dq`30dWWuW>tuf&%?jUoB<1FvBE3 zhK_{o)sP^DLM|xDRMnC}BS2UxQCT4sD3YgI{Al+HZmsPNAjDfe!~!+r#!2U29Sfp} zNpRs9ydGeX8M3E3!%WFY#EQ+y*U`BES17sX(CYA>W)KExw0K3XKt&gzZIM}sgjkS_ zZ<&Q*xp{W3s|tq_g#nT5t<|C}$av!s3hZF_jdJod?R*t1s5bZ{Yhkr|XdMhD^HxB| zRjvY(2o6C!vMNI@B=EM-*AlJaxnQ(5r+b{wK;fJmbC?9K6d|OA@(JFzwYkay$83OhbSq(@~dl4qzX+jcJ{2sJzjtRO^7L9Jjge~0$WW#t#?WBCBe!819 zpD?&eCaFxKFhddjs3x;e5eGB3;0r{ebM;wLlcLQ^V_qd5Ft2h7;27vtc!GmflwBQ_ zTFBH}iK+*`9F&$Eb}&RrDE%s-8s=2dk)(}|rD6ob6oMzV+AwLT6~kJsV;Y0X$W}kl zw(8rq!Qc*(o+)Gj-2Vni6>TX&MpaGY2$giEYTG0qnEgd1zZ??fzDSkTp zK6U2ius?kYnwRH=?@y2E*ZrrWa``FIDQUgR#{zxG^W!2fq$laY5!M0$nSPxSzH0Re z$0Gqx-&U+`8srbaMdG#P%yuAchcb7%(fm_OBYSkw`8L zSnVRSg2Uf(he?uxVb#XsTyC2!V@r2h3VqmqwfK3k>v%-q6f>#lyxhK>is&o8=lk2Z z3X5fb;AW9~;ACGgF)xdE^VXEGt>+)lZ`eL_I#izY#h4C^v9geirisoRFvbRNk-H_H zV121BLdpp4Qk11=BcN-&;#su{+{B#;o~kmuS_B9PI>O(H_tt~d8NAS9h|pnrSE0cb`#;Q$~jnlb5+YNI7Zq`lK=aq?8fc6f#@K;XG?e z@C7=sYJ!v|$2Mn1IMB`*94Zht$+NR~%rt1*rd08i)rA|L#5D`Ou<3i8vurttYW zV#8BL@<`PN#kfj^Y1I~kvT)285Hh(?Ja^+|Ty!|$#}SLIqQOmwhFPoQI196LHy`>! z!F03TuRb;8nixEFXSd;Rg<(b=t4R+nn>9jAv}&wi+esv}t)SMoJp(=2 zCe;>MU)Clh2L6ypVf6IW69I09Y3_`C`sh7+m{YXfOQV5QU&kXR8|AE*PX+1=XE|HF zL`$HY1F77^vz*N&$L)C*y`AKIJID8SlJkuLZznn5PICT=CpjMz(5jW#D|uRM^~Ff` zKu$mW_&jNS|67mBY(0aY1IJCj;;8JZIz{-EgX+0v6N z5|Y0DSRVAz1Q7#^Y}?j(Nu38VUS{V^o^Ko5JZ_* zlMDidy7jhPgxZzuMOQ+aN)~Z?W&ANDN z&M}^`2&15Y{(C@vu6s0{@uU(d-g^6jl$GIO`9SD!n(3DGk5=92>vYooDO;*#ei z>fB#kKKMWL?0oNx+}-W!pO)^P9j=^RF~TD%`N!+LyiQg9oU1?ZdCotNg!#RT^&WHJ z2*!P$d_8aV>HFBE`HzLE^~To;?Bn}+i#)^&*6r!~C7Z9uIA#&K`?$Y*dePLv$_un7 zcm167ozZSf`1$$2e~8W8?X4>4A9G&l|7;1`OZ{dP^cg4I(q#dINjTDF@#%b9-qZKF zzC`T)J909%yL{&7x*#~U=G4=>-n-=ASFN#?^(oQYKgqbOuk%BV`Skj}bhc}Ed)oue zEQV;*bWf*p{?p#>r_GqfC$s0>#j$gLgZKi=IB9~>cRpk8gkJ&=96DW=JMXr zH*ojeUj8*R_G#!n;Y)&hh}P0Fd~GaCeN+5E%9u`)Wq{x@;Pn`kC(t*M$3WoC`WayW zY&#B?yYVx&8=Y|;&)N0%_KwQ&a=*3+k?i!Kr9~M}KkT~~C3Us0@U2%YBJHH%a%@nv zb=NX29*l^G-!;k!45m5t#I)Mb|N?){s@-_L{fePkr;^FxicXX<#!{#urv_?_)jk^x)W1u+nU{U-9+ zLj#kTi6-nIgn&P7@fxZj?ski8SwR2n>LOQNU%;>X?d@a6TzxIyGB;e5sFFn-`{$10 z%WTFTqjHSQhG*-CI4v$J4l{4hw~HX#YWZoe&-!cYY0qV&#};Y8`yKcP=l$y}Hc#L* zE=`}D_M*-o0PFMeBG{*+ca~K*F_-u2d&=!`Kn>yu(jGK*%FO6xs>-@!MeqmYx!S*{ zw(ysrpXblFe&JMsUc>Um>%-%iAH9}m+Ns~x-{MS4v7eIev7o)v*SkIjAour4U~zB! zM<#X*JftBnhdNpIXW-Go=orG-jdd&Cg$4*^r(woQGkvS*SFZir89fxp`O&g{HmF{w zet5`FIl2%hbqqF|p0-RI`yH?|UAxr{kV{l!&Y;&83e-&6mb3&JdwI+=;yY~x`ZMor zhgY*W$Dj#z?aLf`bc7hEr=eoe>iYwCO_{fx=F!yo7$kFzcm1xS@|fkCr^<`ZJ!MobPXYfRvgXs@1Q)@0WPX*sdiM zx1_wimly9J;-MX7(><4^t*a_ntcs=HofYfV_HoMwy$_v_F}Okb?$?tgZ;jry_C&O0 zE&t*&;*D!m?m z3k_?fltYS@sK~_MQH<=c5e4VO6(6u29O7TO;}*%uzJy%O>GwnBeVD&r;^DhJ2GJ&a zyNbZGEDUrS+}|QONdJ3 z*~$zN;fe%)<*w8(5;*)uqB-!_C`zV!lBf)?tGF>RSu!K$Zsc6Yj0%_#L);1N9x9V7 z&n;i6Es)lqsuIsws2CCrfa_#d=nFcg)#5-^XPSl^t5M2uZvIMW3rxGY0T za1CRL&kAZlm9aGh9>$3=)uiUN8HI-6CTA^?l>Nk4W?h}LFD3DQ7+FzId+34bK)Vyn z76P!6tunWJkxeJOK4=+c_&|b$@n;Jfnr!Q83@v|=E@RDh#y#Bt|xGx(0J->jjD?)(?#%I$_;rDCb zF^ImAIm|ozLz*@_8a0nj{C7}2i*JvkzO||;g(bo7q94!oniGVqpXZC$k1TlPEkiY7 zy<-F8F{ulFkD#_l)~KN+*QlN5+u_^LUZ&0!72eM(wFA@Pt}J*#{YE!T+eDYp<1%-A zyYJ^031}c4zs}ybfU7OXsCT!lnLY1!7k?@7rqFe?1xvVPzFltjv5NC$xdQ?%7WU^| z_GbdpVZG5pgZI?o`pU^1a%7M5*~ETV_cX9E`Bay$+o`YzW`%2q;l$pZ(+@c-WDtHu z1(j~7=2TGbI6~*(CJj*eYaj{r_8ve|1I?M-+Wb|gW`y^yCUUlgySW(Kp9nin>U00E ztJFLD*yR}oK^`*JK$dLVG?3$q8%tj#8f_8;hCZFODa5Z_z?Hgsn!5fGo^rsWy)qNZ zTI!VlHacX{Rjzo=WHZr=5&;v2?4XD%OuW%*hy$tkE&=H;yNABJkJuD!{QW=Mwb!i8 zLkyb-8V@;BA7g@lKc*D4zZ|~K%4}<2vT`?kIy*rcYE5lPCny4 zHg8WeqR{nx*94E-UM=5M4|c!a^w$sY%`(q#(BzS(-mU8QaP<0ai@FgbeaI0d)&l#7 zMhoyZ7z_!Ex`&68IAnv(;k58kt)1YqcAXk|zi{U#2SKIhBx>{x%XIga+41n`tT{OL zMitq&%5`iUB8j(CT87@>yxMHcDj=#-$ri5nF({l^RgCXuqRKYy^r}jnW>;zIyhnng zl^t@MDnq-&4A&NQJmjot4^)Y9xre?3KU|Ev_=t3%rhFx-@6uUT_L{HcTgWvd>sqL^ z(cOigJF(R(UXcMM?OHR;tnEanZdYWU8Sct2bn_BxTcWv zw+S5A>hPE|26^FUc&8nVlXK_ya||Ko)8}+}M4@Imi15cfK$Q#7QluXy#|GgK3tB79 zOM;!DAJ8(Xz-5TnJ7~FH{VpRC=@LWn% zD{!2H4W!x5pV;@|p_9eCa|_x08ot|;AK-ovpmexC0sP#6`Hb0)PPbuUaE*IqkyOl=GG%{RjFOgCr2@8>pq0< zQ9NM4oKB%VBmcRt%X)ymr`O;7u(g&=6>EjJZ{8!Nclb%*47@2}^y93DkHR$PyCf4M z#3S_Z5`?ok@-?i{?(#S2dUEt!+4G?d_gv~+_R{(C++^a+;hC0Wq@0%Pv+QD?zObXZ z4l}&HxIC|3bAxxTGrVP=?g{O<3+cF1(PdmFcs0F#j0E!HLH;F(2CSG0y7tB(0jBpi zfX$+{By|RB27DYs->b&VVitRjMu0+}#V644ehB}zarI#i37Cx(#D%s7vfHu^&&m>B8NnS@At|oyM>V(gwvqp__l}(WgL7@msI;(S!95eVU)Z5k`gkYT&&)#b5%CY zq|;8!hXcJ*QJToOe}a^HgB@P# zD|lt4*wb-{wzF;2K2jXkMB)`=7?{?8Fi{J+KUU-HlK{9&n?#Vyc(Tw#tnZ+xE@w2< z6JfSbRHt;#(J5&7BJ9-gRX~S%i!n~U&j1QDcTM(_lu3KUq*z6c!%+K8n_B}|a+XTF zd=Fq#JJ&n{Dvd|L7UgktkE?)@bqt>Pud{-s#O=0R5V>rFa4p4eiI$30Z77OEACJsA zBQI1}6Ap?7@+*w`;AZ{rXtvW;L*4?`cD5KPGg9bp{^60#Q9ehU$28+1lCp3SJj_RK z4x4(iBpG0N+%5juqs$3i){Zc9r#N1;xoNhicuRRCCiI~b`!XcXe-|U%EpqX53pymO zdlp(a#pvXR>FbBc)%qD?HGHQ?;Y0hDBks!Y`84kHyRhdycy;ED!|H&hOr7t1seTX+Dm>_LO9M;#)b$tS^{rJy3L$&raK7UHA)s@!~(9+=|=^` zFVaenO=bQ%Lz~qR_P8``MoKY{1<^c13(HrP0ex6ag-i2LtH(*~gp)_%YRWzBAbyWO zS2aJPxBWGGyqFoI-*nH&gKK|8k=SH8Gg`Ot#*W)(b~HzWySMvuhM#W}0UwY#MZVGM z``1imC^_y2A@e6J!(RedngK8Cjnn$@X9D#HjRP6ZY%TSM{l1Uldu{k9^GzT1$NJD; z?z1l9=l^Z`8+u=wlGV7Xk@F5neB`&|NamgQU_bC``vqP9IXmZE0DI$`dO=Go6E#Ec ziE=m~4;tJhb1of+p?98?O9~>6hV0){+E5dUth4cCOENDwh@rshSi1J46e%3CbJFCF zyO>sv)#*aalu}JI)R1arX76@)FD;V?v?pbd<4(Z|WmMY!p|A|Kp#W@9-Wnt7;8}vl zkrh}k1M&9;ALI2DDj{elH3hy+gNHzOq#8uw zn2zI(CZfrUtKoC9aFb=fYM!Ip+c*fY4884JU^oOfRllpIIdQcGu0MO{M^u;KSyPWW zCp)#bJ5m%MuPMd0BpcJ8XvlESwj~F1=87!dTS{}+8jg4S1De|^R}8DQxwrc;Dde%Zq~U8QMoSDyn4^;iWO)_Iz?gw8^uzDc6C4GbB{S=EXC_-8RBdhVbC2f-osWrp1N?Zc2rs z^IJVLKKl#UNYrAAqrv6tCv&3~9lE)A(!)1r?t?L%;ae*{9jT}1Ng{;Yg|(L5-~V#? zmFv;T@~w67)}2CpMP~vLpMIFjp#YRNaCn zcaIjMB&8M9r_h~YaP7bg2i1h)^!4NU<^A|PzW;q#DznXh_tQxguR%fr z&0K`9u2WAb*$%LsEKLK(9Q!Ws{O+Z7XDz<<^Zb%aIcrLt;O%Naqs|aBeVDbaFL95$ zB~Y9AfQrhHRc#2ciHb*41-JgD7tpI6BHtT3lJt0%Uk|>_E;Qd-i6b zvk^r-m=^KGMhVD8vcR#;&Mm`wU%iBO+Tu8EKMMJ!y;RaN+DF(kN8p&U2Ox|qy z_mwbq7c%elweLdw*AB!-B+6!v*ek1cJM`)8_*yJ%6cfPs%dCplE$FXg3M|*S&xb^B za#Kd)#siXWHW$@*?biTnufHB%#wA489d8n-g~ zIbfyF$5A}Cur7+C3`4c{N6Iddw2JHG7l8I%;Kj}AI;E9R6EY{+)lzWW39`qRu8Z}} zYgo6WiLZPCtR^xOg6L?E&=VY-c`92*T514{$X^D+L}+{#8hDlvxGgMe8m2k<71F?e z)*=X3bNOwukOmvd1J+j6(Th#7#-6tkmDu&JcdBJMes?TtmW)>iu`@5jxwFp7Rg>e9 z1d^b}B0-&Chi}uLko-|!RwuXiVu8BJqr~@$-QlQV;Q~oqK~QyqktVZ(B{o?y^S&P6 zeR`V&HkNLBZP1oi&UsfdkFYQGC3uN*Q=&#*oY z2Hhk+ycYh3gRD<_Xf66l9R}^$WE_=xyeG1W>`le#r8YQ@IOsmL<~5|1p5o zh*cW*i?y3&cCj~!h0r!JE1i@}j`E5F^`by_)=bBw;CU=!i=LZh<13dzt+ZDjWsnpp zt*92c^n0P&Aey=0l3%ru-W>MK1@OGDv8mc=B{iAM=?Gicb+{QDvW>UT9)smzB2Mn~ z8?zwcHo-{2He#VAy5M<9(Y%9S0}=N3T$X#v;>v-{wu;Z_J1z4`eME73hfuDSzvHm` z8_AX&bI#XxOW40Kr%PK}9k$#j*nJ60((8 zUL})Pm4SIXe-NZ)2^NHY6CrYgj97gGx?V4Csx$WMYDq z4DY%i&{rV4BpW`-oe-;eLj@NjO)g}w*ag3O!FGVdqvYfj1;UM#q=l54C|%%7pcQtp zojBQ4E>{Z0rb1c$PRbgX3mgVeLj-ciYmQIc76aQll>aKn{FAOk#9u#KNS9HpW2;)C z)U_z@E{Qd?a$6n9N)!Zn(i1REHs{C*e@&5izuPA{j3%kP)6Hnb2Fqc=K^=+`014VA zj`@Za=)N>)9Y5e{NH9LPQw+d8qH6T~ea)nF-_luZz%iI}m#dnnq$9biBm2aE+?xd! zls+v5{TEBl#k)$yL6b-8S~n=z?`uZ6~DeKXu}BHA?~;xZscB|3;> z3)e+nygws$v>jy>MiaCHy{R{^Dle)I9vc$0&In8(Zq6SJcCmiwhW%dIFmpkuD}Q*56y@$ zN~kr^46eY1NW0^Zb|J#-=-PVO@t|SR|4Tb2qaVSH5}qVk#>Cr+tt6U0ycYMGOy+=d z>bP7Q=DGr!u!QMwVm`ULmJ9TEa#a9_(f0avHJ5=H_Jq|=P*#xz`^YpMf-@6N}w_=Zvy$AE`R$ zTo!p+=m2}RsIUqF8id%Y7@E=8RbMqX(F{eXB5Lk8%!%NZ9uveid|!8~2m}-&)<0|- z86M^l)%r@3tP8XK7IBp2F03wJV&I~kb@g!tu<)|?Rmtw+m!g*QIuP7i9K+ZxTIA&5PTEBk?4Ei zl^gx-##!`pe}6BCg)$gX1dq+Rr0yur-X6%{>q$5loy(Vto_TbVZ}nL;R>PFHsanenVW>Vo`!lT zFSzw_Go14sU-w<4zq6Cvo^Y{m`Ti83kcPR{WoIqF>xWL*R&XaqtBJKW`k|RY=hW{K zjEgl_@$0mC;O3)el9MlQ!pIij zaSGC>`?SuqCL||WAep{4Zolo*)PL?b|hU{^NLUiuJSD69!65jGkdoDe@gCTS%HKo5Pu1j2UW5H70B`G z8IXKyw!As0e5;349cs2spz)W7#xER->mm{ZZ=erB0bLzi=bBg1fna?FmIF87xZH;v zSdS-LGfY5_RFfJ~Z9y=*3xGxhvdtSNuH*eya)3M0twTqZ7=@})Qhd#jbwOc$sb-pm zxkJpmvuB&v-92)CZc90HYboob`25rPPcXrR<5W7F-3+TYvSrYi?RC1S)4)_N&W z1u14r_5q;OO~90Z`bAcD*ge&a2AZy{^Km2vm;dRB#R_`+kzWkOR=$_6n? z?x4yA?R^ImXnI?Tj&!O%u2F6^ozk+++$R;h{O-_jAodmyS$rE!43Em((AgyR>aC7x znH-NLZIW2@FL*vY>JKdC0GHJz+Mwwy40IdGGX_$lfSIi@^_jJNJoWU~3a=EeYPhA4 zrUOkT3{kL2ky^2Os*ko4OU?!j9-%q&e3COoNZNpRcureXw zY+(P-ULYfIe;=T43%h>83*tB9!CA5KFd#WF%0Dv#`*EBfw=WiiAAWmfN{OKTgmTfi zRMEp_68^<$MS`L7@r)*_<196QXevVN%Zn>w%#?yS zS$Nq8A!&o@UC9>a9AhEyj9eI@zmRj?Y*r03Nc9 zL*P;rm{9UdPm*AItCQyL*L}C(F%KotMKPCwR=H?AV{sR|k zkt^&5K81+w|8AM8>Nt6Pin}at>kDxo$2XQvfzvO*W*7>g7`Zt=H4JK-Kq`klxtJoe zJ9N$>egVQrK2#gV$|3?Q^zEuF(~ALqXdXVmbgK~g?#?Ho*hYLTU#VrDQkAXJz;Pr2 z>b>j@x@N!vAwFvvG2UR`_q~M^!4O{hRtQ3$OW0 z9cYD&`ioa|qIt8eKu~wThu*{4xN1X}v^C_x{oI!AXgZ-QJ-Dz<)>W1?KE4fvZw()e zTe;w9hinhZ7-$i)Bqb6)??((}FqF>L^+lqCTeTxGc=$b)d+SJ^DQHNP&a8D8Xi$gi8YL z5KXD#^d3d4>$`Bor#!kGLDfkMM}i>RFX8QA>EAQv+I79p#Oxcv=ncGvfr9w`-dL&K z7m6_;IWE*=;d9O$ENTG~)?!uuXduVOSNynN3?wQ`;eO2?JYy4a(=Ly2QC@Moa1Z#T zp3kcsNXJ27dp7hNXINIBG8yC(3~-^|T!RP8SQHmAxGoS!K%ksP563?h%N?}k zofQ*UozhalA8|9T*)hkOHD37*Ne7sK(M4FlrL2tQNq5aH=eblA_aY#Yl*h#lK2qNQ z#gIbL#$XEHixBx69oATTS}>)YcLfeSuQALmylg1Yr}uRW5fa541%5VfgRt1FQos`o z!f8PmW3)Ac6QNFmGAxjfqTQE+DgK*N4j1utn?6G2$SPyU3Z;x)sYkqWP4XG1!Viww zv?7;>8>aYYhC-4RY7VeX6(0w)s$BG|)-qr%5u%-+k3Oj;r}0vmv*=mQbZqHYR_Y*; zf*GC}#_h_h;C)?_JNj3gyUWzI6kXbpyUT^s%lg8NQGPbpS2q3*C|II_k|$=f9A$l7 z9t^0{Meg-*K=Q=cs(&1CN7-O$Q4ohqP(D09dUL^uP!{AL36q^s349wa1Tg0dvt7^u zlOxypecX(SVZ#h(@l|_+PAfur#2`Qz$pbWbe};j@K0+Xn^Qkha<8#GpR?LiJiQ zn_b%+lr%j zLe@fqxNBs!CgwirR3R~wtQ7HrU69WTS)Pg}Cy;eGm2F&__=qRf6iV(C5hob;BP>Yn zX+Z|I8DQXs%^64yZ~y>)k*hMeM}km4Ckztf`WlX|0Y2>$n;d54m9AK@0qh}^P{3}F z9iA-8<1C6@eJZ!0dd4H;nE16kG6^pOo~MndwNwcw_rS2D8=~Z1%c6J9lK?crDR9|g zM#ic6z5xN9eJ&K_qvFLQ99O!wA%r=j05riNAKYm$N;o=%Gc}x9c-DA4o}<$;_ceI1 zXVZ-(V%^tmYv0^j`#4pF?BJ~Z>kr~4CL(I4k+D!Pf2U#Ep-LWy_ z$WRE%x3LjSgix7cfAAFlP(eiDLoC zgcs`kW&*$@QN%=+dD4i?M5n8AFt5rv0BN5;wxpw|IK&l>N>B$ zKF?A>Q9lJIP`PAs6k=mo182W071Eqwp|+1XfmE*~LC{eNrAo(|-z^>VnMTCy6q6F2 z6S1Mmg5;wdO`(B0K8pk@d~JGg()haRZUpWbz*>-{)-jRI$y#0?_JMjf&s0DXUS{@5 z?EaXTYh18biI!QS`gnn-D^dbv@v7D8xCD(3lc#Z+#c!qmE$8gT+AnXmj;Q5rgya(oqc*ykjfbG@4pYCL z^)O>a%|bk~B97jEkiN#$A5IxO6e1Ge-U8H`!0fa1YYpNAq%Z~<_xg*O7Yr5xQpXaO z7R}!eis8LpNP{Ze9ro7cYk(=k2>hec09!27)E~HkFRQno+-br>vaO|IO`yQyLc=kh%Ey3M(MIs{9JOx!*lu zLKHO2eDPKl0wf4eQUN%x8G(GJ46~iF5sJagc7odZ1I{jt6A1u|V=UL~%IIxtx8*S>du@9p_B9QLb+6ts+cbn_NivpbGx^klEaiC*&|4pn)o@9U^~n&8!g zjJD}>YpRiDPblY?k39-mZ9`cDeN9XVJY6_U1-5^13?UuQq11?xtawdZ=TQ!tB>-cL66V z@c;PF-w>;_t)`EUS?*uHyfvOp5@S1!MG}mVH@u&vGou$4r1bUSLCMQF-ASdG2FN#; zZJy8^b4AS@o9}1@H{frdC#aGn%%^v}*tegzdd^FWM_z0?BZa4>L+k#(otGZ8^Mst2 z2wZs5PD@W@ymDsbmJUkKv!A^ug2NhL+jsL9`AxPBsOIRW=E*@XcJKXS zoW?-m2^`bEI@BpAVqPLMcM|z`av=e^2S=|i7eO!UOBY{kL(&g|<_hiul{&5wBrv^` z|Cki!YxiFB{q8zHOOdx%AEktD$7Rw1z_@WMrj>d(2lLVPowxU}NEVaTZL=C;f=lk% zla21BpF#UFy50n>S}JVNSEtwBzBL8Mz7;mnS06v2%t`c`0y}T}GOXHf{QoF;TMG=K z_4hpdztt}UAOMCr8=~N`5o!$LRXY9h|K(c@hZ8dijMq0v@a{BuXReDA=I`-(#ywtr zj(KLviQC;6&TILkZj|cv&@0y&yk zVjw9;Y1MtysXvN+oWajB<+97N;N4`}$Uu(lB3u* zHc8NJ=Tk zZF~3g$e<2U0buu*-Pu9VLrM6Yk~{wC;us@r0}i9=9Tc!IgnSO3eF7C!YG~mA`^hI> z=d*#LXoGH#bLW)9F{}&Srw1kDvH00aQ@CTi90n_XkkE0<*@Knv83SfpLdHYENU(?- zO*p|H;PA@6OI7z?x z<0{J_nU=OjZ18br0pvbknzx0-xAa#7uX=esnZEj9PDPD7q@dDu<4?KFU6W}4LaWwi z2GfX|O?rVaR05(^ltAf`{b&s?w}`9Q2^Zw2n83@kXo!at)K-N%UW=iec2oZqD*;)5p10JKesH5iNrb= zvxtOf*O!GlSYoESEs+kM^vzAMwR6lN&n!EbWuPEPFB5Rg{l*-Gqp_l73j|6zK>keyFzx&hyAOrRe+IU38k5g0ay|gZV1+c@O+nw;alL@dhsuE4ITHP9 z@^9I z`!k0QVNwaw0d^Y;=mg!XljW8Tm`>d^Fa`WB(p0l#O^B7{>76(Gw}aC(zjtDNc@)eT zxw<=)eTZnod-1lvsT7gE{N95U8{O|Y@?)UZXaqSS3^medqJcchJ}Iqu;V$P3Rfg)r zoULPr2Jwm2n-FmqBA42m8ZbxuVB5h3$mU}~m zv9)8{fo?gcQ;-b5ot{&gk0G{7Oa$t3TEP&qzMgc$hCh{~(Vc@R6^a1U3W#NiBkLsB zG1M(Cqi^nLP*AY=m}L=lF`3nogi~J%C#_oT4mr0~(aBaH2+Jnrz%G5F6JY{RTtFrk z$B)q;L|kZi$Ns~i4f_+aJ9H?VWx>gmo7e)0v3YG6H;ulD_y5yr7N|oMmgcXLg#v08 z&ix>H)$^wTH-3d3(2z~Ds55e0$qDk@oCo?*hDC|HT=w`Q%KhT>=#*{prjgXicrBNp z@=>rDdovC+@X&)~DTcm$_%_S`_^w|4@Aaf{N5buaE)awlZzIKD0&u(s;>X|E2NP|E2L>z6v*+Cv3T-m>&w)Ip5*wF8i&|z2SZ9 z3hvmSd?}xNw=_FdJU{UO0sjdevHpwz$m@TC2N+S~pWuNZ0J>oSl}S&F3Xp&cV*h9{ z>$??(9G=%=G=&*K2dB3~?}~l}ty-~&xu3y5g}s5s2hxiHreUDlTD)MBJlMVe9|N|6 z4idw_<=-$hF9028haKftZ99c#$?R%^j!E~r7db$w%dS#QG15Ta!t=unifft&CXkS& z7{$iNJMeFMiqy*UHerB`UN5AheipIM-{ z>R!k&uhWS_*2_}%^T76rRcGMhmyX&ZjUf>@eivFBu+FGFA1xaPjcSi)KvtMY||rtZXt8Q zr#AYOaSTF29yx}K;lbGkVDMXsvqa9M{w;G^H>p#Qw8DG4!o#fe9XlQuaw>~5Xq!vr zpE)=l6NatRiVON-geda`y!=A={w3!92w4NYO{uWI*`AnD% z+HMWuKC}P7n{$l8-d$zlSB6mxFc6^BYqWNHmCB+Ur`f`1!h)s>w(I>vQ0`)@wfrL| zHrY{@J4<4kp6y%3q{umzdCcmPa*|l4F;iO=up24W>-~Cw_2PX`2H?c*)C)ZjDM=(q zw^+SVN50o#WEY31MWU!NA=oku^lI#~Lfn$Znb;?rL#j{%%5=rVMu;Qxi_}AGvTZX8 zVi$XvPdnRU@RYD&Qx>cl2<1`+_b zPR2Z%PXn2MathGZ-r_Cz18o(ZNtRN!?P4+_f|-5|K754fB%W zv3^?trngZa)ywnFQGkY?wUVHxbgbc)-l zuDfK|ve!?*;XO^SYt6Ho8GBZKQ7>+91ZOQbrnS_t>Qc>)Z~pJl_UPearue z@#)vl*np5$?Xlq6H!{V-NwVt~%w*j+QHtPUw)x!@DGo3AZuJD=W+D*yf%1ghKi1&>qD_sth` z>gjlHo7&wkf3?(ot?@RK*=0(`h0H0QFa0?>{A(>Bd*_rOi-Z<)ERXExwf6XN<=!W~ zv*ei6xL=57%*L04mf{?Xu2WPD^M?*q2uFTy8%r4Gz)T6FA{&mB zDN?m^T0S09n@t%Z?4*bU&LubD*yda#;I!`%q}9AY8EHs~eC%^EUWPaK{a%@YL7?kL0#3>SN5Dj}E_7Hpz$`BT)LDeZFsRh{x zrbIfhIhJ6kZHD6u z6E)JOrX&>S9I`Sjbxu6ivu>|IEGgObPAKDK+5i)gm%2`s@~+sV8gE0cxkw4c!m67A ztPr;hcxdKxS?OS1iFNvGO@$3yb^ajyZ8mLXe+n+odbw^pZmi!gj?FlCA9~yHF$*&9 zvl@)n2AZ<`4QIyOGZusaPm4mN=9gLi;>|ePTe(dnmnQ<7r<%iW#7FV#YgjJi8_f05 zyT?3x0eyYHzs)iHdvVL99F3`?;~4x;lqwXWVkX>l+^wZeJ1>`lYOeOT+-{sl*SBdq zdn*0{#_v9#a!aESbjDKepDwQZ$1Z;z-@9MIqcIYvae!uQfJH=aCsl45Hu!Y1Y4g_N zmS-<}tjz8G+}j{t3=I$CBEO8P>Vv|Pk)=e3$&o|3sW}=P*!3vYZ`CdES`RZ4lO)_v zTaS;;0>dy;=zLH3Bc*S<0Y_rwQHV(V^wka*>Nc=212_RUwPXkr zQ#qu60e7)kX%$X2&dG~XSAl=tLiXx=4%S&rIQ1N4#63l0R~x6lzx#bJtj}dlG=C0S z8Ee8JMso4_4huB{y!9%M7rfGEtA+zL;@$b?TV0LkVBeJk>3D__li>>AW63dTVrz%E zsSYDk2WbqNSzL+X&pjh^XP+~of!x|T`uIJd#RX~@WzPD-B%o%=>@s`x=hUo_Et>?{-JW4Ai=L1#g=c3Y zvil!DsYJ(yyW!x_?9?Xfm4W?;X+>++ZUHPk%M*->MR`&F2T!~x|AQw_@(ll$_3NDM zV`NLjg4SmDhu_FI!w5Qi1m8eIvM@AE4W zuXI_8)KcoTxjaZKR7MqTld0+50`3}b&8a9iVw=#OofA!A?@$40_d$7GRev%OR>WhqFu?p&{mv-OJBP>q2# zW|YI6Rr*N59nqcVnl)qzF|j04!zF*xCEvnd=n*E~x)~xsK2OIbz90w`{h4<32t86k zPc~7-IbN_^2a3%AS7=!`bbN1-Vv20hA4E&!_LYlKZ=}K_9{M*5NZNq#_f+rDTTvU7 zH-;<2VcK-MU->A>qGK(!|4WHY;y)&WBl#Z_u_EIBkBI~x|6?MdQaSUDxHr{??o^6r z>f&25D;*Hy3aCV!en<2o4K=_I?7w;;#@B$}aftjUy+n>U>@HI@Fsw#S1QZSYeacAG&C5;f)`=z%SH0mXDG zicf~IdcSeKZyon9WzW5^N|GR*jc;gH@X#!W&?;~i4LN*niwTVfy@QeEn@pUOw==?| zc@ga87HD^fuJPgU`7Gebqe#`AVQt;ea+Xr%!WUr`#OK7NDHp+ajlUN=hUZL8rNYqu zx$`HRF&7o(!y{2&NdU&|N=0Fod{~$9PXM!}Wb%1O#KLHE042)HAv^e789TSr83Rdo ziIsKJ?Wc0zX4sjI|X%?=nS>|X7{ zBPKq7+al?efl&3rBfHmk^upr@KxWnk4tujG?T6}(oR1+r{)MV}gV4tu!}Fp`>l5AU z0{AKM8~F)k1RlKH3*i_n^*vCZE3Ajj*1woPdaG+ozwY78UElq?&dq+ETp|mp{xhkF zLF#*g6|#NxZA)H3Bf9hW8lo2QUp}qVq^IDvC$xnje<)BRWq*JFJzZpa^n}T}*3-c` zQAB*fYZJbXWdR7$L#kXRhvj^N#lt`Ja1N(u1ocxI?bIe=tOp{HD8PTI+b;S>nPGpdlyL=KU3Eq`@17%&s zBqt#WG_qT!t>jEr!XXBQ$(i*;&O2y1(;z0-e8$0JB5E9?qT(nqbY7(LO2leLg^#4f zC}P<~=phrHP%T6wa39<*$2z7ercPYPysJhqHfX50t7`EOCUE2r|tAyg9p|k|3wxY3p3J06h>*$4gd-R zzIVku?btn+Xz! zs*?_WF#GBM-NwpT-dHcbO;AFFPqN$(a+Fwxv<0%kdH0Dx%?0dC0kC(LnQI;`Pa=`e=0i?y zsx$cM-#Vf7qZ{mL7>-XEZ$0_W$b)FR(CHve9sA+V*0T-U-cWMi0a}il9y!~z>!2Uv zvP8-TO`Nw%C&CQ1wN4_b_=5#K6+U4P3rvy7S`QK|03~RC~k|(q-A5myn3mP0xSxc4Kku^|&@9D26qBEo$AP95pW$XbO7B@7?Il-QXfvF+ z*G5dZjy|3UTpYqZ^ajOJj!0z~>a`OooN88-uTJTJcFVsUPpjGwTuhs401s&=l(DPR z1?Y>GvPL4f^}Lp{uKtHUoopk!wdRHyiaoWpH4Mthr=8{E-g-lop{@4-+CQ{O6|wnl zc4lX_VIl>_Ed%>W5D>2uTL{T8hXbZYMF#hqhs(dZzm`0&-d`@TrJ)lZZyxPrx___r z=T5h!59*bl6MCkbL-Ry%4?NX}wS1oF150c=82cMIuzx!(&D6i?oaEN;NWc7vhZoWt zE}`!p+^3N%b5T}Q28#79$k7gDLZmguy8@(?o@by1Ubql(uICDMs^C_~2MBmQa}t6!e=PSn{UgGbKaO?zS(Zys)$Ov} z$tF(x9{eMRA|7$$S>o80eqU6c?8jK{!d9-o0gC5+N1+JnSrvt%vHty6aP4)5!&9>z zqm|ISB}lLUVI2|`e%9t26&dn|rs+ktSW(%7qV;o3d>@AtbE9&ErD2>!((HGgB0dfL z+yR9YtS~w%t?c(`Ob+fo*rk|*GCJ!u+kp<9PC(gP2Cn;{u)^|6waHISUYab$Ae&L0dvLa&tIIKW`nOOxy*GRPaN(L*vRzZu`n(nyHw3S$G05ZkzyD zaZCADMhV0A#UfZ=Du3PKOjsH}i3eS7hV>;VoLV$rKzm039pbQH;nOi7tEtTKrS5E0D_ z)!*KIHjXWWzfll7w?c3ag+Y6jKcy@XY26xh9HPTz&vvwu9?7kmC>b-R7sHxz*XBzb zTyi4J=A>FcLeExs8_eIy7u8472vpYSQ171T5MP!(DVDIv;0g1fw!=~LL=Zf;*JqCJ zFLOnsIa!448L+8Uy!|Pt5LO7A;(IxPv)KGNRC03ch}OVkJ#86o!aa%1I#3$J97)_A zbYYRHvfKqm09BLUeVrC(D@%^o$?hVAiD^n^*;g9G-76DAVZ>=0wAecA%2JebKZA_rvy*p$15ztkeY0?BajV57_#uPWwfV2^`uG0F3~jM-*Ap;Y>H&(44s^wKI4=4 zX25;532m|7j917QLV#E=&_(Ot;Gx&`Xx6IB%_`fRYb;VsF6hpPOg*lnTDyOtBCvIU zk%+Rpg1g-N>o9h9J0CQoD&Dd&jDu5NiW0<|t3Y)c=O?9zmr_c7qvpwlKy(I~8e-v? zy~Jdz)BllYq7$cPWULkJu#q2r}f;kuE ze^#zkMxMiYcxoe??$yPB*fOK{7kH9*@?sg<1=xsNmxWijYQ$8+7F9#_jlygEP1St1 zg~ZXpn9({0o%uiLJ}w!KZ_esR5U#|v;|n`5tN9v*(0 zp8QW*pp2V1L(b{ZVA2-ihWSyJ>hx{&gb3r%^z%FPbB-rB#rkVEsA<)N0&;_tjUeE~ z0=b8tmtY~@cb>8ENYhA`B+FAA+EQGlHr{k@)XiyL^TFh$0eTmBLy>)4w)JH6;~v_^ zos0`@1)8$$5EAa>yNzBhhH-0is9)>Z&pHr(KV81lFsfy4Srzk$Qla|>W@aw8%nQ_4 zBG~{_6rqM3YR#@gT@x1Qfxv7WbC8;V3%&>s0~fw#cF-`bI7x`g4Rg9%=hsU#hqxEL zNGY+xQIKkF3-M_{74EIFfHbF7)ojXViky!ppGE7Ndpe01!q^25^|Q8+jf`FL;(})p zCW!`tTmMUL^~7PowtwM5c@Xb$glrMYVUeW>&{fi*H1o|9fnvO5;b5iA)hjuo)VYitvfOdgwHeMd>&{C;#IF8XTY~iAH47`?utMu54(TV#JZQ=ATrRQrnAbmC~b` z;5K{$3o7Qs7qMCbgF9ezW}x9+o)ZnJF^+)!gGD&twg~(0h{A)u_jWi8y9F{$E6TM6 zNt~{?d}7pXJ`)w3g}OIfstdUCYBDUWeqq3Q;<%V+>hn{Iz(SAi^X#2*9&0x5*dI9( zgWWo@9)?0%Jm(@_8Bhq_C`uv0UO?UBZSzp<8L!eQG%j6lKfLU%*Ouv%fup%h>`nn%2EehC0YvtRtgJxqP6mc;>8R?!L0QkvxX(mSI)oq#JjRHGz1fCPvPoFMZYSzZ^lc=~A} z;EZV^trD72A0(6NPHqAv0v3H^UO;+g&u)u^3InGzz==OoB@(qg)L}i`(P&}(+w*64Bw=v`;H|x04B|;6fEFERAKJStR}qIk-@i$& z$oI&hKbKE1NSU_HtEI%DNhE|}dEuFhmpM4%_h^UI*Lj8EmEVSMps|o6aDpR3()Ibr zQ4FFTZMmv6r=t%T*zJ_30--22;P`*3u$4C3h;hc_2f{kYXH!p70-~BB$vle`?x`(D zqI)+DRh5b4!YUAxCR&Eze-3(0L!H8f9B`p`KBB4kScPQ7Whx9zh-cW?(i_AoHb}>-xX1p zz)BV5Ui;rTqN%SQee2kA1?(H6*pUjcO|f$A8uebzQE7V3w>;C6CJzD((o}u-6Bzqf z5zv0YI-E~41Z4HS-=Dg4fp4sP*!g!oSNP>BB06ZgZZqhvSC$wcMXBH*mA zywCbs%1GqbA_A|0+^@x3O24U((sd35Y)e=A^8vL@AAO%W916c=(o;j9c>NxyhsqWX zEnYkgSEQ#ZiS6AZ0Z@+i$2ozM@BSWEm913^v#r39Do(}a-p61K|1uMam984CnTEdPnXo`I;kR-9 z?k&Sr?qN`mE}l0SzpqNAjyasJR~#4!mo$hkd&e9wcLWk3s<{S_&eX76mjOQw`)i$+ z0*xu!e!CbPGU$6yJg8pj%&I=3#yRobY6h;>=fY25aaxZiD76=1*St%Zwz!(duQ>I= zOJ&E~=TMWpcx_FxCY)|J-vdk$^E@Q2aE?Ii!D-5AH_>uGLk38#D0e@wh~OPwb#4_0 zC?iArWhM2BAk21VCB(q%7-{v^J~h@baQ8(P6cTS-@j{h3uL01xHSCW87lLYEakomB z(wQwW>{CWQm$KPUe-52R_RCHJ#X(8O!n3czM_piK8iIQF$a{8 zT4ReFRNoxB?Kcmo&GsQF@4wQLRa8~mm8r(_U^au9AfrbP| zEaxMr`GkW?cd^`Op0O#j<$Qn_ih4oXDzm@V+UrZHc}5IeJw3b^b9Yx8-qdByld{;9 z9gW+JjaS>N;>%>gjz>DJgSXMvJ9xN1z{kJ$;)DA67aX=Q1>jsLM->EUnTzI$)znJ5 zWY%Xm>rJO~LVFbMcua)_OAaWUXP>3P3$-v%mAVYcfXRY}5gahW@v7Np_W+IKfVJ7DD5A zI%4&xD(d6}jddOoVt5dwrrk6#h#^EozTwASI8FdyZV*-sTH5d7_oj0wFd zN1VJc^O z>N8@Ci<;F`Y)b=38O}t%&(eO+Yk@ReW(|kLM+nM^b=bldkcQ@K?;@gF2Hv-AU}3Kk zU;}xKY+-$ookZy7AZum?(ne#aDDr;@Pk_3BJ}?^|BmJ$=07C~&%7QjYtZA_AAOEa0 zU7=Cc$HI2j;@R}`g+LZ5<=O!#;q+j)mIO}18(`T&Nx+wU@E2V71&Y+}43sd}h4d~2 zf#G()Mx3l|zT%Q97KwUIw4}GuVT{RrMFJb&Xc=LQq!MXFo7L;#*lzcme8E3J?x|&0 zB~{Z9U#1RoEBj`!-qvy;jl$^!v=Ddr{8VGXm^kpb5!TUkFA5M#6 zsK?XU(t17&7JG20<||~KbHLXAJ-80SriF~LgQM>HhoYyvDOl=EEs~3Mc;p@u)}J33 zsLs|5@)Jk5BV6Uml?N4J#m#@6!*)7>0aXYqo7S>RIV+~#lqThbVkU6Z+q(#U%sjfR zun*UKS$bdlDAauF@QSImZ!{gXfu~iP9n!kqY`Q6>UAG}(SGl$gfO5>4Nk{C83`Ht& z!$Pd7jhwcbBb=OEwCC7+GIEMeZ^DE^W=ecc>}|93%y^Eyx{dA-QgVOpCD?`YHDMBI z;D9jVJ-ELkbk8Na8<&soufZ?(E(dutr@>b1AO%H{D*DV8C5lW5y_p!7l!VZfv9jq; zy(u+iyVUYJQGI{Yf4FT)kmXo^mThf2SPmY7%hYqqptDW zhI28MqI!Nqw{0`pyjT@`0Smx6*S)B0Gp^5H!cwE9z~*hyKDhn?re2+1INR=COZxB?qOOHm%9}AgNV+(3<4&6Kw^|@n5vy&f|G+y=Xe0You~ufwtHN`Uf$iVpg~jZpx;Ym*z_fB8 zF>zXgD2Y+q>)IV}1Kuab8@q0s%+OIFPjEF1R^}qku+os;Fx&T7KqD4}vqE83$Im_k zVNaer8;H72gqXfq4*~CmzHF<8EWiM`*gnV)@XW6c+wx>tb;ClpQ1X%a6JDC&AQAL2 z@f|NJ+*BIzIV?dAzkpTuvL85vtv1t-ohxsD*51ZxF)+p`f5lFdFm$Z{+qj~EvAb|@ z>e@c<9hkm!D%;CRIbyWpMrq}?EZpW~C%GKAEbP@fy%xGpWqI&L+T3%H?H7RoV|| z)vGeXYDQT*GFD0AhZn0{O`E#ABU~?Vr{=%2xxkS8=BHoHN~&EiHZ#ex(`_r1B6b=D z3mC+at}6BtRr<`{QjJ{oho@ge3nx%?4f^d~52R!5MHaR&feHO%7(r0g;dCaB7+pIwZm4iARQW zF?Vg$EKEzv_aWW<@hSm7u$@8}fi-ORI;6}Za-4Nc1Tq|VZmPQ9T~vNBa7io zYO!B0Op=I#CG^nK-OzQwSx3sXb$Qt4^X$=)#tj}QcYr>>F{`q>&X0R zOccs(eawi(lw(OWx=#*f0|p7I-o^6jh3mJK0JCIv9$52%4NriWOI1pEIz zPHw2h`q6Qq5b`G->=HKe1sLfZiRNtC)25#BY;_&Wndh1_^jK(GIiydPpt}L=Tn-g&DB2F^B7D~^ ze$RX@b%o$Q3cl{IBm&Csw|BF1sjqRy9`ia)g_*Rgn{glS=G;8zc805d>*>U5X8V+0 zY(qZJ@4tLsJ~m6z{TZdnVFxl7mHj`KZjY~o-c=&=Vi5xPb!utdIVHpg;G(kka;wka zW67YbG_yhtMCU~iW1?%M=4=;*VHCqv-%ri1U;iDTbNE!Kx*NI;iFD*rr!)*yniyVRR+uUFKn6>z z5!Nf|T7R->Qq^Ti$CXp(O$0=AJ5eW#d|;nl*Z3v1#hS4iL)I$8Y5S!VfhJmNH6r$o zYSM2n&km-9abl6;16S*do%i1>8M^iTp%rmUPzMr&q(G5WsQ63qSjhr1t-K@E?2spY zOpth?2h&@g_}??IU_=8Z$y^d%(A4NE)T@3r4o*!Flu^j=|6B&!*-(yz)PXt%HjyI+ zU5~_K0r4zp^^-;0*9g^q%8r>O@8_D27HYP*r6XPV;yoiU5cUseJvZu3_kE>YjoN`1TLCTtWgXTNeY6JHk7>F>-{B|B4kesw|! zyhF&#Yq|yh#S7$!{41is^Arf7L$dt@!`y*y1)+_9D;5K`!t$}C`eZ7nHpO#5w~|bG zg{YQiqOsJD2L+Wg6cl~`ybEooN-_?Rr*F z=aAY1g&Ai@42Wbv+0}dfQFkO6|3E#AzKAM3mUs8tq1W&8c9Qv3?2|*YoTc5ObpqgGuxL<#?HiQEAY0xbz~H0Jl0XU3m))2dG7 zxN1FinPm+aWTO#8tTajqhBLs&>c>Q|d;Cj-HS6dKdfvC+RoBM;MKCYFoywM~j{l^K z4Jbq$Ab+XNq~(!_)>xBrqYkU_)b_ZYn%P>;*#o#2d4%=vZ8W<9@U=G{ zq@3pD4$iz{squ_F&6LP>V0sS2U!E5)e=~7U1~3GYs$1F7X}H9;qaBX;Hb}1b)48O9eMnq=>4@=+ttI2 ztaqNSF1!Xmm;Y#HK@9TWW*y_NCMoGCg@f1Z(Q*whjnK4?&>KZribGKOc%;`D==i)v zBlE1+GWs2~bfuE=_!k*8)$N9i1(ncV8aMNP$mH?zQ|dOcuA@HI%+ez>f`{pDP!p() zIdv2=yDI9Hj>Gt~K>tg*&xv-3Dy}YwcKFJWw1i;r2xVX~Fgg_Yjs_F2igpBUATQ-2 zF7+1Q9K793Yms2y3id{V{R}4I!)-VuDEBnp4x^5J5u>eM^12llNaKN7KTkD66qW78Bq+66QUDINR}nvFAUj0bdUEQz zIlgU32_@?}XEGVuLk`L#mZh|~8p?o*W1eJpi8TuNmcbb6D~4)0k9aUGI!XVGhHH2* zckT{_{p}r`Aho$TnLxz^_>>~JLXH!r>?^BN@zdIMTT-XhI`Ld_p-0CnB6%p0q3GQ{ z6tc^cRJ_XKn4?h)({p1W<0R`T_9HUVrgdpvn)kBWAkGQSZHjyG+8_@n$k$7!mCP5X_Y}J zZX9!0NzsH8D=i_*!jc+dhfm{1Cp?n8EyUCBd_Yoyq(dpG1i5EBD6(ir^gx`@Tsx`_ z(1qa({WXOX`D@(V!WDZ!HC^#A!Cu1Y#`pmD8$G0q&N9V^QUg3xl(in0Bb$39zIAKG zYM%(X>6n~1E^Pxf?MI&0(M=z{kQ^uY)rPLRJF+E6_A%u_!g{syipgz1>`UV2RZkV0 zo^me%B2=aCUZo~-Ckuh@TE=>?>ORUH@%pD!H3&e8lFfs)5zzr8zr?oe;sk?B z*9=!}6H1`|l;8P%2RHwpEO)(^x5C9^!E>5nyuDPx-wVfA>4Lw;2M)Zbn@QVhJ=2Alhci1(8q7Krtl5`G^_8}lIzQ8Wy zM+&IiFl)N@-dNs9_&|XjIl!F=lvRLK$S<=z2P#ji910|xp3q~?CNHSMTF8ExoIY8J zCPWw@OHb037c)Z&3q%4w#xg}&mxW5=kLx|#W;>ISb%yb(rj%BYPqD9;p{Ir8kw3tr zeOsX(2}Cm)oP3wSs6sn;?RUnqWe8^NkQv9Wx`oLtmimKBta1?_jh`8@0lA(I(`bxZ z=@zJIX>!r?G#$Jgh0{Dnh#a;h{l&cVh{<4szJh(bhRdBJMq)_efHWw0f;#M-}05B z`riJGRaFim(YjT4mg+rXRWEQ8-wtUcNLvzNhm_ej6U9`^H%d*dN9Uw=K&rl0;Z&4h zz$DmfVsgB|knv9UmQzXQ;djH-2f9UDje5p{V{gC8=B zMng=(E*g~liXuETtY(PtEXd+~mvMoZAP5^uJ1~6Mich?W zm}VDT<~ET|O2{Q^rk|C>0P|G&^W^48GU~>flW(DZmMaMdXE&(9rEtoyxW_*8__q$q zhAxO=if~S$vgrBZTK?(z%rzeP(3U0Lp#}ILZq;Q4wYd@9i!e&9;*d9@-`QsMHH!8T znxy3%$2D~wBQDg#kzyh7JD>Dz7%`AuV)e)&KxYb#344Tz&4~_=oWLJpn&jxy|2S79 zBDO(2hM65x|FxiaJ*lDUGOia_Xcr3C!4;cx~NNaHEBo0mhF}s{#3jyl%4gL zW#cVN3lAs(qZ5X$VN)$v@TWUJKjo?Xef4a3SpLEL>%Hya-=9)t1ZxS#ab_dIZ4{pz zL}mZCY2U4qKHM5JfH&bR^^?-w%l!l6?_+&NmyNfFA+DoMj?tQ~ZkjXOJ8De#0T_?I zxQi6CmI(d}+wgA6sy4HKsnt7)%O0S!u7}x@*s~Lq&$8Jo#_Ej6(_Bve3a4?3}{}1~qCHn_j%F z5p-Kini|!}ecn=;3}!1CNA7qZWYhpT0 zA2xNmBQ{*!v{i7!L5U>aJkmYk!h+TDdMh?V>19(IA;634CGl;8X^3)pz5VywpP^6j zbEEh33A+$D@H*LdiMaR9L~(Uou<#1|fsXQ;N^GHWqY$u7O1vHE#XB2Isoj~{wMWQy zCwzXA9XDlLWPp=RO1F-WAn!}Fzn~>BbjKi9hNz^d>~H#M#a#9#87{z6?-WH&pu}H2 zu+CVxPM-NcaFO76z(+KmwFdvTd7!2T_WUMCHp;gFF>X>D5MnnnV}yXXnH9D?OATGP zI;$7({%rGkGg(&m>azlyx;tNIndJERHv8>%;7-Ud9Xjq4Z`Kw%SZ~8vj?j0wd0;zL zbD<`cNIivIR@=BLGNuW+eGeJ2-C*cr`u1F2_lJR2o7Q{NG`TcA)8K{^o zc?$$W_uh&&C;En`RG!T!uys+FW7uic4CB#zR8g}h>dCX+JKLME!;OYIN9E3N{ymwd zk5*o^HXg>Xd+h(HR06@Z}&x*KVi{CPy)?PMQ*(I7j0bq7EgPEY&f~V#iljr zh=FU__XyFjDFx?Yp||f>gof=)Bk+wG;?;0jPKtO6iwRHK#&8Q4^s`0jF+Pt)lN0aK zc<$9oxk4SIYEw*5;A6t|vMqIzDR2?KPF=GxZ4*7NnR`Y?$cf{}zN+R>nq8x%#ff9r zvD_OJSbNJ!M7--uy%D*L(V3(7p3p>xKq)89!x0t$00kJ=_qIc3pjG7rC!wtEGe|E? zsia9fc#`_%X$Q-T>qB90-8YGen=V`^LEs$m1J{aF1UxRz1kwT0{Xd6o7c{K9b71$G zN&=BIud=C3_o3&7*&dOmV#9-EMTh_BjDJ>EjtaMAx;$OP!z{?0(!n#AHq~KdUr3;> z%nkK|BNht?{$;pBqJV2l&{(iFLt|*NnS1BCY~h?o7|S9%)?S;0RlN?(E{H1hV)xH1 zHT zNx9{qjRfqixfOElvi0+hCBTtK!POBBRt>T&d=iz5+a?+Sq8o8|&K0@37y-Qdz>kbZWtU-$4}YL-e2>6>x$N~j0)_bBePqO+Eh zt)wEeOT5t|2Lr$>*7cJm>{OtBfICvy#MxL(rE+AA~?&S!90$_Q))Z3Lnjliq{Ouch)Ge-rZy^rWb` z_f}wujo^?~Fikmr-)s4GpTg+IZZK zVzW1=&I?lw!-Dln=s=>vu9~T{_RyJ9krMxfoArhu7^gIHQQ?SxP~tmBI#1AB(Bbdf zfbB@~?LhIXXzf-YegW_|b|bhw9~+4fzkJdD7e?lMrn{ZoS&!_G)wf6A>3mrw_+<(H zl_rgiC2%qB7nSj|bfTx=Q7uS%e%eZI>EQTS;Q(|0vArL-*zXtSu!{srEaq;fKuxUo zZM~U0Xp<=7orrK!%UR-UVN1F#o&R|yWcfihlw|}S;=T=dDaN`Qir-SJ58m_=A2bBT z_AS#bcqs)IlnsSrZ{82w63Jn9RkMw$Q@RxoBz2u&nI~2Uw%m9XUi4kAU}Kvh zm?AUuL1^+{R;@a9c)vS!$cHA?v^8{lLa}Cb|Jv%>9Yjq(2_v^Lcr4`tL`lF2@LgCS z{#9Z`_-3L8DvQX#L*jV@KvxQ?JpYrzATKeL!HQ*Fj@s|S_L3FEIwK_8(VdUVTK_yJ z9$){X3(d|oEH%6ajG_pmVQ8>cZJ2jV0yH+-A9YrA?td-LCbcX@{N;?KbQ00kvVWcP z{VXfv__ULLhrnTk+gO4|u+#Uk$;V5M4wx#%rBziIs#TL(zt2iKJ&4HQTF^JFt-YLv zlYX(gCc4N(Dcs==!p_=nub)Q<$(Ej^xQu21f5N!1q(sdvZ>EEkM3 zL?@M2T~E?=&*L`Rf2lP)jmvtiFFG5^U+jG#9 zRXyA_B(~NB)>Y>kqrscg4bf{&C(GBPqv(}~q8*2*_3!iQU)m+RU|(xdQzNXB?^`UB z!AvFbS55w?#48tnClHPeP2QeOvkMdK+~r1?X*)^FleA=p?5n36*12S|RjS199Daf8 zp}aZ9AiLE|-w$g6rF^i=l?eT4VY*u?Scd<$I4O6`aZ@T7k{Q-A&_jhz>;2S=MF}_e zzYrYIlf@ElxS9m5p&Mh~i+YG#e+#0MF@N(v;MWrN5dAhaWP4lj<>_1-GZ{@`UmMr1 zJ;#B9I!LRo1xV-MSeOFg?$*n68Uv)lE$9ojZmINk4!$2LT)HQNWYW#9|7_J4DON4n zsU7Ht)G@5>yixJaf1bj{>uP<{T!Pwx*_}s!v&LU%uvf5!5q*)8*6d9zpWOSBOI@Rn+rcW( zkr9A|R`^K!7qD1vmQM%O?(LP3a47>{~ro{%l2p>b8_?h63(F91QsrS|~M?Bo4<2PbvHd(r)`n%Wn z&ng3jNc_c_L7v)Edp-F`{QA3%slC9KtT6=!s*z|0PLYS?=y<6llNFgZAhp_5O7i4J z1rb1_@^Za(Q0;sO3L0zFN+w+5%IV1P_%kaco+%Q`Ww0TVS3)ClWx^?nE(0UwpQw30 z`-~+Km9=q>1ikA0iuU_YXZllOTiOs;m0%BHU3~F{wQXe&~*LmJfvIqTdP?u zA%!eQQ$HJaESw1qL>gy#o}}nTOlq;a)ePyl!^6=!#43y=0C+n%LU7E%fe!S{EKy%F{kPK@@sH&{JcKF4_2ol^JA34KSfQ28$<~c~3wF{-7I6XO( zawKhF1dD74yj6Tlv{vUEfQf9k^!=nrxSzJUS%L>&+dNDor&eO^^}ML8vVJq(;Ick4 zTr_z=h9$#BZY#AFZvQ6S3z>Dv4`IsTci-EsT$cR<^l4u2udZkI?YE1+WmmH$?%T%HC$(9b+pW`!XJ8vx7y<3Yn}wvY4~H0c5?y|g79#wZt3&`+7NR>ISSEr>_RDym!GKkHqzL|I)`+ z&DVCU&0+&Mz%)%Vzpbgm(ZAns7iCI$o5=Yyy{31Y(@Gb2uDZ5Oh5PpFpr*8;Cxu^Z z7P!0FJWPXPp+_PsGX`MaylZ|RL7>ex6;feloUCgGIP`r$L^PjvNQV0K`OIxL-`9Ur z&}=AgJ7u{r+SzP=yQ|A@vwk+I{lbx%UsF4!?|&3W^|~g$32^m{wZPbsEl9y9T*~{} z{m3Yl!6qZ~aD1+G^r+O#w!FmLKOqSf)Z$D$$M*eJ({8JDcu3|gDDoY&09~`8<|Syi zzOZ?IYEw*Slx2n%VsJP)EA-17?e$ji1xXVPw=UXGX{O`0WxD^5c|#X53+A)WMSLIg zXG)uA#jYEmPY07>UtsOHLu-+dVX@7qo&^Wc6GR z9o9~)mv>n;j}vC=H2!}9(bAkCB;esgKB;R3Y&Yo4qo^oXu4MxEDw$mpraD!I5|gb1 zP4J~&pn{g9QmFc4qv%-`q zwK$OXP#JlW9Wn3Q9{S*A?waZR^e7+?WmItERVfU0oyt@J#Zv4&39bz`8WO5Go_S*7 z9gYWg!#$5tam6G5^UQ#WN}N$5F_br$nNq}CToAv zu0PuyPMLiD?jZ=}B`{CZ)(&}qnFA_o40#~cJ=|7Rmc0#WZbc4x;447+eEjzBDvtZN zkKqh<8?*eIdsm4E)3MOk58+-WkCoh3V5@G8u1l|;n`p)yIHMc0b%b5q3_cz%9zRdk zmY;?0b5!39ryf7=XGm{3IUenurLk_>p8#|9{%ZoKO5h*YuOzx)NaWPS`e@^Q=TWqM zzBv{>>aRu4iD89!H3v?l&a<};&;C-QLj}@GfJRBRcoJvfaeL1X%x>h*##w(k$oGr8 zw~Sc7Z){!0V4$K@25?y3*(aR^G;|8`MR&X8Jn$?E+~)_ksmE_ekMLxB(B9RC%|jI*2X zo8rAu!c@fKc)N?cTig=e3zIx^wwplZrf$tg?OET!5a43eF41xEdmk5iJDl`Ad + + + + + + + + + + + + + + + diff --git a/infra/s3/bootstrap.sh b/infra/s3/bootstrap.sh new file mode 100755 index 00000000..0a888d25 --- /dev/null +++ b/infra/s3/bootstrap.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -eu + +for name in rustfs_root_access_key rustfs_root_secret_key s3_access_key_id s3_secret_access_key; do + if [ ! -s "/run/secrets/$name" ]; then + echo "missing S3 secret: $name" >&2 + exit 1 + fi +done + +root_key=$(cat /run/secrets/rustfs_root_access_key) +root_secret=$(cat /run/secrets/rustfs_root_secret_key) +app_key=$(cat /run/secrets/s3_access_key_id) +app_secret=$(cat /run/secrets/s3_secret_access_key) + +if [ "$root_key" = "$app_key" ]; then + echo 'API and RustFS root access keys must differ' >&2 + exit 1 +fi + +# These values become path and JSON fragments below. +case "$S3_BUCKET" in + messenger-attachments) ;; + *) echo 'unsupported S3_BUCKET in bootstrap' >&2; exit 1 ;; +esac + +rc=/usr/bin/rc +policy=messenger-attachments-api-v1 + +"$rc" alias set root http://s3:9000 "$root_key" "$root_secret" \ + --region "$S3_REGION" --bucket-lookup path >/dev/null +"$rc" bucket create "root/$S3_BUCKET" --ignore-existing +# A new bucket is private by default; this also removes a stale public policy. +"$rc" bucket anonymous set private "root/$S3_BUCKET" + +cat > /tmp/api-policy.json </dev/null 2>&1; then + "$rc" admin policy create root/ "$policy" /tmp/api-policy.json +fi +if ! "$rc" admin user info root/ "$app_key" >/dev/null 2>&1; then + "$rc" admin user add root/ "$app_key" "$app_secret" >/dev/null +fi +"$rc" admin policy attach root/ "$policy" --user "$app_key" >/dev/null + +"$rc" alias set app http://s3:9000 "$app_key" "$app_secret" \ + --region "$S3_REGION" --bucket-lookup path >/dev/null +"$rc" bucket list "app/$S3_BUCKET" >/dev/null +echo "S3 bucket and API identity ready: $S3_BUCKET" diff --git a/internal/broker/rabbitmq.go b/internal/broker/rabbitmq.go new file mode 100644 index 00000000..892bd0ee --- /dev/null +++ b/internal/broker/rabbitmq.go @@ -0,0 +1,119 @@ +package broker + +import ( + "context" + "errors" + "net" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + Exchange = "images" + Queue = "image.jobs" + Route = "process" + dead = "image.invalid" +) + +type boundedConn struct{ net.Conn } + +func (c boundedConn) Write(p []byte) (int, error) { + if err := c.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil { + return 0, err + } + return c.Conn.Write(p) +} + +// Connect supplies a freshly declared channel after a broker connection loss. +func Connect(url string) (*amqp.Connection, *amqp.Channel, error) { + dial := amqp.DefaultDial(5 * time.Second) + conn, err := amqp.DialConfig(url, amqp.Config{ + Heartbeat: 10 * time.Second, Locale: "en_US", + Dial: func(network, addr string) (net.Conn, error) { + c, err := dial(network, addr) + if err != nil { + return nil, err + } + return boundedConn{c}, nil + }, + }) + if err != nil { + // Connection errors may include credentials in a URL. + return nil, nil, errors.New("RabbitMQ connection unavailable") + } + ch, err := conn.Channel() + if err == nil { + err = declare(ch) + } + if err != nil { + Close(conn) + return nil, nil, err + } + return conn, ch, nil +} + +func Close(conn *amqp.Connection) { + _ = conn.CloseDeadline(time.Now().Add(time.Second)) +} + +func declare(ch *amqp.Channel) error { + if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil { + return err + } + if _, err := ch.QueueDeclare(dead, true, false, false, false, nil); err != nil { + return err + } + if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{ + "x-queue-type": "classic", "x-dead-letter-exchange": "", "x-dead-letter-routing-key": dead, + }); err != nil { + return err + } + return ch.QueueBind(Queue, Route, Exchange, false, nil) +} + +// Publisher has exactly one in-flight publish. A failed/ambiguous publish +// invalidates its channel before the next event can be attempted. +type Publisher struct { + ch *amqp.Channel + returns <-chan amqp.Return + confirms <-chan amqp.Confirmation +} + +func NewPublisher(ch *amqp.Channel) (*Publisher, error) { + if err := ch.Confirm(false); err != nil { + return nil, err + } + return &Publisher{ + ch: ch, returns: ch.NotifyReturn(make(chan amqp.Return, 1)), + confirms: ch.NotifyPublish(make(chan amqp.Confirmation, 1)), + }, nil +} + +func (p *Publisher) Publish(ctx context.Context, id string, payload []byte) error { + seq := p.ch.GetNextPublishSeqNo() + if err := p.ch.PublishWithContext(ctx, Exchange, Route, true, false, amqp.Publishing{ + ContentType: "application/json", DeliveryMode: amqp.Persistent, MessageId: id, Body: payload, + }); err != nil { + return err + } + select { + case <-p.returns: + return errors.New("outbox message was not routed") + case confirm, ok := <-p.confirms: + if !ok || confirm.DeliveryTag != seq || !confirm.Ack { + return errors.New("outbox publish was not confirmed") + } + // RabbitMQ sends basic.return before its confirm for an unrouted + // mandatory publish. The client's frame reader enqueues both in order. + // With a single in-flight message any queued return is for this ID. + select { + case <-p.returns: + return errors.New("outbox message was not routed") + default: + return nil + } + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..4169f4a4 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,180 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Database struct { + PostgresURL string + RedisAddr string +} + +type S3 struct { + Endpoint string + Region string + Bucket string + AccessKeyID string + SecretAccessKey string +} + +type Config struct { + JobMaxAttempts int + HTTP struct { + Host string + Port string + } + Database Database + + Session struct { + TTL time.Duration + Store string + } + + S3 S3 + + AppEnv string + CacheURL string +} + +func Load() (*Config, error) { + config := &Config{} + sessionTTL, err := getSessionTTL() + + if err != nil { + return nil, err + } + config.Session.TTL = sessionTTL + config.JobMaxAttempts, err = EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20) + if err != nil { + return nil, err + } + + config.HTTP.Host = os.Getenv("APP_HOST") + if config.HTTP.Host == "" { + config.HTTP.Host = "0.0.0.0" + } + + config.HTTP.Port = os.Getenv("APP_PORT") + if config.HTTP.Port == "" { + config.HTTP.Port = "8080" + } + + config.AppEnv = os.Getenv("APP_ENV") + if config.AppEnv == "" { + config.AppEnv = "lab" + } + + store := os.Getenv("SESSION_STORE") + + config.Session.Store = strings.TrimSpace(string(store)) + if config.Session.Store == "" { + return nil, fmt.Errorf("empty session store data") + } + + config.Database, err = getDatabaseURL() + if err != nil { + return nil, err + } + + config.S3, err = getS3Data() + + if err != nil { + return nil, err + } + + return config, nil + +} + +func getSessionTTL() (time.Duration, error) { + value, ok := os.LookupEnv("SESSION_TTL_SECONDS") + if !ok { + return time.Hour, nil + } + + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/int64(time.Second)) { + return 0, fmt.Errorf("invalid SESSION_TTL_SECONDS: %q", value) + } + + return time.Duration(seconds) * time.Second, nil +} + +func LoadAttachmentCleanup() (postgresURL string, objectStorage S3, err error) { + postgresURL, err = LoadPostgresURL() + if err != nil { + return "", S3{}, err + } + objectStorage, err = getS3Data() + return postgresURL, objectStorage, err +} + +func LoadPostgresURL() (string, error) { + postgresData, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE")) + if err != nil { + return "", fmt.Errorf("read database URL: %w", err) + } + postgresURL := strings.TrimSpace(string(postgresData)) + if postgresURL == "" { + return "", fmt.Errorf("empty database URL") + } + return postgresURL, nil +} + +func getDatabaseURL() (Database, error) { + postgresURL, err := LoadPostgresURL() + if err != nil { + return Database{}, err + } + db := Database{PostgresURL: postgresURL} + + redisData := os.Getenv("REDIS_ADDR") + + db.RedisAddr = strings.TrimSpace(string(redisData)) + + if db.RedisAddr == "" { + return db, fmt.Errorf("redis addres is empty") + } + + return db, nil + +} + +func getS3Data() (S3, error) { + s3 := S3{ + Endpoint: os.Getenv("S3_ENDPOINT"), + Region: os.Getenv("S3_REGION"), + Bucket: os.Getenv("S3_BUCKET"), + } + + if s3.Endpoint == "" { + return s3, fmt.Errorf("empty s3 endpoint") + } + if s3.Region == "" { + return s3, fmt.Errorf("empty s3 region") + } + if s3.Bucket == "" { + return s3, fmt.Errorf("emtpy s3 bucket") + } + accessKeyIDPath := os.Getenv("S3_ACCESS_KEY_ID_FILE") + + accessKeyID, err := os.ReadFile(accessKeyIDPath) + if err != nil { + return s3, fmt.Errorf("empty s3 access_key_id") + } + s3.AccessKeyID = strings.TrimSpace(string(accessKeyID)) + + secretAccessKeyPath := os.Getenv("S3_SECRET_ACCESS_KEY_FILE") + secretAccessKey, err := os.ReadFile(secretAccessKeyPath) + if err != nil { + return s3, fmt.Errorf("empty s3 secret_access_key_file") + } + s3.SecretAccessKey = strings.TrimSpace(string(secretAccessKey)) + + return s3, nil + +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go new file mode 100644 index 00000000..e94e4a9d --- /dev/null +++ b/internal/config/pipeline.go @@ -0,0 +1,77 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Pipeline struct { + BrokerURL string + Concurrency int + Prefetch int + Timeout time.Duration + Lease time.Duration + Retention time.Duration + CleanupGrace time.Duration + CleanupPeriod time.Duration + RetryBase time.Duration +} + +func EnvInt(name string, fallback, minimum, maximum int) (int, error) { + value := os.Getenv(name) + if value == "" { + return fallback, nil + } + n, err := strconv.Atoi(value) + if err != nil || n < minimum || n > maximum { + return 0, fmt.Errorf("%s must be between %d and %d", name, minimum, maximum) + } + return n, nil +} + +func LoadPipeline() (Pipeline, error) { + var cfg Pipeline + path := os.Getenv("BROKER_URL_FILE") + if path != "" { + data, err := os.ReadFile(path) + if err != nil { + return cfg, fmt.Errorf("cannot read broker URL file: %w", err) + } + cfg.BrokerURL = strings.TrimSpace(string(data)) + } + duration := func(dst *time.Duration, name string, fallback, minimum, maximum int) error { + n, err := EnvInt(name, fallback, minimum, maximum) + *dst = time.Duration(n) * time.Second + return err + } + var err error + if cfg.Concurrency, err = EnvInt("WORKER_CONCURRENCY", 1, 1, 4); err != nil { + return cfg, err + } + if cfg.Prefetch, err = EnvInt("WORKER_PREFETCH", 1, 1, 16); err != nil { + return cfg, err + } + for _, d := range []struct { + dst *time.Duration + name string + fallback, minimum, maximum int + }{ + {&cfg.Timeout, "JOB_TIMEOUT_SECONDS", 60, 1, 600}, + {&cfg.Lease, "JOB_LEASE_SECONDS", 90, 2, 900}, + {&cfg.Retention, "STAGING_RETENTION_SECONDS", 86400, 1, 604800}, + {&cfg.CleanupGrace, "CLEANUP_GRACE_SECONDS", 600, 1, 604800}, + {&cfg.CleanupPeriod, "CLEANUP_INTERVAL_SECONDS", 5, 1, 600}, + {&cfg.RetryBase, "JOB_RETRY_BASE_SECONDS", 2, 1, 15}, + } { + if err := duration(d.dst, d.name, d.fallback, d.minimum, d.maximum); err != nil { + return cfg, err + } + } + if cfg.Lease < cfg.Timeout+time.Second { + return cfg, fmt.Errorf("JOB_LEASE_SECONDS must exceed JOB_TIMEOUT_SECONDS") + } + return cfg, nil +} diff --git a/internal/database/postgres.go b/internal/database/postgres.go new file mode 100644 index 00000000..76a8d2f9 --- /dev/null +++ b/internal/database/postgres.go @@ -0,0 +1,21 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func NewPostgres(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) { + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, err + } + + if err = pool.Ping(ctx); err != nil { + pool.Close() + return nil, err + } + + return pool, nil +} diff --git a/internal/database/redis.go b/internal/database/redis.go new file mode 100644 index 00000000..5ebefaa2 --- /dev/null +++ b/internal/database/redis.go @@ -0,0 +1,31 @@ +package database + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +func NewRedis(ctx context.Context, addr string) (*redis.Client, error) { + + options := &redis.Options{ + Addr: addr, + DialTimeout: 3 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 2 * time.Second, + PoolSize: 10, + PoolTimeout: 2 * time.Second, + ContextTimeoutEnabled: true, + } + + client := redis.NewClient(options) + + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + + return client, nil + +} diff --git a/internal/database/sqlc/attachment_variants.sql.go b/internal/database/sqlc/attachment_variants.sql.go new file mode 100644 index 00000000..5558324d --- /dev/null +++ b/internal/database/sqlc/attachment_variants.sql.go @@ -0,0 +1,88 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachment_variants.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createAttachmentVariant = `-- name: CreateAttachmentVariant :exec +INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9 +) +` + +type CreateAttachmentVariantParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 + Name string + ObjectKey string + ContentType string + SizeBytes int64 + Width int32 + Height int32 + Sha256 string +} + +func (q *Queries) CreateAttachmentVariant(ctx context.Context, arg CreateAttachmentVariantParams) error { + _, err := q.db.Exec(ctx, createAttachmentVariant, + arg.AttachmentID, + arg.PipelineVersion, + arg.Name, + arg.ObjectKey, + arg.ContentType, + arg.SizeBytes, + arg.Width, + arg.Height, + arg.Sha256, + ) + return err +} + +const listAttachmentVariants = `-- name: ListAttachmentVariants :many +SELECT attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants +WHERE attachment_id = $1 AND pipeline_version = $2 +ORDER BY name +` + +type ListAttachmentVariantsParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 +} + +func (q *Queries) ListAttachmentVariants(ctx context.Context, arg ListAttachmentVariantsParams) ([]AttachmentVariant, error) { + rows, err := q.db.Query(ctx, listAttachmentVariants, arg.AttachmentID, arg.PipelineVersion) + if err != nil { + return nil, err + } + defer rows.Close() + var items []AttachmentVariant + for rows.Next() { + var i AttachmentVariant + if err := rows.Scan( + &i.AttachmentID, + &i.PipelineVersion, + &i.Name, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Width, + &i.Height, + &i.Sha256, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/attachments.sql.go b/internal/database/sqlc/attachments.sql.go new file mode 100644 index 00000000..f8649f79 --- /dev/null +++ b/internal/database/sqlc/attachments.sql.go @@ -0,0 +1,306 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachments.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const attachmentExistsByObjectKey = `-- name: AttachmentExistsByObjectKey :one +SELECT EXISTS ( + SELECT 1 FROM attachments + WHERE object_key = $1::text OR staging_key = $1::text + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1::text + UNION ALL + SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text +) +` + +func (q *Queries) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + row := q.db.QueryRow(ctx, attachmentExistsByObjectKey, objectKey) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const createAttachment = `-- name: CreateAttachment :one +WITH legacy AS ( + INSERT INTO attachments + (chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256) + VALUES ($1, $2, $3, + $4::text, $4::text, + $5, $6, $7) + RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version +), job AS ( + INSERT INTO attachment_jobs (attachment_id, pipeline_version) + SELECT id, pipeline_version FROM legacy + RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key +), event AS ( + INSERT INTO outbox (job_id, payload, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at), + 'initial/' || id + FROM job + RETURNING job_id +) +SELECT legacy.id, legacy.chat_id, legacy.owner_id, legacy.filename, legacy.object_key, legacy.content_type, legacy.size_bytes, legacy.sha256, legacy.created_at, legacy.updated_at, legacy.status, legacy.error_code, legacy.staging_key, legacy.pipeline_version FROM legacy CROSS JOIN event +` + +type CreateAttachmentParams struct { + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ObjectKey string + ContentType string + SizeBytes int64 + Sha256 string +} + +type CreateAttachmentRow struct { + ID pgtype.UUID + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ObjectKey pgtype.Text + ContentType string + SizeBytes int64 + Sha256 string + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + Status string + ErrorCode pgtype.Text + StagingKey pgtype.Text + PipelineVersion int32 +} + +// Compatibility for the synchronous API until CreateQueuedAttachment is wired in. +func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (CreateAttachmentRow, error) { + row := q.db.QueryRow(ctx, createAttachment, + arg.ChatID, + arg.OwnerID, + arg.Filename, + arg.ObjectKey, + arg.ContentType, + arg.SizeBytes, + arg.Sha256, + ) + var i CreateAttachmentRow + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const createQueuedAttachment = `-- name: CreateQueuedAttachment :one +INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, 'queued' +) RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version +` + +type CreateQueuedAttachmentParams struct { + ID pgtype.UUID + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ContentType string + SizeBytes int64 + Sha256 string + StagingKey pgtype.Text +} + +func (q *Queries) CreateQueuedAttachment(ctx context.Context, arg CreateQueuedAttachmentParams) (Attachment, error) { + row := q.db.QueryRow(ctx, createQueuedAttachment, + arg.ID, + arg.ChatID, + arg.OwnerID, + arg.Filename, + arg.ContentType, + arg.SizeBytes, + arg.Sha256, + arg.StagingKey, + ) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const getAttachmentByID = `-- name: GetAttachmentByID :one +SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE id = $1 +` + +func (q *Queries) GetAttachmentByID(ctx context.Context, attachmentID pgtype.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachmentByID, attachmentID) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const getAttachmentForUser = `-- name: GetAttachmentForUser :one +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.object_key, a.content_type, a.size_bytes, a.sha256, a.created_at, a.updated_at, a.status, a.error_code, a.staging_key, a.pipeline_version +FROM attachments AS a +JOIN chat_members AS cm ON cm.chat_id = a.chat_id +WHERE a.id = $1 + AND cm.user_id = $2 +` + +type GetAttachmentForUserParams struct { + AttachmentID pgtype.UUID + UserID pgtype.UUID +} + +func (q *Queries) GetAttachmentForUser(ctx context.Context, arg GetAttachmentForUserParams) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachmentForUser, arg.AttachmentID, arg.UserID) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const getAttachmentsForMessages = `-- name: GetAttachmentsForMessages :many +SELECT message_id, attachment_id, position +FROM messages_attachments +WHERE message_id = ANY($1::uuid[]) +ORDER BY message_id, position +` + +type GetAttachmentsForMessagesRow struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + Position int16 +} + +func (q *Queries) GetAttachmentsForMessages(ctx context.Context, messageIds []pgtype.UUID) ([]GetAttachmentsForMessagesRow, error) { + rows, err := q.db.Query(ctx, getAttachmentsForMessages, messageIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetAttachmentsForMessagesRow + for rows.Next() { + var i GetAttachmentsForMessagesRow + if err := rows.Scan(&i.MessageID, &i.AttachmentID, &i.Position); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listAttachmentKeys = `-- name: ListAttachmentKeys :many +SELECT id, object_key, created_at +FROM attachments +WHERE object_key > $1 +ORDER BY object_key +LIMIT $2 +` + +type ListAttachmentKeysParams struct { + AfterKey pgtype.Text + PageSize int32 +} + +type ListAttachmentKeysRow struct { + ID pgtype.UUID + ObjectKey pgtype.Text + CreatedAt pgtype.Timestamptz +} + +func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) { + rows, err := q.db.Query(ctx, listAttachmentKeys, arg.AfterKey, arg.PageSize) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListAttachmentKeysRow + for rows.Next() { + var i ListAttachmentKeysRow + if err := rows.Scan(&i.ID, &i.ObjectKey, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const tryLockAttachmentObjectKey = `-- name: TryLockAttachmentObjectKey :one +SELECT pg_try_advisory_xact_lock( + hashtextextended($1::text, 0) +)::boolean AS locked +` + +func (q *Queries) TryLockAttachmentObjectKey(ctx context.Context, objectKey string) (bool, error) { + row := q.db.QueryRow(ctx, tryLockAttachmentObjectKey, objectKey) + var locked bool + err := row.Scan(&locked) + return locked, err +} diff --git a/internal/database/sqlc/attachments_jobs.sql.go b/internal/database/sqlc/attachments_jobs.sql.go new file mode 100644 index 00000000..859de6bc --- /dev/null +++ b/internal/database/sqlc/attachments_jobs.sql.go @@ -0,0 +1,529 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachments_jobs.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const claimJob = `-- name: ClaimJob :one +WITH claimed AS ( + UPDATE attachment_jobs AS j + SET lease_generation = j.lease_generation + 1, + lease_until = $1::timestamptz, + attempts_started = j.attempts_started + 1, + next_attempt_at = NULL, + updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = $2 + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + AND j.attempts_started < j.max_attempts + AND $1::timestamptz > clock_timestamp() + AND ( + (a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp()) + OR (a.status = 'processing' AND j.lease_until <= clock_timestamp()) + ) + RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +), expired AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at + FROM claimed + WHERE attempt.job_id = claimed.id + AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL +), recorded AS ( + INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at) + SELECT id, lease_generation, attempts_started, updated_at FROM claimed + RETURNING job_id +), processing AS ( + UPDATE attachments AS a + SET status = 'processing', updated_at = claimed.updated_at + FROM claimed + WHERE a.id = claimed.attachment_id + RETURNING a.id +) +SELECT claimed.id, claimed.attachment_id, claimed.pipeline_version, claimed.attempts_started, claimed.max_attempts, claimed.next_attempt_at, claimed.lease_generation, claimed.lease_until, claimed.created_at, claimed.updated_at, claimed.published_original_key FROM claimed JOIN recorded ON recorded.job_id = claimed.id +JOIN processing ON processing.id = claimed.attachment_id +` + +type ClaimJobParams struct { + NewLeaseUntil pgtype.Timestamptz + JobID pgtype.UUID +} + +type ClaimJobRow struct { + ID pgtype.UUID + AttachmentID pgtype.UUID + PipelineVersion int32 + AttemptsStarted int32 + MaxAttempts int32 + NextAttemptAt pgtype.Timestamptz + LeaseGeneration int32 + LeaseUntil pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + PublishedOriginalKey pgtype.Text +} + +func (q *Queries) ClaimJob(ctx context.Context, arg ClaimJobParams) (ClaimJobRow, error) { + row := q.db.QueryRow(ctx, claimJob, arg.NewLeaseUntil, arg.JobID) + var i ClaimJobRow + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.CreatedAt, + &i.UpdatedAt, + &i.PublishedOriginalKey, + ) + return i, err +} + +const clearStagingReference = `-- name: ClearStagingReference :execrows +UPDATE attachments +SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END, + staging_key = NULL +WHERE id = $1 AND staging_key = $2 +` + +type ClearStagingReferenceParams struct { + ID pgtype.UUID + StagingKey pgtype.Text +} + +func (q *Queries) ClearStagingReference(ctx context.Context, arg ClearStagingReferenceParams) (int64, error) { + result, err := q.db.Exec(ctx, clearStagingReference, arg.ID, arg.StagingKey) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const completeJob = `-- name: CompleteJob :execrows +WITH completed AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(), + published_original_key = $1::text + FROM attachments AS a + WHERE j.id = $2 + AND j.lease_generation = $3 AND j.lease_until > clock_timestamp() + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + AND (SELECT count(*) FROM attachment_variants AS v + WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3 + RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'ready', finished_at = completed.updated_at + FROM completed + WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation +) +UPDATE attachments AS a +SET status = 'ready', object_key = $1::text, updated_at = completed.updated_at +FROM completed +WHERE a.id = completed.attachment_id +` + +type CompleteJobParams struct { + ObjectKey string + JobID pgtype.UUID + LeaseGeneration int32 +} + +func (q *Queries) CompleteJob(ctx context.Context, arg CompleteJobParams) (int64, error) { + result, err := q.db.Exec(ctx, completeJob, arg.ObjectKey, arg.JobID, arg.LeaseGeneration) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const createAttachmentJob = `-- name: CreateAttachmentJob :one +INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts) +VALUES ($1, $2, + $3::timestamptz, $4) +RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key +` + +type CreateAttachmentJobParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 + AvailableAt pgtype.Timestamptz + MaxAttempts int32 +} + +func (q *Queries) CreateAttachmentJob(ctx context.Context, arg CreateAttachmentJobParams) (AttachmentJob, error) { + row := q.db.QueryRow(ctx, createAttachmentJob, + arg.AttachmentID, + arg.PipelineVersion, + arg.AvailableAt, + arg.MaxAttempts, + ) + var i AttachmentJob + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.CreatedAt, + &i.UpdatedAt, + &i.PublishedOriginalKey, + ) + return i, err +} + +const failJob = `-- name: FailJob :execrows +WITH failed AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = $2 + AND j.lease_generation = $3 + AND ( + j.lease_until > clock_timestamp() + OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts + AND $1::text = 'retry_exhausted') + ) + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'failed', error_code = $1::text, finished_at = failed.updated_at + FROM failed + WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation +) +UPDATE attachments AS a +SET status = 'failed', error_code = $1::text, updated_at = failed.updated_at +FROM failed WHERE a.id = failed.attachment_id +` + +type FailJobParams struct { + ErrorCode string + JobID pgtype.UUID + LeaseGeneration int32 +} + +func (q *Queries) FailJob(ctx context.Context, arg FailJobParams) (int64, error) { + result, err := q.db.Exec(ctx, failJob, arg.ErrorCode, arg.JobID, arg.LeaseGeneration) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getAttachmentByStagingKey = `-- name: GetAttachmentByStagingKey :one +SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE staging_key = $1::text +` + +func (q *Queries) GetAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachmentByStagingKey, stagingKey) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const getAttachmentJobState = `-- name: GetAttachmentJobState :one +SELECT + j.id, j.attachment_id, j.pipeline_version, + CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status, + j.lease_generation, + j.lease_until, + coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired, + j.attempts_started, + j.max_attempts, + j.next_attempt_at +FROM attachment_jobs AS j +JOIN attachments AS a ON a.id = j.attachment_id +WHERE j.id = $1::uuid +` + +type GetAttachmentJobStateRow struct { + ID pgtype.UUID + AttachmentID pgtype.UUID + PipelineVersion int32 + Status string + LeaseGeneration int32 + LeaseUntil pgtype.Timestamptz + LeaseExpired bool + AttemptsStarted int32 + MaxAttempts int32 + NextAttemptAt pgtype.Timestamptz +} + +func (q *Queries) GetAttachmentJobState(ctx context.Context, jobID pgtype.UUID) (GetAttachmentJobStateRow, error) { + row := q.db.QueryRow(ctx, getAttachmentJobState, jobID) + var i GetAttachmentJobStateRow + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.Status, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.LeaseExpired, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + ) + return i, err +} + +const listFailedAttachmentJobs = `-- name: ListFailedAttachmentJobs :many +SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at +FROM attachment_jobs j +JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version +WHERE a.status = 'failed' +ORDER BY a.updated_at DESC, j.id +LIMIT $1 +` + +type ListFailedAttachmentJobsRow struct { + ID pgtype.UUID + AttachmentID pgtype.UUID + AttemptsStarted int32 + ErrorCode pgtype.Text + UpdatedAt pgtype.Timestamptz +} + +func (q *Queries) ListFailedAttachmentJobs(ctx context.Context, batchSize int32) ([]ListFailedAttachmentJobsRow, error) { + rows, err := q.db.Query(ctx, listFailedAttachmentJobs, batchSize) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListFailedAttachmentJobsRow + for rows.Next() { + var i ListFailedAttachmentJobsRow + if err := rows.Scan( + &i.ID, + &i.AttachmentID, + &i.AttemptsStarted, + &i.ErrorCode, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockActiveAttachmentJob = `-- name: LockActiveAttachmentJob :one +SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id +WHERE j.id = $1 + AND j.lease_generation = $2 + AND j.lease_until > clock_timestamp() + AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version +FOR UPDATE OF j +` + +type LockActiveAttachmentJobParams struct { + JobID pgtype.UUID + LeaseGeneration int32 +} + +// Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion. +func (q *Queries) LockActiveAttachmentJob(ctx context.Context, arg LockActiveAttachmentJobParams) (AttachmentJob, error) { + row := q.db.QueryRow(ctx, lockActiveAttachmentJob, arg.JobID, arg.LeaseGeneration) + var i AttachmentJob + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.CreatedAt, + &i.UpdatedAt, + &i.PublishedOriginalKey, + ) + return i, err +} + +const lockAttachmentJobForPipeline = `-- name: LockAttachmentJobForPipeline :one +SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +FROM attachment_jobs j +WHERE j.attachment_id = $1 AND j.pipeline_version = $2 +FOR UPDATE +` + +type LockAttachmentJobForPipelineParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 +} + +func (q *Queries) LockAttachmentJobForPipeline(ctx context.Context, arg LockAttachmentJobForPipelineParams) (AttachmentJob, error) { + row := q.db.QueryRow(ctx, lockAttachmentJobForPipeline, arg.AttachmentID, arg.PipelineVersion) + var i AttachmentJob + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.CreatedAt, + &i.UpdatedAt, + &i.PublishedOriginalKey, + ) + return i, err +} + +const lockAttachmentJobForReplay = `-- name: LockAttachmentJobForReplay :one +SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +FROM attachment_jobs j WHERE j.id = $1 +FOR UPDATE +` + +func (q *Queries) LockAttachmentJobForReplay(ctx context.Context, id pgtype.UUID) (AttachmentJob, error) { + row := q.db.QueryRow(ctx, lockAttachmentJobForReplay, id) + var i AttachmentJob + err := row.Scan( + &i.ID, + &i.AttachmentID, + &i.PipelineVersion, + &i.AttemptsStarted, + &i.MaxAttempts, + &i.NextAttemptAt, + &i.LeaseGeneration, + &i.LeaseUntil, + &i.CreatedAt, + &i.UpdatedAt, + &i.PublishedOriginalKey, + ) + return i, err +} + +const replayFailed = `-- name: ReplayFailed :execrows +WITH replayed AS ( + UPDATE attachment_jobs AS j + SET max_attempts = j.attempts_started + $1::int, + next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = $2 + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + AND a.status = 'failed' AND a.staging_key IS NOT NULL + AND j.lease_until IS NULL AND j.next_attempt_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM attachment_replays AS r WHERE r.request_id = $3::uuid + ) + RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +), recorded AS ( + INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts) + SELECT $3::uuid, id, $4, $5, $1 + FROM replayed +), event AS ( + INSERT INTO outbox (job_id, payload, available_at, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at), + next_attempt_at, 'replay/' || id || '/' || $3::uuid + FROM replayed +) +UPDATE attachments AS a +SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at +FROM replayed WHERE a.id = replayed.attachment_id +` + +type ReplayFailedParams struct { + ExtraAttempts int32 + JobID pgtype.UUID + RequestID pgtype.UUID + OperatorID string + Reason string +} + +func (q *Queries) ReplayFailed(ctx context.Context, arg ReplayFailedParams) (int64, error) { + result, err := q.db.Exec(ctx, replayFailed, + arg.ExtraAttempts, + arg.JobID, + arg.RequestID, + arg.OperatorID, + arg.Reason, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const scheduleRetry = `-- name: ScheduleRetry :execrows +WITH scheduled AS ( + UPDATE attachment_jobs AS j + SET lease_until = NULL, next_attempt_at = $1::timestamptz, + updated_at = clock_timestamp() + FROM attachments AS a + WHERE j.id = $2 + AND j.lease_generation = $3 AND j.lease_until > clock_timestamp() + AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + AND j.attempts_started < j.max_attempts + AND $1::timestamptz > clock_timestamp() + RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key +), recorded AS ( + UPDATE attachment_attempts AS attempt + SET outcome = 'retry', error_code = $4::text, finished_at = scheduled.updated_at + FROM scheduled + WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation +), event AS ( + INSERT INTO outbox (job_id, payload, available_at, dedup_key) + SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at), + next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1) + FROM scheduled +) +UPDATE attachments AS a +SET status = 'queued', updated_at = scheduled.updated_at +FROM scheduled WHERE a.id = scheduled.attachment_id +` + +type ScheduleRetryParams struct { + AvailableAt pgtype.Timestamptz + JobID pgtype.UUID + LeaseGeneration int32 + ErrorCode string +} + +func (q *Queries) ScheduleRetry(ctx context.Context, arg ScheduleRetryParams) (int64, error) { + result, err := q.db.Exec(ctx, scheduleRetry, + arg.AvailableAt, + arg.JobID, + arg.LeaseGeneration, + arg.ErrorCode, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/database/sqlc/chat_members.sql.go b/internal/database/sqlc/chat_members.sql.go new file mode 100644 index 00000000..10326c9c --- /dev/null +++ b/internal/database/sqlc/chat_members.sql.go @@ -0,0 +1,27 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: chat_members.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createChatMember = `-- name: CreateChatMember :exec +INSERT INTO chat_members(chat_id, user_id) +VALUES ($1, $2) +` + +type CreateChatMemberParams struct { + ChatID pgtype.UUID + UserID pgtype.UUID +} + +func (q *Queries) CreateChatMember(ctx context.Context, arg CreateChatMemberParams) error { + _, err := q.db.Exec(ctx, createChatMember, arg.ChatID, arg.UserID) + return err +} diff --git a/internal/database/sqlc/chats.sql.go b/internal/database/sqlc/chats.sql.go new file mode 100644 index 00000000..ce96287a --- /dev/null +++ b/internal/database/sqlc/chats.sql.go @@ -0,0 +1,118 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: chats.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createChat = `-- name: CreateChat :one +INSERT INTO chats (title) +VALUES ($1) +RETURNING id, title, created_at +` + +func (q *Queries) CreateChat(ctx context.Context, title string) (Chat, error) { + row := q.db.QueryRow(ctx, createChat, title) + var i Chat + err := row.Scan(&i.ID, &i.Title, &i.CreatedAt) + return i, err +} + +const getChatByID = `-- name: GetChatByID :one + +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE c.id = $1 +GROUP BY c.id +` + +type GetChatByIDRow struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz + MemberIds []pgtype.UUID +} + +func (q *Queries) GetChatByID(ctx context.Context, id pgtype.UUID) (GetChatByIDRow, error) { + row := q.db.QueryRow(ctx, getChatByID, id) + var i GetChatByIDRow + err := row.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.MemberIds, + ) + return i, err +} + +const getChatsByUserID = `-- name: GetChatsByUserID :many +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE EXISTS (SELECT 1 + FROM chat_members membership + WHERE membership.chat_id = c.id + AND membership.user_id = $1) + AND ( + $2::timestamptz IS NULL + OR (c.created_at, c.id) > + ( + $2::timestamptz, + $3::uuid + ) + ) +GROUP BY c.id +ORDER BY c.created_at ASC, c.id ASC +LIMIT $4 +` + +type GetChatsByUserIDParams struct { + UserID pgtype.UUID + AfterCreatedAt pgtype.Timestamptz + AfterID pgtype.UUID + LimitCount int32 +} + +type GetChatsByUserIDRow struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz + MemberIds []pgtype.UUID +} + +func (q *Queries) GetChatsByUserID(ctx context.Context, arg GetChatsByUserIDParams) ([]GetChatsByUserIDRow, error) { + rows, err := q.db.Query(ctx, getChatsByUserID, + arg.UserID, + arg.AfterCreatedAt, + arg.AfterID, + arg.LimitCount, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetChatsByUserIDRow + for rows.Next() { + var i GetChatsByUserIDRow + if err := rows.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.MemberIds, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/db.go b/internal/database/sqlc/db.go new file mode 100644 index 00000000..a28f6fc5 --- /dev/null +++ b/internal/database/sqlc/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/database/sqlc/message_idempotence.sql.go b/internal/database/sqlc/message_idempotence.sql.go new file mode 100644 index 00000000..20c68aa7 --- /dev/null +++ b/internal/database/sqlc/message_idempotence.sql.go @@ -0,0 +1,99 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: message_idempotence.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteExpiredMessageIdempotency = `-- name: DeleteExpiredMessageIdempotency :exec +DELETE FROM message_idempotency +WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now() +` + +type DeleteExpiredMessageIdempotencyParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string +} + +func (q *Queries) DeleteExpiredMessageIdempotency(ctx context.Context, arg DeleteExpiredMessageIdempotencyParams) error { + _, err := q.db.Exec(ctx, deleteExpiredMessageIdempotency, arg.UserID, arg.ChatID, arg.IdempotencyKey) + return err +} + +const getIdempotentMessage = `-- name: GetIdempotentMessage :one +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at +FROM message_idempotency AS i +JOIN messages AS m ON m.id = i.message_id +WHERE i.user_id = $1 + AND i.chat_id = $2 + AND i.idempotency_key = $3 +` + +type GetIdempotentMessageParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string +} + +func (q *Queries) GetIdempotentMessage(ctx context.Context, arg GetIdempotentMessageParams) (Message, error) { + row := q.db.QueryRow(ctx, getIdempotentMessage, arg.UserID, arg.ChatID, arg.IdempotencyKey) + var i Message + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ) + return i, err +} + +const insertMessageIdempotency = `-- name: InsertMessageIdempotency :execrows +INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id) +VALUES ($1, $2, $3, $4) +ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING +` + +type InsertMessageIdempotencyParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string + MessageID pgtype.UUID +} + +func (q *Queries) InsertMessageIdempotency(ctx context.Context, arg InsertMessageIdempotencyParams) (int64, error) { + result, err := q.db.Exec(ctx, insertMessageIdempotency, + arg.UserID, + arg.ChatID, + arg.IdempotencyKey, + arg.MessageID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const lockMessageRequest = `-- name: LockMessageRequest :exec +SELECT pg_advisory_xact_lock(hashtextextended( + 'message/' || ($1::uuid)::text || '/' || + ($2::uuid)::text || '/' || $3::text, 0)) +` + +type LockMessageRequestParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string +} + +func (q *Queries) LockMessageRequest(ctx context.Context, arg LockMessageRequestParams) error { + _, err := q.db.Exec(ctx, lockMessageRequest, arg.UserID, arg.ChatID, arg.IdempotencyKey) + return err +} diff --git a/internal/database/sqlc/messages.sql.go b/internal/database/sqlc/messages.sql.go new file mode 100644 index 00000000..aa11ba59 --- /dev/null +++ b/internal/database/sqlc/messages.sql.go @@ -0,0 +1,91 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: messages.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createMessage = `-- name: CreateMessage :one +INSERT INTO messages (chat_id, sender_id, text) +VALUES ($1, $2, $3) +RETURNING id, chat_id, sender_id, text, created_at +` + +type CreateMessageParams struct { + ChatID pgtype.UUID + SenderID pgtype.UUID + Text string +} + +func (q *Queries) CreateMessage(ctx context.Context, arg CreateMessageParams) (Message, error) { + row := q.db.QueryRow(ctx, createMessage, arg.ChatID, arg.SenderID, arg.Text) + var i Message + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ) + return i, err +} + +const getMessagesByChatID = `-- name: GetMessagesByChatID :many + +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 + AND ( + $2::timestamptz IS NULL + OR (created_at, id) > + ($2::timestamptz, + $3::uuid + ) + ) +ORDER BY created_at ASC, id ASC +LIMIT $4 +` + +type GetMessagesByChatIDParams struct { + ChatID pgtype.UUID + AfterCreatedAt pgtype.Timestamptz + AfterID pgtype.UUID + LimitCount int32 +} + +func (q *Queries) GetMessagesByChatID(ctx context.Context, arg GetMessagesByChatIDParams) ([]Message, error) { + rows, err := q.db.Query(ctx, getMessagesByChatID, + arg.ChatID, + arg.AfterCreatedAt, + arg.AfterID, + arg.LimitCount, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Message + for rows.Next() { + var i Message + if err := rows.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/messages_attachments.sql.go b/internal/database/sqlc/messages_attachments.sql.go new file mode 100644 index 00000000..2a5b1b00 --- /dev/null +++ b/internal/database/sqlc/messages_attachments.sql.go @@ -0,0 +1,107 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: messages_attachments.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createMessagesAttachments = `-- name: CreateMessagesAttachments :execrows +INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) +SELECT $1, $2, $3, $4 +FROM attachments +WHERE id = $2 AND chat_id = $3 +` + +type CreateMessagesAttachmentsParams struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + ChatID pgtype.UUID + Position int16 +} + +func (q *Queries) CreateMessagesAttachments(ctx context.Context, arg CreateMessagesAttachmentsParams) (int64, error) { + result, err := q.db.Exec(ctx, createMessagesAttachments, + arg.MessageID, + arg.AttachmentID, + arg.ChatID, + arg.Position, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getMessagesAttachments = `-- name: GetMessagesAttachments :many +SELECT attachment_id, position +FROM messages_attachments +WHERE message_id = $1 +ORDER BY position ASC +` + +type GetMessagesAttachmentsRow struct { + AttachmentID pgtype.UUID + Position int16 +} + +func (q *Queries) GetMessagesAttachments(ctx context.Context, messageID pgtype.UUID) ([]GetMessagesAttachmentsRow, error) { + rows, err := q.db.Query(ctx, getMessagesAttachments, messageID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetMessagesAttachmentsRow + for rows.Next() { + var i GetMessagesAttachmentsRow + if err := rows.Scan(&i.AttachmentID, &i.Position); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockAttachmentsForMessage = `-- name: LockAttachmentsForMessage :many +SELECT id, status FROM attachments +WHERE chat_id = $1 AND id = ANY($2::uuid[]) +ORDER BY id FOR SHARE +` + +type LockAttachmentsForMessageParams struct { + ChatID pgtype.UUID + AttachmentIds []pgtype.UUID +} + +type LockAttachmentsForMessageRow struct { + ID pgtype.UUID + Status string +} + +func (q *Queries) LockAttachmentsForMessage(ctx context.Context, arg LockAttachmentsForMessageParams) ([]LockAttachmentsForMessageRow, error) { + rows, err := q.db.Query(ctx, lockAttachmentsForMessage, arg.ChatID, arg.AttachmentIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []LockAttachmentsForMessageRow + for rows.Next() { + var i LockAttachmentsForMessageRow + if err := rows.Scan(&i.ID, &i.Status); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/models.go b/internal/database/sqlc/models.go new file mode 100644 index 00000000..0932dacb --- /dev/null +++ b/internal/database/sqlc/models.go @@ -0,0 +1,124 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package sqlc + +import ( + "github.com/jackc/pgx/v5/pgtype" +) + +type Attachment struct { + ID pgtype.UUID + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ObjectKey pgtype.Text + ContentType string + SizeBytes int64 + Sha256 string + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + Status string + ErrorCode pgtype.Text + StagingKey pgtype.Text + PipelineVersion int32 +} + +type AttachmentAttempt struct { + JobID pgtype.UUID + LeaseGeneration int32 + Attempt int32 + StartedAt pgtype.Timestamptz + FinishedAt pgtype.Timestamptz + Outcome pgtype.Text + ErrorCode pgtype.Text +} + +type AttachmentJob struct { + ID pgtype.UUID + AttachmentID pgtype.UUID + PipelineVersion int32 + AttemptsStarted int32 + MaxAttempts int32 + NextAttemptAt pgtype.Timestamptz + LeaseGeneration int32 + LeaseUntil pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz + PublishedOriginalKey pgtype.Text +} + +type AttachmentReplay struct { + RequestID pgtype.UUID + JobID pgtype.UUID + OperatorID string + Reason string + ExtraAttempts int32 + CreatedAt pgtype.Timestamptz +} + +type AttachmentVariant struct { + AttachmentID pgtype.UUID + PipelineVersion int32 + Name string + ObjectKey string + ContentType string + SizeBytes int64 + Width int32 + Height int32 + Sha256 string +} + +type Chat struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz +} + +type ChatMember struct { + ChatID pgtype.UUID + UserID pgtype.UUID +} + +type Message struct { + ID pgtype.UUID + ChatID pgtype.UUID + SenderID pgtype.UUID + Text string + CreatedAt pgtype.Timestamptz +} + +type MessageIdempotency struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string + MessageID pgtype.UUID + CreatedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz +} + +type MessagesAttachment struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + ChatID pgtype.UUID + Position int16 +} + +type Outbox struct { + ID pgtype.UUID + JobID pgtype.UUID + Payload []byte + AvailableAt pgtype.Timestamptz + SentAt pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + DedupKey string +} + +type User struct { + ID pgtype.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt pgtype.Timestamptz +} diff --git a/internal/database/sqlc/outbox.sql.go b/internal/database/sqlc/outbox.sql.go new file mode 100644 index 00000000..1220670e --- /dev/null +++ b/internal/database/sqlc/outbox.sql.go @@ -0,0 +1,111 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: outbox.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createInitialOutboxEvent = `-- name: CreateInitialOutboxEvent :exec +INSERT INTO outbox (job_id, payload, available_at, dedup_key) +SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at), + next_attempt_at, 'initial/' || id +FROM attachment_jobs WHERE id = $1::uuid +` + +func (q *Queries) CreateInitialOutboxEvent(ctx context.Context, jobID pgtype.UUID) error { + _, err := q.db.Exec(ctx, createInitialOutboxEvent, jobID) + return err +} + +const enqueueExpiredLeases = `-- name: EnqueueExpiredLeases :many +WITH expired AS ( + SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key + FROM attachment_jobs AS j + JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp() + AND NOT EXISTS ( + SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation + ) + ORDER BY j.lease_until, j.id + LIMIT $1 + FOR UPDATE OF j SKIP LOCKED +) +INSERT INTO outbox (job_id, payload, dedup_key) +SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, + LEAST(attempts_started + 1, max_attempts), clock_timestamp()), + recovery_key +FROM expired +ON CONFLICT (dedup_key) DO NOTHING +RETURNING id, job_id, payload, available_at, sent_at, created_at, dedup_key +` + +func (q *Queries) EnqueueExpiredLeases(ctx context.Context, batchSize int32) ([]Outbox, error) { + rows, err := q.db.Query(ctx, enqueueExpiredLeases, batchSize) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Outbox + for rows.Next() { + var i Outbox + if err := rows.Scan( + &i.ID, + &i.JobID, + &i.Payload, + &i.AvailableAt, + &i.SentAt, + &i.CreatedAt, + &i.DedupKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markOutboxEventSent = `-- name: MarkOutboxEventSent :execrows +UPDATE outbox SET sent_at = clock_timestamp() +WHERE id = $1 AND sent_at IS NULL +` + +func (q *Queries) MarkOutboxEventSent(ctx context.Context, id pgtype.UUID) (int64, error) { + result, err := q.db.Exec(ctx, markOutboxEventSent, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const nextPendingOutboxEvent = `-- name: NextPendingOutboxEvent :one +SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key +FROM outbox +WHERE sent_at IS NULL AND available_at <= statement_timestamp() +ORDER BY available_at, id +LIMIT 1 +FOR UPDATE SKIP LOCKED +` + +func (q *Queries) NextPendingOutboxEvent(ctx context.Context) (Outbox, error) { + row := q.db.QueryRow(ctx, nextPendingOutboxEvent) + var i Outbox + err := row.Scan( + &i.ID, + &i.JobID, + &i.Payload, + &i.AvailableAt, + &i.SentAt, + &i.CreatedAt, + &i.DedupKey, + ) + return i, err +} diff --git a/internal/database/sqlc/users.sql.go b/internal/database/sqlc/users.sql.go new file mode 100644 index 00000000..6e423e55 --- /dev/null +++ b/internal/database/sqlc/users.sql.go @@ -0,0 +1,77 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: users.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createUser = `-- name: CreateUser :one + +INSERT INTO users (username, display_name, password_hash) +VALUES ($1, $2, $3) +RETURNING id, username, display_name, password_hash, created_at +` + +type CreateUserParams struct { + Username string + DisplayName string + PasswordHash string +} + +func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) { + row := q.db.QueryRow(ctx, createUser, arg.Username, arg.DisplayName, arg.PasswordHash) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} + +const getUserByID = `-- name: GetUserByID :one + +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE id = $1 +` + +func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error) { + row := q.db.QueryRow(ctx, getUserByID, id) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} + +const getUserByUsername = `-- name: GetUserByUsername :one +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE username = $1 +` + +func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User, error) { + row := q.db.QueryRow(ctx, getUserByUsername, username) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} diff --git a/internal/database/sqlc/versioned_pipeline.sql.go b/internal/database/sqlc/versioned_pipeline.sql.go new file mode 100644 index 00000000..3ffc3651 --- /dev/null +++ b/internal/database/sqlc/versioned_pipeline.sql.go @@ -0,0 +1,111 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: versioned_pipeline.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getPipelineJobID = `-- name: GetPipelineJobID :one +SELECT id FROM attachment_jobs +WHERE attachment_id = $1::uuid AND pipeline_version = $2::int +` + +type GetPipelineJobIDParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 +} + +func (q *Queries) GetPipelineJobID(ctx context.Context, arg GetPipelineJobIDParams) (pgtype.UUID, error) { + row := q.db.QueryRow(ctx, getPipelineJobID, arg.AttachmentID, arg.PipelineVersion) + var id pgtype.UUID + err := row.Scan(&id) + return id, err +} + +const getPublishedOriginalKey = `-- name: GetPublishedOriginalKey :one +SELECT published_original_key FROM attachment_jobs +WHERE attachment_id = $1::uuid AND pipeline_version = $2::int + AND published_original_key IS NOT NULL +` + +type GetPublishedOriginalKeyParams struct { + AttachmentID pgtype.UUID + PipelineVersion int32 +} + +func (q *Queries) GetPublishedOriginalKey(ctx context.Context, arg GetPublishedOriginalKeyParams) (pgtype.Text, error) { + row := q.db.QueryRow(ctx, getPublishedOriginalKey, arg.AttachmentID, arg.PipelineVersion) + var published_original_key pgtype.Text + err := row.Scan(&published_original_key) + return published_original_key, err +} + +const isPublishedOriginalKey = `-- name: IsPublishedOriginalKey :one +SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text) +` + +func (q *Queries) IsPublishedOriginalKey(ctx context.Context, objectKey string) (bool, error) { + row := q.db.QueryRow(ctx, isPublishedOriginalKey, objectKey) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const queueAttachmentUpgrade = `-- name: QueueAttachmentUpgrade :one +UPDATE attachments +SET pipeline_version = 2, status = 'queued', staging_key = $1::text, + object_key = NULL, error_code = NULL, updated_at = clock_timestamp() +WHERE id = $2::uuid AND pipeline_version = 1 +RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version +` + +type QueueAttachmentUpgradeParams struct { + SourceKey string + ID pgtype.UUID +} + +func (q *Queries) QueueAttachmentUpgrade(ctx context.Context, arg QueueAttachmentUpgradeParams) (Attachment, error) { + row := q.db.QueryRow(ctx, queueAttachmentUpgrade, arg.SourceKey, arg.ID) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.ErrorCode, + &i.StagingKey, + &i.PipelineVersion, + ) + return i, err +} + +const supersedeAttachmentJob = `-- name: SupersedeAttachmentJob :exec +WITH superseded AS ( + UPDATE attachment_jobs + SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + WHERE id = $1::uuid + RETURNING id, updated_at +) +UPDATE attachment_attempts AS attempt +SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at +FROM superseded +WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL +` + +func (q *Queries) SupersedeAttachmentJob(ctx context.Context, jobID pgtype.UUID) error { + _, err := q.db.Exec(ctx, supersedeAttachmentJob, jobID) + return err +} diff --git a/internal/fault/pause.go b/internal/fault/pause.go new file mode 100644 index 00000000..9e784a9b --- /dev/null +++ b/internal/fault/pause.go @@ -0,0 +1,43 @@ +// Package fault provides one-shot pause points for an isolated lab Compose project. +package fault + +import ( + "context" + "errors" + "os" + "path/filepath" + "time" +) + +func Pause(ctx context.Context, point, id string) error { + dir := os.Getenv("LAB5_FAULT_DIR") + if dir == "" { + return nil + } + base := filepath.Join(dir, point) + if err := os.Rename(base+".arm", base+".used"); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if err := os.WriteFile(base+".hit", []byte(id+"\n"), 0644); err != nil { + return err + } + wait, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + poll := time.NewTicker(50 * time.Millisecond) + defer poll.Stop() + for { + if _, err := os.Stat(base + ".release"); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + select { + case <-wait.Done(): + return wait.Err() + case <-poll.C: + } + } +} diff --git a/internal/imageproc/processor.go b/internal/imageproc/processor.go new file mode 100644 index 00000000..3ad30daa --- /dev/null +++ b/internal/imageproc/processor.go @@ -0,0 +1,164 @@ +package imageproc + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "highload/internal/model" +) + +const ( + MaxBytes = 10 * 1024 * 1024 + MaxPixels = 40_000_000 +) + +var ( + ErrInvalidImage = errors.New("invalid image") + ErrResourceLimit = errors.New("image processing resource limit") +) + +type File struct { + Path string + Variant model.AttachmentVariant +} + +type variantSpec struct { + name, extension, mime string + width, height int + args []string +} + +func Generate(ctx context.Context, dir, input, mime string, version int32) ([]File, error) { + var jpegQuality, webpQuality string + switch version { + case 1: + jpegQuality, webpQuality = "85", "80" + case 2: + jpegQuality, webpQuality = "90", "72" + default: + return nil, fmt.Errorf("unsupported image pipeline version %d", version) + } + f, err := os.Open(input) + if err != nil { + return nil, err + } + cfg, format, decodeErr := image.DecodeConfig(io.LimitReader(f, MaxBytes)) + _ = f.Close() + if decodeErr != nil || (format != "png" && format != "jpeg") || mime != "image/"+format || + cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > MaxPixels/cfg.Height { + return nil, ErrInvalidImage + } + // Paths are generated on the server; input format is independently verified. + source := format + ":" + input + orient, err := magick(ctx, "identify", "-ping", "-format", "%[orientation]", source) + if err != nil { + return nil, err + } + + w, h := cfg.Width, cfg.Height + switch strings.TrimSpace(orient) { + case "LeftTop", "RightTop", "RightBottom", "LeftBottom": + w, h = h, w + } + tw, th := fit(w, h, 256) + ow, oh := fit(w, h, 1280) + side := min(w, h) + specs := []variantSpec{ + {name: "thumbnail", extension: "jpg", mime: "image/jpeg", width: tw, height: th, + args: []string{"-resize", geometry(tw, th), "-quality", jpegQuality}}, + {name: "crop", extension: "jpg", mime: "image/jpeg", width: 128, height: 128, + args: []string{"-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage", "-resize", "128x128!", "-quality", jpegQuality}}, + {name: "optimized", extension: "webp", mime: "image/webp", width: ow, height: oh, + args: []string{"-resize", geometry(ow, oh), "-quality", webpQuality, "-define", "webp:method=4"}}, + } + files := make([]File, 0, len(specs)) + for _, spec := range specs { + out := filepath.Join(dir, spec.name+"."+spec.extension) + args := []string{"-regard-warnings", source, "-auto-orient"} + if spec.mime == "image/jpeg" { + args = append(args, "-background", "white", "-alpha", "remove", "-alpha", "off") + } + args = append(args, spec.args...) + args = append(args, "-strip", out) + if _, err := magick(ctx, args...); err != nil { + return nil, err + } + meta, err := inspectFile(ctx, out, spec) + if err != nil { + return nil, err + } + files = append(files, File{Path: out, Variant: meta}) + } + return files, nil +} + +func fit(w, h, bound int) (int, int) { + if max(w, h) <= bound { + return w, h + } + if w >= h { + return bound, max(1, h*bound/w) + } + return max(1, w*bound/h), bound +} + +func geometry(w, h int) string { + return fmt.Sprintf("%dx%d!", w, h) +} + +func inspectFile(ctx context.Context, path string, spec variantSpec) (model.AttachmentVariant, error) { + var meta model.AttachmentVariant + desc, err := magick(ctx, "identify", "-ping", "-format", "%m %w %h", path) + if err != nil { + return meta, err + } + format := strings.ToUpper(strings.TrimPrefix(spec.mime, "image/")) + if strings.TrimSpace(desc) != format+" "+strconv.Itoa(spec.width)+" "+strconv.Itoa(spec.height) { + return meta, fmt.Errorf("generated variant has unexpected format or size") + } + f, err := os.Open(path) + if err != nil { + return meta, err + } + defer f.Close() + sum := sha256.New() + size, err := io.Copy(sum, f) + if err != nil { + return meta, err + } + return model.AttachmentVariant{Name: spec.name, ContentType: spec.mime, Width: spec.width, Height: spec.height, + SizeBytes: size, SHA256: hex.EncodeToString(sum.Sum(nil))}, nil +} + +func magick(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "magick", args...) + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + return "", ctx.Err() + } + if err == nil { + return string(out), nil + } + message := strings.ToLower(string(out)) + if strings.Contains(message, "cache resources exhausted") || strings.Contains(message, "time limit") || + strings.Contains(message, "memory allocation") || strings.Contains(message, "list length exceeds") { + return "", ErrResourceLimit + } + if strings.Contains(message, "corrupt") || strings.Contains(message, "unexpected end") || + strings.Contains(message, "insufficient image data") || strings.Contains(message, "decode") { + return "", ErrInvalidImage + } + return "", fmt.Errorf("ImageMagick command failed: %w", err) +} diff --git a/internal/model/attachment.go b/internal/model/attachment.go new file mode 100644 index 00000000..71dea71c --- /dev/null +++ b/internal/model/attachment.go @@ -0,0 +1,31 @@ +package model + +import "time" + +type Attachment struct { + ID string + ChatID string + OwnerID string + Filename string + ObjectKey string + StagingKey string + ErrorCode *string + Variants []AttachmentVariant + ContentType string + SizeBytes int64 + SHA256 string + Status string + PipelineVersion int32 + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentVariant struct { + Name string + ObjectKey string + ContentType string + SizeBytes int64 + Width int + Height int + SHA256 string +} diff --git a/internal/model/attachment_job.go b/internal/model/attachment_job.go new file mode 100644 index 00000000..d243ff67 --- /dev/null +++ b/internal/model/attachment_job.go @@ -0,0 +1,26 @@ +package model + +import "time" + +type AttachmentJob struct { + ID string + AttachmentID string + PipelineVersion int32 + AttemptsStarted int32 + MaxAttempts int32 + LeaseGeneration int32 + LeaseUntil time.Time +} + +type AttachmentJobState struct { + AttachmentID string + PipelineVersion int32 + ID string + Status string + LeaseGeneration int32 + AttemptsStarted int32 + MaxAttempts int32 + LeaseUntil *time.Time + LeaseExpired bool + NextAttemptAt *time.Time +} diff --git a/internal/model/chat.go b/internal/model/chat.go new file mode 100644 index 00000000..2e58c626 --- /dev/null +++ b/internal/model/chat.go @@ -0,0 +1,10 @@ +package model + +import "time" + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} diff --git a/internal/model/message.go b/internal/model/message.go new file mode 100644 index 00000000..7e969c88 --- /dev/null +++ b/internal/model/message.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Message struct { + ID string + ChatID string + SenderID string + Text string + AttachmentIDs []string + CreatedAt time.Time +} diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000..aa790b76 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time + RevokedAt *time.Time +} diff --git a/internal/model/user.go b/internal/model/user.go new file mode 100644 index 00000000..f2209764 --- /dev/null +++ b/internal/model/user.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} diff --git a/internal/pipeline/cleanup.go b/internal/pipeline/cleanup.go new file mode 100644 index 00000000..9fdfd946 --- /dev/null +++ b/internal/pipeline/cleanup.go @@ -0,0 +1,64 @@ +package pipeline + +import ( + "context" + "log/slog" + "time" + + "highload/internal/repository/postgres" + "highload/internal/storage" +) + +type Cleanup struct { + Repo *postgres.AttachmentRepository + Objects storage.ObjectCleanup + Grace time.Duration + Retention time.Duration + DryRun bool +} + +func (c Cleanup) RunOnce(ctx context.Context) (int, error) { + policy := postgres.CleanupPolicy{Now: time.Now(), Grace: c.Grace, Retention: c.Retention, DryRun: c.DryRun} + removed := 0 + for _, prefix := range []string{"staging/", "results/", "attachments/"} { + var token *string + for { + pageCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + objects, next, err := c.Objects.ListPage(pageCtx, token, prefix) + cancel() + if err != nil { + return removed, err + } + for _, obj := range objects { + eligible, err := c.Repo.CleanupObject(ctx, obj, policy, c.Objects.Delete) + if err != nil { + return removed, err + } + if eligible { + removed++ + } + } + if next == nil { + break + } + token = next + } + } + return removed, nil +} + +func (c Cleanup) Serve(ctx context.Context, period time.Duration) { + for ctx.Err() == nil { + count, err := c.RunOnce(ctx) + if err != nil && ctx.Err() == nil { + slog.Warn("cleanup pass failed", "error", err) + } + if count > 0 { + slog.Info("cleanup pass", "objects", count, "dry_run", c.DryRun) + } + select { + case <-ctx.Done(): + case <-time.After(period): + } + } +} diff --git a/internal/pipeline/worker.go b/internal/pipeline/worker.go new file mode 100644 index 00000000..ec121ae6 --- /dev/null +++ b/internal/pipeline/worker.go @@ -0,0 +1,353 @@ +package pipeline + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "path/filepath" + "time" + + "highload/internal/broker" + "highload/internal/config" + "highload/internal/fault" + "highload/internal/imageproc" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/storage" + + "github.com/google/uuid" +) + +type WorkerRepository interface { + ClaimJob(context.Context, string, time.Time) (*model.AttachmentJob, error) + GetAttachmentJobState(context.Context, string) (*model.AttachmentJobState, error) + GetAttachmentByID(context.Context, string) (*model.Attachment, error) + CompleteJob(context.Context, string, int32, string, []model.AttachmentVariant) (bool, error) + PublishedObjectKeys(context.Context, string, int32) ([]string, error) + ScheduleRetry(context.Context, string, int32, string, time.Time) (bool, error) + FailJob(context.Context, string, int32, string) (bool, error) +} + +type WorkerObjects interface { + storage.ObjectWriter + Head(context.Context, string) error +} + +type Worker struct { + Repo WorkerRepository + Objects WorkerObjects + Config config.Pipeline +} + +type message struct { + SchemaVersion int `json:"schema_version"` + Type string `json:"type"` + JobID string `json:"job_id"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int32 `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func decodeMessage(payload []byte) (message, error) { + var msg message + if len(payload) > 4096 { + return msg, errors.New("oversize job") + } + d := json.NewDecoder(bytes.NewReader(payload)) + d.DisallowUnknownFields() + if err := d.Decode(&msg); err != nil { + return msg, err + } + if err := d.Decode(new(any)); err != io.EOF { + return msg, errors.New("trailing job data") + } + jobID, err1 := uuid.Parse(msg.JobID) + attachmentID, err2 := uuid.Parse(msg.AttachmentID) + if err1 != nil || err2 != nil || msg.SchemaVersion != 1 || msg.Type != "attachment.process.v1" || + msg.PipelineVersion <= 0 || msg.Attempt <= 0 || msg.CreatedAt.IsZero() { + return msg, errors.New("invalid job envelope") + } + msg.JobID, msg.AttachmentID = jobID.String(), attachmentID.String() + return msg, nil +} + +func (w Worker) Serve(ctx context.Context) { + for ctx.Err() == nil { + if err := w.consume(ctx); err != nil && ctx.Err() == nil { + slog.Warn("worker consumer reconnecting", "error", err) + } + select { + case <-ctx.Done(): + case <-time.After(500 * time.Millisecond): + } + } +} + +func (w Worker) consume(ctx context.Context) error { + conn, ch, err := broker.Connect(w.Config.BrokerURL) + if err != nil { + return err + } + defer broker.Close(conn) + if err := ch.Qos(w.Config.Prefetch, 0, false); err != nil { + return err + } + deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, "", false, false, false, false, nil) + if err != nil { + return err + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case delivery, ok := <-deliveries: + if !ok { + return errors.New("broker delivery channel closed") + } + msg, err := decodeMessage(delivery.Body) + if err != nil { + slog.Warn("rejecting invalid job envelope") + if err := delivery.Reject(false); err != nil { + return err + } + continue + } + reject, err := w.handle(ctx, msg) + switch { + case err != nil: + // Closing the channel requeues an unacknowledged delivery. + return err + case reject: + if err := delivery.Reject(false); err != nil { + return err + } + default: + if err := delivery.Ack(false); err != nil { + return err + } + } + } + } +} + +func (w Worker) handle(ctx context.Context, msg message) (bool, error) { + state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID) + if errors.Is(err, repository.ErrNotFound) { + return true, nil + } + if err != nil { + return false, err + } + if state.AttachmentID != msg.AttachmentID || state.PipelineVersion != msg.PipelineVersion { + return true, nil + } + if state.Status == "superseded" { + return false, nil + } + job, err := w.Repo.ClaimJob(ctx, msg.JobID, time.Now().Add(w.Config.Lease)) + if err != nil { + return false, err + } + if job == nil { + return false, w.handleUnclaimed(ctx, msg) + } + slog.Info("job claimed", "job_id", job.ID, "attempt", job.AttemptsStarted, "lease_generation", job.LeaseGeneration) + + attemptCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout) + err = w.process(attemptCtx, job) + cancel() + if ctx.Err() != nil { + return false, ctx.Err() + } + if err != nil { + if errors.Is(err, errLostLease) { + return false, w.handleUnclaimed(ctx, msg) + } + return false, w.saveFailure(ctx, job, err) + } + + pauseCtx, pauseCancel := context.WithTimeout(ctx, 30*time.Second) + defer pauseCancel() + return false, fault.Pause(pauseCtx, "before_ack", job.ID) +} + +func (w Worker) handleUnclaimed(ctx context.Context, msg message) error { + state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID) + if err != nil { + return err + } + switch state.Status { + case "ready": + checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + keys, err := w.Repo.PublishedObjectKeys(checkCtx, state.AttachmentID, state.PipelineVersion) + if err != nil { + return err + } + for _, key := range keys { + if err := w.Objects.Head(checkCtx, key); err != nil { + return err + } + } + return nil + case "failed", "queued", "superseded": + return nil // A queued retry is already in the transactionally durable outbox. + case "processing": + if state.AttemptsStarted >= state.MaxAttempts && state.LeaseExpired { + _, err = w.Repo.FailJob(ctx, msg.JobID, state.LeaseGeneration, "retry_exhausted") + return err + } + return nil // The recovery loop publishes one event for this expired lease. + default: + return errors.New("unrecognized job state") + } +} + +var ( + errLostLease = errors.New("job lease lost") + errOriginalMismatch = errors.New("staging hash or size differs") +) + +func (w Worker) process(ctx context.Context, job *model.AttachmentJob) error { + att, err := w.Repo.GetAttachmentByID(ctx, job.AttachmentID) + if err != nil { + return err + } + if att.PipelineVersion != job.PipelineVersion { + return errLostLease + } + if att.StagingKey == "" { + return errOriginalMismatch + } + dir, err := os.MkdirTemp("", "attachment-job-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + source := filepath.Join(dir, "input") + if err := w.download(ctx, att, source); err != nil { + return err + } + files, err := imageproc.Generate(ctx, dir, source, att.ContentType, job.PipelineVersion) + if err != nil { + return err + } + if err := fault.Pause(ctx, "before_objects", job.ID); err != nil { + return err + } + + prefix := fmt.Sprintf("results/%s/v%d/%d/", job.AttachmentID, job.PipelineVersion, job.LeaseGeneration) + originalKey := prefix + "original" + if err := w.putFile(ctx, originalKey, att.ContentType, source, att.SizeBytes); err != nil { + return err + } + variants := make([]model.AttachmentVariant, 0, len(files)) + for _, f := range files { + v := f.Variant + v.ObjectKey = prefix + filepath.Base(f.Path) + if err := w.putFile(ctx, v.ObjectKey, v.ContentType, f.Path, v.SizeBytes); err != nil { + return err + } + variants = append(variants, v) + } + if err := fault.Pause(ctx, "after_objects", job.ID); err != nil { + return err + } + done, err := w.Repo.CompleteJob(ctx, job.ID, job.LeaseGeneration, originalKey, variants) + if err != nil { + return err + } + if !done { + slog.Info("job finalization rejected", "job_id", job.ID, "lease_generation", job.LeaseGeneration) + return errLostLease + } + slog.Info("job ready", "job_id", job.ID, "lease_generation", job.LeaseGeneration) + return nil +} + +func (w Worker) download(ctx context.Context, att *model.Attachment, path string) error { + if att.SizeBytes <= 0 || att.SizeBytes > imageproc.MaxBytes { + return errOriginalMismatch + } + + reader, err := w.Objects.Get(ctx, att.StagingKey) + if err != nil { + return err + } + defer reader.Close() + f, err := os.Create(path) + if err != nil { + return err + } + sum := sha256.New() + n, copyErr := io.Copy(io.MultiWriter(f, sum), io.LimitReader(reader, att.SizeBytes+1)) + closeErr := f.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if n != att.SizeBytes || hex.EncodeToString(sum.Sum(nil)) != att.SHA256 { + return errOriginalMismatch + } + return nil +} + +func (w Worker) putFile(ctx context.Context, key, mime, path string, size int64) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return w.Objects.Put(ctx, key, mime, f, size) +} + +func (w Worker) saveFailure(ctx context.Context, job *model.AttachmentJob, failure error) error { + code := "processing_failed" + permanent := false + switch { + case errors.Is(failure, errOriginalMismatch), errors.Is(failure, imageproc.ErrInvalidImage): + code, permanent = "invalid_image", true + case errors.Is(failure, imageproc.ErrResourceLimit): + code, permanent = "resource_limit", true + case errors.Is(failure, storage.ErrObjectNotFound): + code, permanent = "original_missing", true + case errors.Is(failure, storage.ErrServiceUnavailable): + code = "s3_unavailable" + case errors.Is(failure, context.DeadlineExceeded): + code = "processing_timeout" + } + dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if permanent || job.AttemptsStarted >= job.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + changed, err := w.Repo.FailJob(dbCtx, job.ID, job.LeaseGeneration, code) + if changed { + slog.Warn("job failed", "job_id", job.ID, "error_code", code) + } + return err + } + // The floor avoids immediate requeue loops. The upper bound grows with + // each attempt and is capped to keep the lab recoverable. + upper := min(30*time.Second, w.Config.RetryBase< 0 { + maxAttempts = attemptLimit[0] + } + return &AttachmentRepository{ + queries: queries, + pool: pool, + maxAttempts: maxAttempts, + } +} + +func (r *AttachmentRepository) CreateAttachment(ctx context.Context, params repository.CreateAttachmentParams) (*model.Attachment, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + chatUUID, err := getUUIDFromString(params.ChatID) + if err != nil { + return nil, convertDBError(err) + } + ownerUUID, err := getUUIDFromString(params.OwnerID) + if err != nil { + return nil, convertDBError(err) + } + + sqlcParams := sqlc.CreateAttachmentParams{ + ChatID: *chatUUID, + OwnerID: *ownerUUID, + Filename: params.Filename, + ObjectKey: params.ObjectKey, + ContentType: params.ContentType, + SizeBytes: params.SizeBytes, + Sha256: params.Sha256, + } + + sqlcAttachment, err := r.queries.CreateAttachment(dbCtx, sqlcParams) + + if err != nil { + return nil, convertDBError(err) + } + + attachment := mapAttachmentSqlcToModel(sqlc.Attachment(sqlcAttachment)) + return attachment, nil + +} + +func (r *AttachmentRepository) GetAttachmentForUser(ctx context.Context, userID, attachmentID string) (*model.Attachment, error) { + userUUID, err := getUUIDFromString(userID) + if err != nil { + return nil, err + } + attachmentUUID, err := getUUIDFromString(attachmentID) + if err != nil { + return nil, err + } + + txCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + params := sqlc.GetAttachmentForUserParams{ + AttachmentID: *attachmentUUID, + UserID: *userUUID, + } + + tx, err := r.pool.BeginTx(txCtx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadOnly, + }) + if err != nil { + return nil, convertDBError(err) + } + defer rollbackTx(ctx, tx) + + txQueries := r.queries.WithTx(tx) + + sqlcAttachment, err := txQueries.GetAttachmentForUser(txCtx, params) + if err != nil { + return nil, convertDBError(err) + } + attachment := mapAttachmentSqlcToModel(sqlcAttachment) + + if sqlcAttachment.Status == "ready" { + variants, err := txQueries.ListAttachmentVariants(txCtx, sqlc.ListAttachmentVariantsParams{ + AttachmentID: sqlcAttachment.ID, + PipelineVersion: sqlcAttachment.PipelineVersion, + }) + if err != nil { + return nil, convertDBError(err) + } + + for _, v := range variants { + attachment.Variants = append(attachment.Variants, model.AttachmentVariant{ + Name: v.Name, + ObjectKey: v.ObjectKey, + ContentType: v.ContentType, + SizeBytes: v.SizeBytes, + Width: int(v.Width), + Height: int(v.Height), + SHA256: v.Sha256, + }) + } + } + + if err = tx.Commit(txCtx); err != nil { + return nil, convertDBError(err) + } + + return attachment, nil + +} + +func (r *AttachmentRepository) ExistsByObjectKey(ctx context.Context, key string) (bool, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + exists, err := r.queries.AttachmentExistsByObjectKey(dbCtx, key) + if err != nil { + return false, convertDBError(err) + } + return exists, nil +} + +func (r *AttachmentRepository) ListAttachmentKeys(ctx context.Context, afterKey string, pageSize int32) ([]repository.AttachmentKey, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + rows, err := r.queries.ListAttachmentKeys(dbCtx, sqlc.ListAttachmentKeysParams{ + AfterKey: pgtype.Text{String: afterKey, Valid: true}, + PageSize: pageSize, + }) + if err != nil { + return nil, convertDBError(err) + } + keys := make([]repository.AttachmentKey, 0, len(rows)) + for _, row := range rows { + keys = append(keys, repository.AttachmentKey{ + ID: row.ID.String(), ObjectKey: row.ObjectKey.String, CreatedAt: row.CreatedAt.Time, + }) + } + return keys, nil +} + +func (r *AttachmentRepository) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + attachmentID, err := getUUIDFromString(id) + if err != nil { + return nil, err + } + row, err := r.queries.GetAttachmentByID(dbCtx, *attachmentID) + if err != nil { + return nil, convertDBError(err) + } + return mapAttachmentSqlcToModel(row), nil +} + +func (r *AttachmentRepository) WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, repository.AttachmentTx) error) (bool, error) { + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + + tx, err := r.pool.BeginTx(txCtx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return false, convertDBError(err) + } + + defer rollbackTx(ctx, tx) + + txQueries := r.queries.WithTx(tx) + + locked, err := txQueries.TryLockAttachmentObjectKey(txCtx, key) + + if err != nil { + return false, convertDBError(err) + } + + if !locked { + return false, nil + } + + txRepo := &AttachmentRepository{queries: txQueries, maxAttempts: r.maxAttempts} + + if err = fn(txCtx, txRepo); err != nil { + return true, err + } + + if err = tx.Commit(txCtx); err != nil { + return true, convertDBError(err) + } + + return true, nil +} + +func (r *AttachmentRepository) CreateQueued(ctx context.Context, queuedID, chatID, ownerID, mime, sha256, key, filename string, size int64) (*model.Attachment, error) { + queuedUUID, err := getUUIDFromString(queuedID) + if err != nil { + return nil, convertDBError(err) + } + + chatUUID, err := getUUIDFromString(chatID) + if err != nil { + return nil, convertDBError(err) + } + ownerUUID, err := getUUIDFromString(ownerID) + if err != nil { + return nil, convertDBError(err) + } + + cqaParams := sqlc.CreateQueuedAttachmentParams{ + ID: *queuedUUID, + ChatID: *chatUUID, + OwnerID: *ownerUUID, + Filename: filename, + ContentType: mime, + SizeBytes: size, + Sha256: sha256, + StagingKey: pgtype.Text{String: key, Valid: true}, + } + + // The upload path already holds the object lock in a transaction. + if r.pool == nil { + return r.insertQueued(ctx, r.queries, cqaParams) + } + + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + tx, err := r.pool.BeginTx(txCtx, pgx.TxOptions{}) + if err != nil { + return nil, convertDBError(err) + } + defer rollbackTx(ctx, tx) + queued, err := r.insertQueued(txCtx, r.queries.WithTx(tx), cqaParams) + if err != nil { + return nil, err + } + if err := tx.Commit(txCtx); err != nil { + return nil, convertDBError(err) + } + return queued, nil +} + +func (r *AttachmentRepository) insertQueued(txCtx context.Context, txQueries *sqlc.Queries, cqaParams sqlc.CreateQueuedAttachmentParams) (*model.Attachment, error) { + + sqlcAttachment, err := txQueries.CreateQueuedAttachment(txCtx, cqaParams) + + if err != nil { + return nil, convertDBError(err) + } + + if _, err := r.createPipelineJob(txCtx, txQueries, sqlcAttachment); err != nil { + return nil, err + } + + return mapAttachmentSqlcToModel(sqlcAttachment), nil + +} + +func (r *AttachmentRepository) ClaimJob(ctx context.Context, jobID string, leaseUntil time.Time) (*model.AttachmentJob, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + jobUUID, err := getUUIDFromString(jobID) + if err != nil { + return nil, convertDBError(err) + } + + arg := sqlc.ClaimJobParams{ + NewLeaseUntil: pgtype.Timestamptz{Time: leaseUntil, Valid: true}, + JobID: *jobUUID, + } + + row, err := r.queries.ClaimJob(dbCtx, arg) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, convertDBError(err) + } + + return &model.AttachmentJob{ + ID: row.ID.String(), + AttachmentID: row.AttachmentID.String(), + PipelineVersion: row.PipelineVersion, + AttemptsStarted: row.AttemptsStarted, + MaxAttempts: row.MaxAttempts, + LeaseGeneration: row.LeaseGeneration, + LeaseUntil: row.LeaseUntil.Time, + }, nil +} + +func (r *AttachmentRepository) GetAttachmentJobState(ctx context.Context, jobID string) (*model.AttachmentJobState, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + jobUUID, err := getUUIDFromString(jobID) + if err != nil { + return nil, convertDBError(err) + } + + row, err := r.queries.GetAttachmentJobState(dbCtx, *jobUUID) + if err != nil { + return nil, convertDBError(err) + } + + state := &model.AttachmentJobState{ + AttachmentID: row.AttachmentID.String(), + PipelineVersion: row.PipelineVersion, + ID: row.ID.String(), + Status: row.Status, + LeaseGeneration: row.LeaseGeneration, + AttemptsStarted: row.AttemptsStarted, + MaxAttempts: row.MaxAttempts, + LeaseExpired: row.LeaseExpired, + } + + if row.LeaseUntil.Valid { + state.LeaseUntil = &row.LeaseUntil.Time + } + if row.NextAttemptAt.Valid { + state.NextAttemptAt = &row.NextAttemptAt.Time + } + + return state, nil + +} + +func (r *AttachmentRepository) CompleteJob( + ctx context.Context, jobID string, leaseGeneration int32, + originalKey string, variants []model.AttachmentVariant, +) (bool, error) { + jobUUID, err := getUUIDFromString(jobID) + if err != nil { + return false, err + } + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + tx, err := r.pool.BeginTx(txCtx, pgx.TxOptions{}) + if err != nil { + return false, convertDBError(err) + } + defer rollbackTx(ctx, tx) + txQueries := r.queries.WithTx(tx) + lockedJob, err := txQueries.LockActiveAttachmentJob(txCtx, sqlc.LockActiveAttachmentJobParams{ + JobID: *jobUUID, + LeaseGeneration: leaseGeneration, + }) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, convertDBError(err) + } + + const maxDimension = 1<<31 - 1 + for _, variant := range variants { + if variant.Width <= 0 || variant.Width > maxDimension || + variant.Height <= 0 || variant.Height > maxDimension { + return false, repository.ErrInvalidInput + } + err := txQueries.CreateAttachmentVariant(txCtx, sqlc.CreateAttachmentVariantParams{ + AttachmentID: lockedJob.AttachmentID, + PipelineVersion: lockedJob.PipelineVersion, + Name: variant.Name, + ObjectKey: variant.ObjectKey, + ContentType: variant.ContentType, + SizeBytes: variant.SizeBytes, + Width: int32(variant.Width), + Height: int32(variant.Height), + Sha256: variant.SHA256, + }) + if err != nil { + return false, convertDBError(err) + } + } + + rows, err := txQueries.CompleteJob(txCtx, sqlc.CompleteJobParams{ + JobID: *jobUUID, + LeaseGeneration: leaseGeneration, + ObjectKey: originalKey, + }) + if err != nil { + return false, convertDBError(err) + } + if rows == 0 { + return false, nil + } + if err := tx.Commit(txCtx); err != nil { + return false, convertDBError(err) + } + return true, nil + +} + +func mapAttachmentSqlcToModel(sa sqlc.Attachment) *model.Attachment { + var errorCode *string + if sa.ErrorCode.Valid { + errorCode = &sa.ErrorCode.String + } + return &model.Attachment{ + Variants: []model.AttachmentVariant{}, + ErrorCode: errorCode, + StagingKey: sa.StagingKey.String, + ID: sa.ID.String(), + ChatID: sa.ChatID.String(), + OwnerID: sa.OwnerID.String(), + Filename: sa.Filename, + ObjectKey: sa.ObjectKey.String, + ContentType: sa.ContentType, + SizeBytes: sa.SizeBytes, + Status: sa.Status, + PipelineVersion: sa.PipelineVersion, + SHA256: sa.Sha256, + CreatedAt: sa.CreatedAt.Time, + UpdatedAt: sa.UpdatedAt.Time, + } + +} diff --git a/internal/repository/postgres/chat.go b/internal/repository/postgres/chat.go new file mode 100644 index 00000000..aedb3fb1 --- /dev/null +++ b/internal/repository/postgres/chat.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "highload/internal/repository" + "slices" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +type ChatRepository struct { + queries *sqlc.Queries + pool *pgxpool.Pool +} + +func NewChatRepository(queries *sqlc.Queries, pool *pgxpool.Pool) *ChatRepository { + return &ChatRepository{ + queries: queries, + pool: pool, + } +} + +func (r *ChatRepository) CreateChat(ctx context.Context, title string, memberIDs []string) (*model.Chat, error) { + txCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + tx, err := r.pool.Begin(txCtx) + if err != nil { + return nil, convertDBError(err) + } + + defer func() { + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + time.Second, + ) + defer rollbackCancel() + + _ = tx.Rollback(rollbackCtx) + }() + txQueries := r.queries.WithTx(tx) + + sqlcChat, err := txQueries.CreateChat(txCtx, title) + if err != nil { + return nil, convertDBError(err) + } + + chat := &model.Chat{ + ID: sqlcChat.ID.String(), + Title: sqlcChat.Title, + CreatedAt: sqlcChat.CreatedAt.Time, + } + + for _, id := range memberIDs { + userID, err := getUUIDFromString(id) + if err != nil { + return nil, convertDBError(err) + } + err = txQueries.CreateChatMember(txCtx, sqlc.CreateChatMemberParams{ChatID: sqlcChat.ID, UserID: *userID}) + + if err != nil { + return nil, convertDBError(err) + } + } + + chat.MemberIDs = slices.Clone(memberIDs) + + err = tx.Commit(txCtx) + if err != nil { + return nil, convertDBError(err) + } + + return chat, nil + +} + +func (r *ChatRepository) GetChatByID(ctx context.Context, id string) (*model.Chat, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidID, err := getUUIDFromString(id) + + if err != nil { + return nil, convertDBError(err) + } + + sqlcChat, err := r.queries.GetChatByID(dbCtx, *uuidID) + + if err != nil { + return nil, convertDBError(err) + } + + chat := mapChatSqlcToModel(&sqlcChat) + + return chat, nil + +} + +func (r *ChatRepository) GetChatsByUserID(ctx context.Context, userID string, limit int, after *repository.ChatPosition) ([]*model.Chat, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidUserID, err := getUUIDFromString(userID) + + if err != nil { + return nil, convertDBError(err) + } + + afterCreatedAt := pgtype.Timestamptz{Valid: false} + afterID := pgtype.UUID{Valid: false} + + if after != nil { + afterCreatedAt = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true} + if uuidFromString, err := getUUIDFromString(after.ID); err == nil { + afterID = *uuidFromString + } else { + return nil, convertDBError(err) + } + } + + chats, err := r.queries.GetChatsByUserID(dbCtx, sqlc.GetChatsByUserIDParams{UserID: *uuidUserID, AfterCreatedAt: afterCreatedAt, AfterID: afterID, LimitCount: int32(limit)}) + + if err != nil { + return nil, convertDBError(err) + } + + resultChats := make([]*model.Chat, len(chats)) + for index, sqlcChat := range chats { + memberIDs := make([]string, len(sqlcChat.MemberIds)) + + for i, memberID := range sqlcChat.MemberIds { + memberIDs[i] = memberID.String() + } + + resultChats[index] = &model.Chat{ + ID: sqlcChat.ID.String(), + MemberIDs: memberIDs, + CreatedAt: sqlcChat.CreatedAt.Time, + Title: sqlcChat.Title, + } + } + + return resultChats, nil +} + +func mapChatSqlcToModel(sqlcChat *sqlc.GetChatByIDRow) *model.Chat { + + memberIDs := make([]string, len(sqlcChat.MemberIds)) + + for i, memberID := range sqlcChat.MemberIds { + memberIDs[i] = memberID.String() + } + + return &model.Chat{ + ID: sqlcChat.ID.String(), + MemberIDs: memberIDs, + CreatedAt: sqlcChat.CreatedAt.Time, + Title: sqlcChat.Title, + } + +} diff --git a/internal/repository/postgres/errors.go b/internal/repository/postgres/errors.go new file mode 100644 index 00000000..b9897742 --- /dev/null +++ b/internal/repository/postgres/errors.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "highload/internal/repository" + "io" + "net" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func convertDBError(err error) error { + var pgErr *pgconn.PgError + var netErr *net.OpError + var dnsErr *net.DNSError + + switch { + case errors.Is(err, context.DeadlineExceeded): + return fmt.Errorf("%w: %v", repository.ErrServiceUnavailable, err) + case errors.Is(err, pgx.ErrNoRows): + return repository.ErrNotFound + case errors.As(err, &pgErr): + switch pgErr.Code { + case "23505": + return repository.ErrAlreadyExists + case "53300", "57P03", "08006", "08001", "57P01", "57P02": + return repository.ErrServiceUnavailable + } + case errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, pgconn.ErrConnClosed): + return repository.ErrServiceUnavailable + case errors.Is(err, repository.ErrInvalidInput): + return repository.ErrInvalidInput + case errors.As(err, &netErr): + return repository.ErrServiceUnavailable + case errors.As(err, &dnsErr): + return repository.ErrServiceUnavailable + } + + return fmt.Errorf("%w: %v", repository.ErrDatabase, err) + +} diff --git a/internal/repository/postgres/message.go b/internal/repository/postgres/message.go new file mode 100644 index 00000000..1bed30af --- /dev/null +++ b/internal/repository/postgres/message.go @@ -0,0 +1,191 @@ +package postgres + +import ( + "context" + "errors" + "highload/internal/database/sqlc" + "highload/internal/model" + "highload/internal/repository" + "slices" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +type MessageRepository struct { + queries *sqlc.Queries + pool *pgxpool.Pool +} + +func NewMessageRepository(queries *sqlc.Queries, pool *pgxpool.Pool) *MessageRepository { + return &MessageRepository{ + queries: queries, + pool: pool, + } +} + +func (r *MessageRepository) CreateMessage(ctx context.Context, chatID, senderID, text, key string, attachmentIDs []string) (*model.Message, error) { + chat, err := getUUIDFromString(chatID) + if err != nil { + return nil, err + } + sender, err := getUUIDFromString(senderID) + if err != nil { + return nil, err + } + + ids := make([]pgtype.UUID, 0, len(attachmentIDs)) + canonical := make([]string, 0, len(attachmentIDs)) + for _, attachmentID := range attachmentIDs { + id, err := getUUIDFromString(attachmentID) + if err != nil { + return nil, err + } + ids = append(ids, *id) + canonical = append(canonical, id.String()) + } + + txCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + tx, err := r.pool.Begin(txCtx) + if err != nil { + return nil, convertDBError(err) + } + defer rollbackTx(ctx, tx) + q := r.queries.WithTx(tx) + + if err := q.LockMessageRequest(txCtx, sqlc.LockMessageRequestParams{UserID: *sender, ChatID: *chat, IdempotencyKey: key}); err != nil { + return nil, convertDBError(err) + } + if err := q.DeleteExpiredMessageIdempotency(txCtx, sqlc.DeleteExpiredMessageIdempotencyParams{UserID: *sender, ChatID: *chat, IdempotencyKey: key}); err != nil { + return nil, convertDBError(err) + } + message, err := q.GetIdempotentMessage(txCtx, sqlc.GetIdempotentMessageParams{UserID: *sender, ChatID: *chat, IdempotencyKey: key}) + switch { + case err == nil: + rows, err := q.GetMessagesAttachments(txCtx, message.ID) + if err != nil { + return nil, convertDBError(err) + } + saved := make([]string, 0, len(rows)) + for _, row := range rows { + saved = append(saved, row.AttachmentID.String()) + } + if message.Text != text || !slices.Equal(saved, canonical) { + return nil, repository.ErrConflict + } + case errors.Is(err, pgx.ErrNoRows): + if len(ids) > 0 { + rows, err := q.LockAttachmentsForMessage(txCtx, sqlc.LockAttachmentsForMessageParams{ChatID: *chat, AttachmentIds: ids}) + if err != nil { + return nil, convertDBError(err) + } + if len(rows) != len(ids) { + return nil, repository.ErrNotFound + } + for _, row := range rows { + if row.Status == "failed" { + return nil, repository.ErrConflict + } + } + } + message, err = q.CreateMessage(txCtx, sqlc.CreateMessageParams{ChatID: *chat, SenderID: *sender, Text: text}) + if err != nil { + return nil, convertDBError(err) + } + for i, id := range ids { + n, err := q.CreateMessagesAttachments(txCtx, sqlc.CreateMessagesAttachmentsParams{ + MessageID: message.ID, AttachmentID: id, ChatID: *chat, Position: int16(i), + }) + if err != nil { + return nil, convertDBError(err) + } + if n != 1 { + return nil, repository.ErrNotFound + } + } + n, err := q.InsertMessageIdempotency(txCtx, sqlc.InsertMessageIdempotencyParams{ + UserID: *sender, ChatID: *chat, IdempotencyKey: key, MessageID: message.ID, + }) + if err != nil { + return nil, convertDBError(err) + } + if n != 1 { + return nil, repository.ErrConflict + } + default: + return nil, convertDBError(err) + } + if err := tx.Commit(txCtx); err != nil { + return nil, convertDBError(err) + } + return &model.Message{ + ID: message.ID.String(), ChatID: message.ChatID.String(), SenderID: message.SenderID.String(), + Text: message.Text, CreatedAt: message.CreatedAt.Time, AttachmentIDs: canonical, + }, nil +} + +func (r *MessageRepository) GetMessagesByChatID(ctx context.Context, chatID string, limit int, after *repository.MessagePosition) ([]*model.Message, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidChatID, err := getUUIDFromString(chatID) + + if err != nil { + return nil, convertDBError(err) + } + + afterCreatedAt := pgtype.Timestamptz{Valid: false} + afterID := pgtype.UUID{Valid: false} + + if after != nil { + afterCreatedAt = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true} + if uuidFromString, err := getUUIDFromString(after.ID); err == nil { + afterID = *uuidFromString + } else { + return nil, convertDBError(err) + } + } + + sqlcMessages, err := r.queries.GetMessagesByChatID(dbCtx, sqlc.GetMessagesByChatIDParams{ChatID: *uuidChatID, AfterID: afterID, AfterCreatedAt: afterCreatedAt, LimitCount: int32(limit)}) + + if err != nil { + return nil, convertDBError(err) + } + + messageIDs := make([]pgtype.UUID, len(sqlcMessages)) + for index, _ := range sqlcMessages { + messageIDs[index] = sqlcMessages[index].ID + } + + messageRow, err := r.queries.GetAttachmentsForMessages(dbCtx, messageIDs) + + if err != nil { + return nil, convertDBError(err) + } + + mapMessagesAtt := make(map[string][]string) + for index, _ := range messageRow { + messageID := messageRow[index].MessageID.String() + attachmentID := messageRow[index].AttachmentID.String() + + mapMessagesAtt[messageID] = append(mapMessagesAtt[messageID], attachmentID) + + } + + messages := make([]*model.Message, len(sqlcMessages)) + + for index, sqlcMessage := range sqlcMessages { + messages[index] = &model.Message{ + ID: sqlcMessage.ID.String(), + ChatID: sqlcMessage.ChatID.String(), + SenderID: sqlcMessage.SenderID.String(), + Text: sqlcMessage.Text, + AttachmentIDs: mapMessagesAtt[sqlcMessage.ID.String()], + CreatedAt: sqlcMessage.CreatedAt.Time} + } + + return messages, nil + +} diff --git a/internal/repository/postgres/pipeline.go b/internal/repository/postgres/pipeline.go new file mode 100644 index 00000000..7f0fb98c --- /dev/null +++ b/internal/repository/postgres/pipeline.go @@ -0,0 +1,167 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "highload/internal/database/sqlc" + "highload/internal/repository" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (r *AttachmentRepository) ScheduleRetry(ctx context.Context, jobID string, generation int32, code string, at time.Time) (bool, error) { + id, err := getUUIDFromString(jobID) + if err != nil { + return false, err + } + n, err := r.queries.ScheduleRetry(ctx, sqlc.ScheduleRetryParams{ + JobID: *id, LeaseGeneration: generation, ErrorCode: code, + AvailableAt: pgtype.Timestamptz{Time: at, Valid: true}, + }) + if err != nil { + return false, convertDBError(err) + } + return n == 1, nil +} + +func (r *AttachmentRepository) FailJob(ctx context.Context, jobID string, generation int32, code string) (bool, error) { + id, err := getUUIDFromString(jobID) + if err != nil { + return false, err + } + n, err := r.queries.FailJob(ctx, sqlc.FailJobParams{ + JobID: *id, LeaseGeneration: generation, ErrorCode: code, + }) + if err != nil { + return false, convertDBError(err) + } + return n == 1, nil +} + +func (r *AttachmentRepository) EnqueueExpiredLeases(ctx context.Context, batchSize int32) (int, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + rows, err := r.queries.EnqueueExpiredLeases(ctx, batchSize) + if err != nil { + return 0, convertDBError(err) + } + return len(rows), nil +} + +// PublishPending holds the outbox row until the broker confirms delivery. +func (r *AttachmentRepository) PublishPending(ctx context.Context, publish func(context.Context, sqlc.Outbox) error) (bool, error) { + txCtx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + tx, err := r.pool.Begin(txCtx) + if err != nil { + return false, convertDBError(err) + } + defer rollbackTx(ctx, tx) + q := r.queries.WithTx(tx) + row, err := q.NextPendingOutboxEvent(txCtx) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, convertDBError(err) + } + if err := publish(txCtx, row); err != nil { + return false, err + } + n, err := q.MarkOutboxEventSent(txCtx, row.ID) + if err != nil { + return false, convertDBError(err) + } + if n != 1 { + return false, fmt.Errorf("outbox row disappeared during publish") + } + if err := tx.Commit(txCtx); err != nil { + return false, convertDBError(err) + } + return true, nil +} + +func (r *AttachmentRepository) ListFailedJobs(ctx context.Context) ([]sqlc.ListFailedAttachmentJobsRow, error) { + return r.queries.ListFailedAttachmentJobs(ctx, 100) +} + +// ReplayFailed checks the staged object while holding the same job lock as cleanup. +func (r *AttachmentRepository) ReplayFailed(ctx context.Context, jobID, requestID, operator, reason string, extra int32, check func(context.Context, string) error) (bool, error) { + id, err := getUUIDFromString(jobID) + if err != nil { + return false, err + } + request, err := getUUIDFromString(requestID) + if err != nil { + return false, err + } + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + tx, err := r.pool.Begin(txCtx) + if err != nil { + return false, convertDBError(err) + } + defer rollbackTx(ctx, tx) + q := r.queries.WithTx(tx) + job, err := q.LockAttachmentJobForReplay(txCtx, *id) + if err != nil { + return false, convertDBError(err) + } + att, err := q.GetAttachmentByID(txCtx, job.AttachmentID) + if err != nil { + return false, convertDBError(err) + } + if att.Status != "failed" || att.PipelineVersion != job.PipelineVersion { + return false, nil + } + if !att.StagingKey.Valid { + return false, fmt.Errorf("staging has expired: %w", repository.ErrNotFound) + } + if err := check(txCtx, att.StagingKey.String); err != nil { + return false, err + } + rows, err := q.ReplayFailed(txCtx, sqlc.ReplayFailedParams{ + JobID: *id, RequestID: *request, OperatorID: operator, Reason: reason, ExtraAttempts: extra, + }) + if err != nil { + return false, convertDBError(err) + } + if rows == 0 { + return false, nil + } + if err := tx.Commit(txCtx); err != nil { + return false, convertDBError(err) + } + return true, nil +} + +func (r *AttachmentRepository) PublishedObjectKeys(ctx context.Context, attachmentID string, version int32) ([]string, error) { + id, err := getUUIDFromString(attachmentID) + if err != nil { + return nil, err + } + original, err := r.queries.GetPublishedOriginalKey(ctx, sqlc.GetPublishedOriginalKeyParams{ + AttachmentID: *id, PipelineVersion: version, + }) + if err != nil { + return nil, convertDBError(err) + } + variants, err := r.queries.ListAttachmentVariants(ctx, sqlc.ListAttachmentVariantsParams{ + AttachmentID: *id, PipelineVersion: version, + }) + if err != nil { + return nil, convertDBError(err) + } + keys := []string{original.String} + for _, v := range variants { + keys = append(keys, v.ObjectKey) + } + if len(keys) != 4 { + return nil, fmt.Errorf("ready job has an incomplete variant set") + } + return keys, nil +} diff --git a/internal/repository/postgres/pipeline_cleanup.go b/internal/repository/postgres/pipeline_cleanup.go new file mode 100644 index 00000000..dd69933f --- /dev/null +++ b/internal/repository/postgres/pipeline_cleanup.go @@ -0,0 +1,118 @@ +package postgres + +import ( + "context" + "errors" + "strconv" + "strings" + "time" + + "highload/internal/database/sqlc" + "highload/internal/storage" + + "github.com/jackc/pgx/v5" +) + +type CleanupPolicy struct { + Now time.Time + Grace time.Duration + Retention time.Duration + DryRun bool +} + +// CleanupObject synchronizes with upload, finalization and operator replay. +// All candidate metadata is rechecked while holding the relevant lock. +func (r *AttachmentRepository) CleanupObject(ctx context.Context, obj storage.ObjectInfo, policy CleanupPolicy, deleteObject func(context.Context, string) error) (bool, error) { + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + tx, err := r.pool.Begin(txCtx) + if err != nil { + return false, convertDBError(err) + } + defer rollbackTx(ctx, tx) + q := r.queries.WithTx(tx) + locked, err := q.TryLockAttachmentObjectKey(txCtx, obj.Key) + if err != nil || !locked { + return false, err + } + + staging, err := q.GetAttachmentByStagingKey(txCtx, obj.Key) + keepObject := false + switch { + case err == nil: + version := staging.PipelineVersion + _, err = q.LockAttachmentJobForPipeline(txCtx, sqlc.LockAttachmentJobForPipelineParams{ + AttachmentID: staging.ID, PipelineVersion: staging.PipelineVersion, + }) + if err != nil { + return false, err + } + staging, err = q.GetAttachmentByID(txCtx, staging.ID) + if err != nil { + return false, err + } + if staging.PipelineVersion != version || staging.StagingKey.String != obj.Key || + (staging.Status == "ready" && staging.ObjectKey.String == obj.Key) { + return false, nil + } + if staging.Status != "ready" && (staging.Status != "failed" || staging.UpdatedAt.Time.Add(policy.Retention).After(policy.Now)) { + return false, nil + } + // An upgrade can stage the original published by the previous version. + keepObject, err = q.IsPublishedOriginalKey(txCtx, obj.Key) + if err != nil { + return false, err + } + case !errors.Is(err, pgx.ErrNoRows): + return false, err + default: + if obj.LastModified.IsZero() || obj.LastModified.Add(policy.Grace).After(policy.Now) { + return false, nil + } + if strings.HasPrefix(obj.Key, "results/") { + // Keys include the job version and the generating lease. + parts := strings.Split(obj.Key, "/") + if len(parts) != 5 { + return false, nil + } + id, idErr := getUUIDFromString(parts[1]) + version, versionErr := strconv.ParseInt(strings.TrimPrefix(parts[2], "v"), 10, 32) + generation, generationErr := strconv.ParseInt(parts[3], 10, 32) + if idErr != nil || versionErr != nil || generationErr != nil || version < 1 || generation < 1 { + return false, nil + } + job, err := q.LockAttachmentJobForPipeline(txCtx, sqlc.LockAttachmentJobForPipelineParams{ + AttachmentID: *id, PipelineVersion: int32(version), + }) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, err + } + if err == nil && job.LeaseGeneration == int32(generation) && job.LeaseUntil.Valid { + return false, nil + } + } + exists, err := q.AttachmentExistsByObjectKey(txCtx, obj.Key) + if err != nil || exists { + return false, err + } + } + if policy.DryRun { + return true, nil + } + if !keepObject { + if err := deleteObject(txCtx, obj.Key); err != nil && !errors.Is(err, storage.ErrObjectNotFound) { + return false, err + } + } + if staging.StagingKey.Valid { + if _, err = q.ClearStagingReference(txCtx, sqlc.ClearStagingReferenceParams{ + ID: staging.ID, StagingKey: staging.StagingKey, + }); err != nil { + return false, err + } + } + if err := tx.Commit(txCtx); err != nil { + return false, err + } + return !keepObject, nil +} diff --git a/internal/repository/postgres/user.go b/internal/repository/postgres/user.go new file mode 100644 index 00000000..d30dc071 --- /dev/null +++ b/internal/repository/postgres/user.go @@ -0,0 +1,85 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "time" +) + +type UserRepository struct { + queries *sqlc.Queries +} + +func NewUserRepository(queries *sqlc.Queries) *UserRepository { + return &UserRepository{queries: queries} +} + +func (r *UserRepository) Create(ctx context.Context, username, displayName string, passwordHash string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + sqlcUser, err := r.queries.CreateUser(dbCtx, sqlc.CreateUserParams{Username: username, DisplayName: displayName, PasswordHash: passwordHash}) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil + +} + +func (r *UserRepository) GetUserByID(ctx context.Context, id string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidID, err := getUUIDFromString(id) + + if err != nil { + return nil, convertDBError(err) + } + sqlcUser, err := r.queries.GetUserByID(dbCtx, *uuidID) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil + +} + +func (r *UserRepository) GetUserByUsername(ctx context.Context, username string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + sqlcUser, err := r.queries.GetUserByUsername(dbCtx, username) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil +} diff --git a/internal/repository/postgres/utils.go b/internal/repository/postgres/utils.go new file mode 100644 index 00000000..7acd7bcf --- /dev/null +++ b/internal/repository/postgres/utils.go @@ -0,0 +1,26 @@ +package postgres + +import ( + "context" + "highload/internal/repository" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func getUUIDFromString(id string) (*pgtype.UUID, error) { + var uuidID pgtype.UUID + + if err := uuidID.Scan(id); err != nil { + return nil, repository.ErrInvalidInput + } + + return &uuidID, nil +} + +func rollbackTx(ctx context.Context, tx pgx.Tx) { + rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + defer cancel() + _ = tx.Rollback(rollbackCtx) +} diff --git a/internal/repository/postgres/versioned_pipeline.go b/internal/repository/postgres/versioned_pipeline.go new file mode 100644 index 00000000..2a61a74a --- /dev/null +++ b/internal/repository/postgres/versioned_pipeline.go @@ -0,0 +1,87 @@ +package postgres + +import ( + "context" + "fmt" + + "highload/internal/database/sqlc" + "highload/internal/repository" + + "github.com/jackc/pgx/v5/pgtype" +) + +func (r *AttachmentRepository) createPipelineJob(ctx context.Context, q *sqlc.Queries, att sqlc.Attachment) (pgtype.UUID, error) { + job, err := q.CreateAttachmentJob(ctx, sqlc.CreateAttachmentJobParams{ + AttachmentID: att.ID, PipelineVersion: att.PipelineVersion, + AvailableAt: att.UpdatedAt, MaxAttempts: r.maxAttempts, + }) + if err != nil { + return pgtype.UUID{}, convertDBError(err) + } + if err := q.CreateInitialOutboxEvent(ctx, job.ID); err != nil { + return pgtype.UUID{}, convertDBError(err) + } + return job.ID, nil +} + +// UpgradePipeline serializes the version switch with version 1's finalization, +// retries and staging cleanup on its job row. The version 2 job starts fresh. +func (r *AttachmentRepository) UpgradePipeline(ctx context.Context, attachmentID string, checkSource func(context.Context, string) error) (string, bool, error) { + id, err := getUUIDFromString(attachmentID) + if err != nil { + return "", false, err + } + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + tx, err := r.pool.Begin(txCtx) + if err != nil { + return "", false, convertDBError(err) + } + defer rollbackTx(ctx, tx) + q := r.queries.WithTx(tx) + oldJob, err := q.LockAttachmentJobForPipeline(txCtx, sqlc.LockAttachmentJobForPipelineParams{ + AttachmentID: *id, PipelineVersion: 1, + }) + if err != nil { + return "", false, convertDBError(err) + } + att, err := q.GetAttachmentByID(txCtx, *id) + if err != nil { + return "", false, convertDBError(err) + } + if att.PipelineVersion == 2 { + jobID, err := q.GetPipelineJobID(txCtx, sqlc.GetPipelineJobIDParams{AttachmentID: *id, PipelineVersion: 2}) + if err != nil { + return "", false, convertDBError(err) + } + return jobID.String(), false, nil + } + if att.PipelineVersion != 1 { + return "", false, fmt.Errorf("cannot upgrade pipeline version %d", att.PipelineVersion) + } + source := att.StagingKey + if att.Status == "ready" { + source = att.ObjectKey + } + if !source.Valid { + return "", false, fmt.Errorf("original has expired: %w", repository.ErrNotFound) + } + if err := checkSource(txCtx, source.String); err != nil { + return "", false, err + } + if err := q.SupersedeAttachmentJob(txCtx, oldJob.ID); err != nil { + return "", false, convertDBError(err) + } + upgraded, err := q.QueueAttachmentUpgrade(txCtx, sqlc.QueueAttachmentUpgradeParams{ID: *id, SourceKey: source.String}) + if err != nil { + return "", false, convertDBError(err) + } + jobID, err := r.createPipelineJob(txCtx, q, upgraded) + if err != nil { + return "", false, err + } + if err := tx.Commit(txCtx); err != nil { + return "", false, convertDBError(err) + } + return jobID.String(), true, nil +} diff --git a/internal/repository/redis/errors.go b/internal/repository/redis/errors.go new file mode 100644 index 00000000..8453fc0e --- /dev/null +++ b/internal/repository/redis/errors.go @@ -0,0 +1,38 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "highload/internal/repository" + "io" + "net" + + "github.com/redis/go-redis/v9" +) + +func convertDBError(err error) error { + + category := repository.ErrDatabase + + var netErr *net.OpError + + if errors.Is(err, redis.Nil) { + category = repository.ErrNotFound + } + if errors.As(err, &netErr) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, context.DeadlineExceeded) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, redis.ErrPoolTimeout) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) { + category = repository.ErrServiceUnavailable + } + + return fmt.Errorf("%w: %v", category, err) +} diff --git a/internal/repository/redis/session.go b/internal/repository/redis/session.go new file mode 100644 index 00000000..bab4db7e --- /dev/null +++ b/internal/repository/redis/session.go @@ -0,0 +1,127 @@ +package redis + +import ( + "context" + "encoding/hex" + "encoding/json/v2" + "fmt" + "highload/internal/model" + "highload/internal/repository" + "time" + + "github.com/redis/go-redis/v9" +) + +type SessionRepository struct { + client *redis.Client +} + +func NewSessionRepository(client *redis.Client) *SessionRepository { + return &SessionRepository{client: client} +} + +type sessionData struct { + UserID string `json:"user_id"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` +} + +func (r *SessionRepository) Create(ctx context.Context, tokenHash []byte, userID string, sessionTTL time.Duration) (*model.Session, error) { + key := "session:" + hex.EncodeToString(tokenHash) + now := time.Now().UTC() + data := sessionData{ + UserID: userID, + CreatedAt: now, + ExpiresAt: now.Add(sessionTTL), + } + value, err := json.Marshal(data) + + if err != nil { + return nil, fmt.Errorf("%w: encoding session %v", repository.ErrDatabase, err) + + } + + err = r.client.Set(ctx, key, value, sessionTTL).Err() + + if err != nil { + return nil, convertDBError(err) + } + + session := &model.Session{ + TokenHash: tokenHash, + UserID: data.UserID, + CreatedAt: data.CreatedAt, + ExpiresAt: data.ExpiresAt, + RevokedAt: nil, + } + + return session, nil + +} + +func (r *SessionRepository) GetSessionByTokenHash(ctx context.Context, tokenHash []byte) (*model.Session, error) { + key := "session:" + hex.EncodeToString(tokenHash) + + value, err := r.client.Get(ctx, key).Result() + if err != nil { + return nil, convertDBError(err) + } + + var data sessionData + + err = json.Unmarshal([]byte(value), &data) + + if err != nil { + return nil, fmt.Errorf("%w: decode session %v", repository.ErrDatabase, err) + + } + + session := &model.Session{ + TokenHash: tokenHash, + UserID: data.UserID, + CreatedAt: data.CreatedAt, + ExpiresAt: data.ExpiresAt, + RevokedAt: nil, + } + + return session, nil +} +func (r *SessionRepository) RevokeSession( + ctx context.Context, + tokenHash []byte, +) (int, error) { + key := "session:" + hex.EncodeToString(tokenHash) + + conn := r.client.Conn() + defer conn.Close() + + deleted, err := conn.Del(ctx, key).Result() + if err != nil { + return 0, convertDBError(err) + + } + + result, err := conn.Do(ctx, "WAITAOF", 1, 0, 2000).Slice() + if err != nil { + return 0, convertDBError(err) + + } + + if len(result) != 2 { + return 0, fmt.Errorf( + "%w: WAITAOF: expected 2 counters, got %d", + repository.ErrDatabase, len(result), + ) + } + + local, ok := result[0].(int64) + + if !ok { + return 0, fmt.Errorf("%w: expected int64, got %T", repository.ErrDatabase, result[0]) + } + if local != 1 { + return 0, fmt.Errorf("%w: WAITAOF expected local fsync = 1, got %d", repository.ErrServiceUnavailable, local) + } + + return int(deleted), nil +} diff --git a/internal/repository/session.go b/internal/repository/session.go new file mode 100644 index 00000000..01775c08 --- /dev/null +++ b/internal/repository/session.go @@ -0,0 +1,13 @@ +package repository + +import ( + "context" + "highload/internal/model" + "time" +) + +type SessionRepository interface { + Create(ctx context.Context, tokenHash []byte, userID string, sessionTTL time.Duration) (*model.Session, error) + GetSessionByTokenHash(ctx context.Context, tokenHash []byte) (*model.Session, error) + RevokeSession(ctx context.Context, tokenHash []byte) (int, error) +} diff --git a/internal/repository/user.go b/internal/repository/user.go new file mode 100644 index 00000000..a55d99f7 --- /dev/null +++ b/internal/repository/user.go @@ -0,0 +1,12 @@ +package repository + +import ( + "context" + "highload/internal/model" +) + +type UserRepository interface { + Create(ctx context.Context, username, displayName string, passwordHash string) (*model.User, error) + GetUserByID(ctx context.Context, id string) (*model.User, error) + GetUserByUsername(ctx context.Context, username string) (*model.User, error) +} diff --git a/internal/security/password.go b/internal/security/password.go new file mode 100644 index 00000000..d9fac229 --- /dev/null +++ b/internal/security/password.go @@ -0,0 +1,143 @@ +package security + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + memory = 64 * 1024 + iterations = 3 + parallelism = 2 + saltLength = 16 + hashLength = 32 +) + +var ErrInvalidHash = errors.New("invalid password hash") + +func HashPassword(password string) (string, error) { + + salt := make([]byte, saltLength) + + if _, err := rand.Read(salt); err != nil { + return "", err + } + encodedSalt := base64.RawStdEncoding.EncodeToString(salt) + + hash := argon2.IDKey( + []byte(password), + salt, + iterations, + memory, + parallelism, + hashLength, + ) + + encodedHash := base64.RawStdEncoding.EncodeToString(hash) + + hashPassword := fmt.Sprintf( + "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + memory, + iterations, + parallelism, + encodedSalt, + encodedHash, + ) + + return hashPassword, nil + +} + +func VerifyPassword(password, encodedHash string) (bool, error) { + parts := strings.Split(encodedHash, "$") + + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" { + return false, ErrInvalidHash + } + versionStr, ok := strings.CutPrefix(parts[2], "v=") + if !ok { + return false, ErrInvalidHash + } + + version, err := strconv.ParseUint(versionStr, 10, 32) + if err != nil || version != uint64(argon2.Version) { + return false, ErrInvalidHash + } + + params := strings.Split(parts[3], ",") + if len(params) != 3 { + return false, ErrInvalidHash + } + + parseParam := func(value, prefix string, bits int) (uint64, error) { + number, ok := strings.CutPrefix(value, prefix) + if !ok { + return 0, ErrInvalidHash + } + + return strconv.ParseUint(number, 10, bits) + } + + m, err := parseParam(params[0], "m=", 32) + if err != nil { + return false, ErrInvalidHash + } + + t, err := parseParam(params[1], "t=", 32) + if err != nil { + return false, ErrInvalidHash + } + + p, err := parseParam(params[2], "p=", 8) + if err != nil { + return false, ErrInvalidHash + } + + memory := uint32(m) + iterations := uint32(t) + parallelism := uint8(p) + + if memory < 8*uint32(parallelism) || + memory > 256*1024 || + iterations < 1 || iterations > 10 || + parallelism < 1 || parallelism > 8 { + return false, ErrInvalidHash + } + + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, ErrInvalidHash + } + + if len(salt) != saltLength { + return false, ErrInvalidHash + } + expectedHash, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, ErrInvalidHash + } + + if len(expectedHash) != hashLength { + return false, ErrInvalidHash + } + + computedHash := argon2.IDKey( + []byte(password), + salt, + iterations, + memory, + parallelism, + uint32(len(expectedHash)), + ) + + return subtle.ConstantTimeCompare(computedHash, expectedHash) == 1, nil + +} diff --git a/internal/service/attachment.go b/internal/service/attachment.go new file mode 100644 index 00000000..e69b3204 --- /dev/null +++ b/internal/service/attachment.go @@ -0,0 +1,207 @@ +package service + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "highload/internal/fault" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/storage" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +const MaxAttachmentBytes = 10 * 1024 * 1024 +const maxImagePixels = 40_000_000 +const attachmentObjectPrefix = "attachments/" + +type AttachmentService struct { + repo repository.AttachmentRepository + chatService *ChatService + objects storage.ObjectWriter +} + +func NewAttachmentService(repo repository.AttachmentRepository, chatService *ChatService, objects storage.ObjectWriter) *AttachmentService { + return &AttachmentService{repo: repo, chatService: chatService, objects: objects} +} + +func (s *AttachmentService) CreateAttachment(ctx context.Context, chatID, userID, filename, declaredContentType string, body io.ReadSeeker, size int64) (*model.Attachment, error) { + + chat, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, err + } + + mimeType, err := validateAttachment(filename, declaredContentType, body, size) + + if err != nil { + return nil, err + } + + id := uuid.NewString() + key := "staging/" + id + "/original" + + if err := ctx.Err(); err != nil { + return nil, err + } + sum := sha256.New() + n, err := io.Copy(sum, io.LimitReader(body, size+1)) + if err != nil || n != size { + return nil, ErrServerError + } + if _, err := body.Seek(0, io.SeekStart); err != nil { + return nil, ErrServerError + } + sumString := hex.EncodeToString(sum.Sum(nil)) + + var attachment *model.Attachment + fn := func(ctx context.Context, tx repository.AttachmentTx) error { + err := s.objects.Put(ctx, key, mimeType, body, size) + + if err != nil { + return convertStorageError(err) + } + + if err := fault.Pause(ctx, "after_staging", id); err != nil { + return err + } + attachment, err = tx.CreateQueued(ctx, id, chat.ID, userID, mimeType, sumString, key, filename, size) + return err + } + + acquired, err := s.repo.WithObjectKeyLock(ctx, key, fn) + + if err != nil { + return nil, convertRepoError(err) + } + + if !acquired { + return nil, ErrServiceUnavailable + } + + return attachment, nil + +} + +func (s *AttachmentService) GetAttachment(ctx context.Context, userID, attachmentID string) (*model.Attachment, error) { + _, err := uuid.Parse(attachmentID) + if err != nil { + return nil, ErrInvalidInput + } + + _, err = uuid.Parse(userID) + if err != nil { + return nil, ErrInvalidInput + } + + attachment, err := s.repo.GetAttachmentForUser(ctx, userID, attachmentID) + if err != nil { + return nil, convertRepoError(err) + } + + return attachment, nil +} + +func (s *AttachmentService) GetAttachmentContent(ctx context.Context, userID, attachmentID, variant string) (string, io.ReadCloser, error) { + attachment, err := s.GetAttachment(ctx, userID, attachmentID) + if err != nil { + return "", nil, err + } + if attachment.Status != "ready" { + return "", nil, ErrConflict + } + key, mime := attachment.ObjectKey, attachment.ContentType + if variant != "" && variant != "original" { + key = "" + for _, candidate := range attachment.Variants { + if candidate.Name == variant { + key, mime = candidate.ObjectKey, candidate.ContentType + break + } + } + } + if key == "" { + return "", nil, ErrNotFound + } + contentReader, err := s.objects.Get(ctx, key) + if err != nil { + return "", nil, convertStorageError(err) + } + return mime, contentReader, nil +} + +func validateAttachment(filename, declaredContentType string, body io.ReadSeeker, size int64) (string, error) { + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || utf8.RuneCountInString(filename) > 255 { + return "", ErrInvalidInput + } + + if strings.ContainsAny(filename, "/\\") || containsControlChars(filename) { + return "", ErrInvalidInput + } + + if size > MaxAttachmentBytes { + return "", ErrPayloadTooLarge + } + if size <= 0 { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + imageConfig, format, err := image.DecodeConfig(io.LimitReader(body, size)) + if err != nil { + return "", ErrUnsupportMediaType + } + + if imageConfig.Height <= 0 || imageConfig.Width <= 0 { + return "", ErrUnsupportMediaType + } + if imageConfig.Width > maxImagePixels/imageConfig.Height { + return "", ErrPayloadTooLarge + } + + var mimeType string + switch format { + case "png": + mimeType = "image/png" + case "jpeg": + mimeType = "image/jpeg" + default: + return "", ErrUnsupportMediaType + + } + if mimeType != declaredContentType { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + _, _, err = image.Decode(io.LimitReader(body, size)) + if err != nil { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + return mimeType, nil +} + +func containsControlChars(s string) bool { + for _, r := range s { + if (r >= 0 && r <= 31) || r == 127 { + return true + } + } + return false +} diff --git a/internal/service/attachment_cleanup.go b/internal/service/attachment_cleanup.go new file mode 100644 index 00000000..dd7cfc01 --- /dev/null +++ b/internal/service/attachment_cleanup.go @@ -0,0 +1,98 @@ +package service + +import ( + "context" + "highload/internal/repository" + "highload/internal/storage" + "strings" + "time" +) + +type AttachmentCleanupOptions struct { + GracePeriod time.Duration + DryRun bool + ListTimeout time.Duration +} + +type AttachmentCleanupService struct { + repo repository.AttachmentRepository + objects storage.ObjectCleanup + options *AttachmentCleanupOptions +} + +type OrphanReport struct { + Pages int `json:"s3_pages"` + Objects int `json:"objects_scanned"` + Checked int `json:"checked_in_db"` + Busy int `json:"busy"` + Candidates int `json:"orphan_candidates"` + Deleted int `json:"objects_deleted"` +} + +func NewAttachmentCleanupService(repo repository.AttachmentRepository, objects storage.ObjectCleanup, options *AttachmentCleanupOptions) *AttachmentCleanupService { + return &AttachmentCleanupService{repo: repo, objects: objects, options: options} +} + +func (s *AttachmentCleanupService) RunOnce(ctx context.Context) error { + _, err := s.ScanOrphans(ctx) + return err +} + +func (s *AttachmentCleanupService) ScanOrphans(ctx context.Context) (OrphanReport, error) { + var report OrphanReport + cutoff := time.Now().Add(-s.options.GracePeriod) + var token *string + + for { + if err := ctx.Err(); err != nil { + return report, err + } + + listCtx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + objects, nextToken, err := s.objects.ListPage(listCtx, token, attachmentObjectPrefix) + cancel() + if err != nil { + return report, err + } + report.Pages++ + + for _, obj := range objects { + report.Objects++ + if !strings.HasPrefix(obj.Key, attachmentObjectPrefix) || + obj.LastModified.IsZero() || !obj.LastModified.Before(cutoff) { + continue + } + + acquired, err := s.repo.WithObjectKeyLock(ctx, obj.Key, func(txCtx context.Context, txRepo repository.AttachmentTx) error { + exists, err := txRepo.ExistsByObjectKey(txCtx, obj.Key) + if err != nil { + return err + } + report.Checked++ + if exists { + return nil + } + report.Candidates++ + if s.options.DryRun { + return nil + } + if err := s.objects.Delete(txCtx, obj.Key); err != nil { + return err + } + report.Deleted++ + return nil + }) + if err != nil { + return report, err + } + if !acquired { + report.Busy++ + } + } + + if nextToken == nil { + return report, ctx.Err() + } + token = nextToken + } +} diff --git a/internal/service/attachment_reconcile.go b/internal/service/attachment_reconcile.go new file mode 100644 index 00000000..9cd554c1 --- /dev/null +++ b/internal/service/attachment_reconcile.go @@ -0,0 +1,162 @@ +package service + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "highload/internal/repository" + "highload/internal/storage" + "io" + "strings" + "time" +) + +const attachmentReconcilePageSize = 200 +const missingSampleLimit = 32 + +type attachmentReconcileStorage interface { + Check(context.Context) error + Head(context.Context, string) error + PutIfAbsent(context.Context, string, string, io.ReadSeeker, int64) error +} + +type AttachmentReconciler struct { + repo repository.AttachmentReconcileRepository + objects attachmentReconcileStorage + options AttachmentCleanupOptions +} + +type MissingReport struct { + Pages int `json:"db_pages"` + Rows int `json:"rows_scanned"` + Heads int `json:"s3_head_attempts"` + Missing int `json:"missing_objects"` + SampleIDs []string `json:"missing_attachment_ids"` +} + +func NewAttachmentReconciler(repo repository.AttachmentReconcileRepository, objects attachmentReconcileStorage, options AttachmentCleanupOptions) *AttachmentReconciler { + return &AttachmentReconciler{repo: repo, objects: objects, options: options} +} + +// ScanMissing checks old committed attachment rows using S3 metadata requests. +func (s *AttachmentReconciler) ScanMissing(ctx context.Context) (MissingReport, error) { + report := MissingReport{SampleIDs: []string{}} + cutoff := time.Now().Add(-s.options.GracePeriod) + if err := s.checkBucket(ctx); err != nil { + return report, err + } + var afterKey string + for { + rows, err := s.repo.ListAttachmentKeys(ctx, afterKey, attachmentReconcilePageSize) + if err != nil { + return report, err + } + report.Pages++ + for _, row := range rows { + report.Rows++ + if !strings.HasPrefix(row.ObjectKey, attachmentObjectPrefix) || + row.CreatedAt.IsZero() || !row.CreatedAt.Before(cutoff) { + continue + } + report.Heads++ + present, err := s.objectPresent(ctx, row.ObjectKey) + if err != nil { + return report, err + } + if !present { + report.Missing++ + if len(report.SampleIDs) < missingSampleLimit { + report.SampleIDs = append(report.SampleIDs, row.ID) + } + } + } + if len(rows) < attachmentReconcilePageSize { + return report, ctx.Err() + } + afterKey = rows[len(rows)-1].ObjectKey + } +} + +// Restore verifies the original bytes before writing a missing object under its existing key. +func (s *AttachmentReconciler) Restore(ctx context.Context, id string, original io.Reader) (string, error) { + attachment, err := s.repo.GetAttachmentByID(ctx, id) + if err != nil { + return "", err + } + if !strings.HasPrefix(attachment.ObjectKey, attachmentObjectPrefix) || + attachment.CreatedAt.IsZero() || !attachment.CreatedAt.Before(time.Now().Add(-s.options.GracePeriod)) { + return "", errors.New("attachment is outside the restore scope or still within grace") + } + if attachment.SizeBytes <= 0 || attachment.SizeBytes > MaxAttachmentBytes { + return "", errors.New("invalid original size in attachment metadata") + } + if err := s.checkBucket(ctx); err != nil { + return "", err + } + + // Read at most one original. Keep the verified bytes stable until Put completes. + data, err := io.ReadAll(io.LimitReader(original, attachment.SizeBytes+1)) + if err != nil { + return "", fmt.Errorf("read backup original: %w", err) + } + sum := sha256.Sum256(data) + if int64(len(data)) != attachment.SizeBytes || hex.EncodeToString(sum[:]) != attachment.SHA256 { + return "", errors.New("backup original does not match the stored size and SHA-256") + } + + var result string + acquired, err := s.repo.WithObjectKeyLock(ctx, attachment.ObjectKey, func(txCtx context.Context, txRepo repository.AttachmentTx) error { + current, err := txRepo.GetAttachmentByID(txCtx, id) + if err != nil { + return err + } + if current.ObjectKey != attachment.ObjectKey || current.SizeBytes != attachment.SizeBytes || + current.SHA256 != attachment.SHA256 || current.ContentType != attachment.ContentType || + !current.CreatedAt.Equal(attachment.CreatedAt) { + return errors.New("attachment metadata changed during restore") + } + present, err := s.objectPresent(txCtx, current.ObjectKey) + if err != nil { + return err + } + if present { + result = "already_present" + return nil + } + if s.options.DryRun { + result = "would_restore" + return nil + } + if err := s.objects.PutIfAbsent(txCtx, current.ObjectKey, current.ContentType, bytes.NewReader(data), current.SizeBytes); err != nil { + return err + } + result = "restored" + return nil + }) + if err != nil { + return "", err + } + if !acquired { + return "", errors.New("attachment key is busy; retry the restore") + } + return result, nil +} + +func (s *AttachmentReconciler) checkBucket(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + defer cancel() + return s.objects.Check(ctx) +} + +func (s *AttachmentReconciler) objectPresent(ctx context.Context, key string) (bool, error) { + headCtx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + defer cancel() + err := s.objects.Head(headCtx, key) + if errors.Is(err, storage.ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} diff --git a/internal/service/auth.go b/internal/service/auth.go new file mode 100644 index 00000000..a9cfeeca --- /dev/null +++ b/internal/service/auth.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/security" + "time" +) + +const tokenType = "Bearer" + +type AuthService struct { + userRepo repository.UserRepository + sessionRepo repository.SessionRepository + sessionTTL time.Duration +} + +func NewAuthService(userRepo repository.UserRepository, sessionRepo repository.SessionRepository, sessionTTL time.Duration) *AuthService { + return &AuthService{ + userRepo: userRepo, + sessionRepo: sessionRepo, + sessionTTL: sessionTTL, + } +} + +func (s *AuthService) Authenticate(ctx context.Context, username, password string) (*model.User, error) { + + userOK := checkUsername(username) + passwordOK := checkPassword(password) + + if !userOK || !passwordOK { + return nil, ErrUnauthorized + } + + user, err := s.userRepo.GetUserByUsername(ctx, username) + + if err != nil { + if errors.Is(err, repository.ErrNotFound) { + return nil, ErrUnauthorized + } + return nil, convertRepoError(err) + } + + isVerify, err := security.VerifyPassword(password, user.PasswordHash) + + if err != nil { + return nil, ErrServerError + } + + if !isVerify { + return nil, ErrUnauthorized + } + + return user, nil +} + +type CreatedSession struct { + Token string + TokenType string + ExpiresAt time.Time + User *model.User +} + +func (s *AuthService) CreateSession(ctx context.Context, username, password string) (*CreatedSession, error) { + user, err := s.Authenticate(ctx, username, password) + if err != nil { + return nil, err + } + + if s.sessionTTL <= 0 { + return nil, ErrServerError + } + + tokenBytes := make([]byte, 32) + _, err = rand.Read(tokenBytes) + if err != nil { + return nil, ErrServerError + } + + token := base64.RawURLEncoding.EncodeToString(tokenBytes) + + tokenHash := sha256.Sum256([]byte(token)) + + session, err := s.sessionRepo.Create(ctx, tokenHash[:], user.ID, s.sessionTTL) + + if err != nil { + return nil, convertRepoError(err) + } + + createdSession := &CreatedSession{ + Token: token, + TokenType: tokenType, + ExpiresAt: session.ExpiresAt, + User: user, + } + + return createdSession, nil + +} + +func (s *AuthService) ValidateSession(ctx context.Context, token string) (string, error) { + + if token == "" { + return "", ErrUnauthorized + } + + tokenHash := sha256.Sum256([]byte(token)) + + session, err := s.sessionRepo.GetSessionByTokenHash(ctx, tokenHash[:]) + if err != nil { + if errors.Is(err, repository.ErrNotFound) { + return "", ErrUnauthorized + } + return "", convertRepoError(err) + } + + if session.RevokedAt != nil { + return "", ErrUnauthorized + } + + if !time.Now().Before(session.ExpiresAt) { + return "", ErrUnauthorized + } + + return session.UserID, nil +} + +func (s *AuthService) RevokeSession(ctx context.Context, token string) error { + tokenHash := sha256.Sum256([]byte(token)) + + _, err := s.sessionRepo.RevokeSession(ctx, tokenHash[:]) + + if err != nil { + return convertRepoError(err) + } + return nil +} diff --git a/internal/service/chat.go b/internal/service/chat.go new file mode 100644 index 00000000..937eacd1 --- /dev/null +++ b/internal/service/chat.go @@ -0,0 +1,194 @@ +package service + +import ( + "context" + "errors" + "highload/internal/model" + "highload/internal/repository" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +type ChatService struct { + repo repository.ChatRepository + userRepo repository.UserRepository + userService *UserService +} + +func NewChatService(repo repository.ChatRepository, userRepo repository.UserRepository, userService *UserService) *ChatService { + return &ChatService{repo: repo, userRepo: userRepo, userService: userService} +} + +func (s *ChatService) Create(ctx context.Context, title string, memberIDs []string, userID string) (*model.Chat, error) { + + trimTitle := strings.TrimSpace(title) + + lenTitle := utf8.RuneCountInString(title) + if lenTitle < 1 || lenTitle > 100 { + return nil, ErrInvalidInput + } + + if trimTitle == "" { + return nil, ErrInvalidInput + } + + if memberIDs == nil || len(memberIDs) == 0 || len(memberIDs) > 100 { + return nil, ErrInvalidInput + } + + seen := make(map[string]struct{}) + for _, memberID := range memberIDs { + parsedID, err := uuid.Parse(memberID) + + if err != nil { + return nil, ErrInvalidInput + } + + normalizedID := parsedID.String() + + if _, exist := seen[normalizedID]; exist { + return nil, ErrInvalidInput + } + + seen[normalizedID] = struct{}{} + + _, err = s.userRepo.GetUserByID(ctx, memberID) + if err != nil { + // NOTE: MemberNotFound –> InvalidInput + if errors.Is(err, repository.ErrNotFound) { + return nil, ErrInvalidInput + } + return nil, convertRepoError(err) + } + } + + parsedUserID, err := uuid.Parse(userID) + if err != nil { + return nil, ErrServerError + } + normalizedUserID := parsedUserID.String() + + _, creatorExists := seen[normalizedUserID] + if !creatorExists { + return nil, ErrInvalidInput + } + + chat, err := s.repo.CreateChat(ctx, title, memberIDs) + + if err != nil { + return nil, convertRepoError(err) + } + + return chat, nil +} + +func (s *ChatService) GetChatsByUserID(ctx context.Context, userID string, limit *int, cursor *string) ([]*model.Chat, *string, error) { + + actualLimit := 50 + + if limit != nil { + actualLimit = *limit + } + + if actualLimit <= 0 || actualLimit > 100 { + return nil, nil, ErrInvalidInput + } + + var chatPosition *repository.ChatPosition + + if cursor != nil { + chCursor, err := decodeCursor[chatCursor](*cursor) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + _, err = uuid.Parse(chCursor.ChatID) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + if chCursor.CreatedAt.IsZero() { + return nil, nil, ErrInvalidInput + } + + _, err = uuid.Parse(chCursor.UserID) + if err != nil { + return nil, nil, ErrInvalidInput + } + + if chCursor.UserID != userID { + return nil, nil, ErrInvalidInput + } + + chatPosition = &repository.ChatPosition{ + CreatedAt: chCursor.CreatedAt, + ID: chCursor.ChatID, + } + } + + chats, err := s.repo.GetChatsByUserID(ctx, userID, actualLimit+1, chatPosition) + + if err != nil { + return nil, nil, convertRepoError(err) + } + + var chCursor *chatCursor + + if len(chats) > actualLimit { + chCursor = &chatCursor{ + ChatID: chats[actualLimit-1].ID, + UserID: userID, + CreatedAt: chats[actualLimit-1].CreatedAt, + } + cursor, err = encodeCursor[chatCursor](*chCursor) + if err != nil { + return nil, nil, ErrServerError + } + + return chats[:actualLimit], cursor, nil + } + + return chats, nil, nil + +} + +func (s *ChatService) GetChatByID(ctx context.Context, chatID, userID string) (*model.Chat, error) { + + _, err := uuid.Parse(chatID) + + if err != nil { + return nil, ErrInvalidInput + } + + _, err = uuid.Parse(userID) + + if err != nil { + return nil, ErrInvalidInput + } + + chat, err := s.repo.GetChatByID(ctx, chatID) + + if err != nil { + return nil, convertRepoError(err) + } + + index := slices.Index(chat.MemberIDs, userID) + + if index == -1 { + return nil, ErrNotFound + } + + return chat, nil +} + +type chatCursor struct { + CreatedAt time.Time `json:"created_at"` + ChatID string `json:"chat_id"` + UserID string `json:"user_id"` +} diff --git a/internal/service/errors.go b/internal/service/errors.go new file mode 100644 index 00000000..7c1be70b --- /dev/null +++ b/internal/service/errors.go @@ -0,0 +1,63 @@ +package service + +import ( + "errors" + "fmt" + "highload/internal/repository" + "highload/internal/storage" +) + +var ( + ErrInvalidInput = errors.New("invalid input") + ErrAlreadyExists = errors.New("user already exists") + ErrNotFound = errors.New("user not found") + ErrServerError = errors.New("server error") + ErrUnauthorized = errors.New("unauthorized") + ErrServiceUnavailable = errors.New("service unavailable") + ErrConflict = errors.New("data conflict") + ErrUnsupportMediaType = errors.New("unsupported mediaType") + ErrPayloadTooLarge = errors.New("payload too large") +) + +func convertRepoError(err error) error { + + var category error + + switch { + + case errors.Is(err, repository.ErrNotFound): + category = ErrNotFound + case errors.Is(err, repository.ErrAlreadyExists): + category = ErrAlreadyExists + case errors.Is(err, repository.ErrInvalidInput): + category = ErrInvalidInput + case errors.Is(err, repository.ErrDatabase) || errors.Is(err, repository.ErrServerError): + category = ErrServerError + case errors.Is(err, repository.ErrServiceUnavailable): + category = ErrServiceUnavailable + case errors.Is(err, repository.ErrConflict): + category = ErrConflict + default: + return err + } + + return fmt.Errorf("%w: %v", category, err) +} + +func convertStorageError(err error) error { + var category error + + switch { + case errors.Is(err, storage.ErrObjectNotFound): + category = ErrNotFound + case errors.Is(err, storage.ErrServiceUnavailable): + category = ErrServiceUnavailable + case errors.Is(err, storage.ErrStorage): + category = ErrServerError + default: + return err + } + + return fmt.Errorf("%w: %v", category, err) + +} diff --git a/internal/service/helper.go b/internal/service/helper.go new file mode 100644 index 00000000..764668cc --- /dev/null +++ b/internal/service/helper.go @@ -0,0 +1,63 @@ +package service + +import ( + "encoding/base64" + "encoding/json/v2" + "regexp" + "strings" + "unicode/utf8" +) + +func encodeCursor[C any](cursor C) (*string, error) { + data, err := json.Marshal(cursor) + + if err != nil { + return nil, err + } + encoded := base64.RawURLEncoding.EncodeToString(data) + + return &encoded, nil + +} + +func decodeCursor[C any](cursor string) (*C, error) { + data, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return nil, ErrInvalidInput + } + + var decoded C + + err = json.Unmarshal(data, &decoded) + + if err != nil { + return nil, ErrInvalidInput + } + + return &decoded, nil +} + +var usernameRegex = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func checkUsername(username string) bool { + return usernameRegex.MatchString(username) +} + +func checkPassword(password string) bool { + + if !utf8.ValidString(password) { + return false + } + + if strings.TrimSpace(password) == "" { + return false + } + + passwordLength := utf8.RuneCountInString(password) + if passwordLength < 12 || passwordLength > 128 { + return false + } + + return true + +} diff --git a/internal/service/message.go b/internal/service/message.go new file mode 100644 index 00000000..dfcff6d6 --- /dev/null +++ b/internal/service/message.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "highload/internal/model" + "highload/internal/repository" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +type MessageService struct { + repo repository.MessageRepository + userService *UserService + chatService *ChatService +} + +func NewMessageService(repo repository.MessageRepository, chatService *ChatService, userService *UserService) *MessageService { + return &MessageService{repo: repo, chatService: chatService, userService: userService} +} + +func (s *MessageService) CreateMessage(ctx context.Context, text, chatID, userID, key string, attachmentIDs []string) (*model.Message, error) { + + _, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, err + } + + lenText := utf8.RuneCountInString(text) + + if lenText > 4000 || (strings.TrimSpace(text) == "" && len(attachmentIDs) == 0) { + return nil, ErrInvalidInput + } + + if len(attachmentIDs) > 10 { + return nil, ErrInvalidInput + } + + seen := make(map[uuid.UUID]struct{}, len(attachmentIDs)) + for _, rawID := range attachmentIDs { + id, err := uuid.Parse(rawID) + if err != nil { + return nil, ErrInvalidInput + } + if _, exists := seen[id]; exists { + return nil, ErrInvalidInput + } + seen[id] = struct{}{} + + } + + message, err := s.repo.CreateMessage(ctx, chatID, userID, text, key, attachmentIDs) + + if err != nil { + return nil, convertRepoError(err) + } + + return message, nil +} + +func (s *MessageService) GetMessagesByChatID(ctx context.Context, chatID, userID string, limit *int, cursor *string) ([]*model.Message, *string, error) { + + actualLimit := 50 + + if limit != nil { + actualLimit = *limit + } + + if actualLimit <= 0 || actualLimit > 100 { + + return nil, nil, ErrInvalidInput + } + + chat, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, nil, err + } + + var messagePosition *repository.MessagePosition + + if cursor != nil { + mesCursor, err := decodeCursor[messageCursor](*cursor) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + if _, err := uuid.Parse(mesCursor.ChatID); err != nil { + return nil, nil, ErrInvalidInput + } + + if _, err := uuid.Parse(mesCursor.MessageID); err != nil { + return nil, nil, ErrInvalidInput + } + + if mesCursor.CreatedAt.IsZero() { + return nil, nil, ErrInvalidInput + } + + if mesCursor.ChatID != chat.ID { + return nil, nil, ErrInvalidInput + } + + messagePosition = &repository.MessagePosition{ + ID: mesCursor.MessageID, + CreatedAt: mesCursor.CreatedAt, + } + + } + + messages, err := s.repo.GetMessagesByChatID(ctx, chat.ID, actualLimit+1, messagePosition) + + if err != nil { + return nil, nil, convertRepoError(err) + } + + if len(messages) > actualLimit { + mesCursor := &messageCursor{ + MessageID: messages[actualLimit-1].ID, + ChatID: chat.ID, + CreatedAt: messages[actualLimit-1].CreatedAt, + } + cursor, err = encodeCursor[messageCursor](*mesCursor) + if err != nil { + return nil, nil, ErrServerError + } + return messages[:actualLimit], cursor, nil + } + + return messages, nil, nil + +} + +type messageCursor struct { + MessageID string `json:"message_id"` + ChatID string `json:"chat_id"` + CreatedAt time.Time `json:"created_at"` +} diff --git a/internal/service/user.go b/internal/service/user.go new file mode 100644 index 00000000..6556150a --- /dev/null +++ b/internal/service/user.go @@ -0,0 +1,70 @@ +package service + +import ( + "context" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/security" + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +type UserService struct { + repo repository.UserRepository +} + +func NewUserService(repo repository.UserRepository) *UserService { + return &UserService{repo} +} + +func (s *UserService) CreateUser(ctx context.Context, username, displayName, password string) (*model.User, error) { + + userOK := checkUsername(username) + passwordOK := checkPassword(password) + + if !userOK || !passwordOK { + return nil, ErrInvalidInput + } + + if !utf8.ValidString(displayName) { + return nil, ErrInvalidInput + } + if strings.TrimSpace(displayName) == "" { + return nil, ErrInvalidInput + } + + if utf8.RuneCountInString(displayName) > 100 { + return nil, ErrInvalidInput + } + + passwordHash, err := security.HashPassword(password) + if err != nil { + return nil, ErrServerError + } + + user, err := s.repo.Create(ctx, username, displayName, passwordHash) + if err != nil { + return nil, convertRepoError(err) + } + + return user, nil + +} + +func (s *UserService) GetUser(ctx context.Context, id string) (*model.User, error) { + + _, err := uuid.Parse(id) + if err != nil { + return nil, ErrInvalidInput + } + + user, err := s.repo.GetUserByID(ctx, id) + + if err != nil { + return nil, convertRepoError(err) + } + + return user, nil +} diff --git a/internal/storage/errors.go b/internal/storage/errors.go new file mode 100644 index 00000000..993ef825 --- /dev/null +++ b/internal/storage/errors.go @@ -0,0 +1,9 @@ +package storage + +import "errors" + +var ( + ErrServiceUnavailable = errors.New("object storage unavailable") + ErrStorage = errors.New("storage error") + ErrObjectNotFound = errors.New("object not found") +) diff --git a/internal/storage/object_cleanup_storage.go b/internal/storage/object_cleanup_storage.go new file mode 100644 index 00000000..43afe5b5 --- /dev/null +++ b/internal/storage/object_cleanup_storage.go @@ -0,0 +1,16 @@ +package storage + +import ( + "context" + "time" +) + +type ObjectCleanup interface { + ListPage(ctx context.Context, token *string, prefix string) ([]ObjectInfo, *string, error) + Delete(ctx context.Context, key string) error +} + +type ObjectInfo struct { + Key string + LastModified time.Time +} diff --git a/internal/storage/object_writer.go b/internal/storage/object_writer.go new file mode 100644 index 00000000..9c3b6239 --- /dev/null +++ b/internal/storage/object_writer.go @@ -0,0 +1,11 @@ +package storage + +import ( + "context" + "io" +) + +type ObjectWriter interface { + Put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error + Get(ctx context.Context, key string) (io.ReadCloser, error) +} diff --git a/internal/storage/s3/client.go b/internal/storage/s3/client.go new file mode 100644 index 00000000..04b7cc5a --- /dev/null +++ b/internal/storage/s3/client.go @@ -0,0 +1,160 @@ +package s3 + +import ( + "context" + "fmt" + "highload/internal/config" + "highload/internal/storage" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type Storage struct { + Client *awss3.Client + Bucket string +} + +func NewStorage(ctx context.Context, cfg *config.S3) (*Storage, error) { + provider := credentials.NewStaticCredentialsProvider( + cfg.AccessKeyID, + cfg.SecretAccessKey, + "", + ) + sdkConfig, err := awsconfig.LoadDefaultConfig( + ctx, + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider(provider), + ) + + if err != nil { + return nil, err + } + + client := awss3.NewFromConfig(sdkConfig, func(opts *awss3.Options) { + opts.BaseEndpoint = aws.String(cfg.Endpoint) + opts.UsePathStyle = true + }) + + return &Storage{ + Client: client, + Bucket: cfg.Bucket, + }, nil +} + +func (s *Storage) Check(ctx context.Context) error { + _, err := s.Client.HeadBucket(ctx, &awss3.HeadBucketInput{ + Bucket: aws.String(s.Bucket), + }) + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Head(ctx context.Context, key string) error { + _, err := s.Client.HeadObject(ctx, &awss3.HeadObjectInput{ + Bucket: &s.Bucket, + Key: &key, + }) + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error { + return s.put(ctx, key, contentType, body, size, nil) +} + +func (s *Storage) PutIfAbsent(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error { + return s.put(ctx, key, contentType, body, size, aws.String("*")) +} + +func (s *Storage) put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64, ifNoneMatch *string) error { + _, err := s.Client.PutObject(ctx, &awss3.PutObjectInput{ + Bucket: &s.Bucket, + Key: &key, + Body: body, + ContentLength: aws.Int64(size), + IfNoneMatch: ifNoneMatch, + ContentType: &contentType}) + + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) { + params := &awss3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &key, + } + objectOutput, err := s.Client.GetObject(ctx, params) + + if err != nil { + return nil, convertS3Error(err) + } + + return objectOutput.Body, nil + +} + +func (s *Storage) Delete(ctx context.Context, key string) error { + + params := &awss3.DeleteObjectInput{ + Bucket: &s.Bucket, + Key: &key, + } + _, err := s.Client.DeleteObject(ctx, params) + + if err != nil { + return convertS3Error(err) + } + + return nil + +} + +func (s *Storage) ListPage(ctx context.Context, token *string, prefix string) ([]storage.ObjectInfo, *string, error) { + input := awss3.ListObjectsV2Input{ + Bucket: &s.Bucket, + Prefix: aws.String(prefix), + ContinuationToken: token, + } + + output, err := s.Client.ListObjectsV2(ctx, &input) + + if err != nil { + return nil, nil, convertS3Error(err) + } + + var objects []storage.ObjectInfo + + for _, obj := range output.Contents { + if obj.Key == nil || obj.LastModified == nil || *obj.Key == "" { + continue + } + + objects = append(objects, storage.ObjectInfo{ + Key: *obj.Key, + LastModified: *obj.LastModified, + }) + } + if !aws.ToBool(output.IsTruncated) { + return objects, nil, nil + } + + nextToken := output.NextContinuationToken + + if nextToken == nil || aws.ToString(nextToken) == "" || aws.ToString(nextToken) == aws.ToString(token) { + return nil, nil, fmt.Errorf("%w: invalid next token", storage.ErrStorage) + } + + return objects, nextToken, nil +} diff --git a/internal/storage/s3/errors.go b/internal/storage/s3/errors.go new file mode 100644 index 00000000..353a969d --- /dev/null +++ b/internal/storage/s3/errors.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "highload/internal/storage" + + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +func convertS3Error(err error) error { + category := storage.ErrStorage + + var sendErr *smithyhttp.RequestSendError + var responseErr *smithyhttp.ResponseError + + switch { + case errors.Is(err, context.Canceled): + //Ошибка storage.ErrStorage + case errors.Is(err, context.DeadlineExceeded) || errors.As(err, &sendErr): + category = storage.ErrServiceUnavailable + + case errors.As(err, &responseErr): + code := responseErr.HTTPStatusCode() + switch code { + case 404: + category = storage.ErrObjectNotFound + case 429, 500, 503, 504, 502: + category = storage.ErrServiceUnavailable + } + } + + return fmt.Errorf("%w: %v", category, err) +} diff --git a/internal/transport/http/handler/attachment.go b/internal/transport/http/handler/attachment.go new file mode 100644 index 00000000..875065a8 --- /dev/null +++ b/internal/transport/http/handler/attachment.go @@ -0,0 +1,159 @@ +package handler + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "io" + "log/slog" + "net/http" + "os" + "time" +) + +type AttachmentHandler struct { + attachmentService *service.AttachmentService + uploadSlots chan struct{} +} + +func NewAttachmentHandler(attachmentService *service.AttachmentService) *AttachmentHandler { + uploadSlots := make(chan struct{}, 1) + return &AttachmentHandler{attachmentService: attachmentService, uploadSlots: uploadSlots} +} + +func (h *AttachmentHandler) CreateAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + chatID := r.PathValue("chat_id") + filename := r.URL.Query().Get("filename") + contentType := r.Header.Get("Content-Type") + + select { + case h.uploadSlots <- struct{}{}: + defer func() { <-h.uploadSlots }() + case <-time.After(1 * time.Second): + w.Header().Set("Retry-After", "1") + httputils.WriteError(w, http.StatusTooManyRequests, httputils.ErrorDetails{ + Code: "rate_limited", + Message: "too many concurrent uploads, try again later", + RequestID: requestID, + }) + return + + case <-r.Context().Done(): + return + } + + file, err := os.CreateTemp("", "attachment-*") + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + defer file.Close() + if err := os.Remove(file.Name()); err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + body := http.MaxBytesReader(w, r.Body, service.MaxAttachmentBytes) + defer body.Close() + size, err := io.Copy(file, uploadBodyReader{body}) + + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + httputils.WriteError(w, http.StatusRequestEntityTooLarge, httputils.ErrorDetails{ + Code: "payload_too_large", + Message: "payload_too_large", + RequestID: requestID, + }) + return + } + if err != nil { + var readErr *uploadBodyReadError + if !errors.As(err, &readErr) { + httputils.WriteServiceError(w, err, requestID) + return + } + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "invalid_request", + RequestID: requestID, + }) + return + } + + attachment, err := h.attachmentService.CreateAttachment(r.Context(), chatID, userID, filename, contentType, file, size) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertAttachmentModelToResponse(attachment) + wrErr := httputils.WriteJSON[attachmentResponse](w, http.StatusAccepted, *response) + if wrErr != nil { + slog.Error("failed to write create attachment response", "error", wrErr, "requestID", requestID) + } + +} + +type uploadBodyReader struct{ io.Reader } + +func (r uploadBodyReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + if err != nil && err != io.EOF { + return n, &uploadBodyReadError{err} + } + return n, err +} + +type uploadBodyReadError struct{ cause error } + +func (e *uploadBodyReadError) Error() string { return e.cause.Error() } +func (e *uploadBodyReadError) Unwrap() error { return e.cause } + +func (h *AttachmentHandler) GetAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + attachmentID := r.PathValue("attachment_id") + + attachment, err := h.attachmentService.GetAttachment(r.Context(), userID, attachmentID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + wrErr := httputils.WriteJSON[attachmentResponse](w, http.StatusOK, *convertAttachmentModelToResponse(attachment)) + if wrErr != nil { + slog.Error("failed to write get attachment response", "error", wrErr, "requestID", requestID) + } +} + +func (h *AttachmentHandler) GetAttachmentContent(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + attachmentID := r.PathValue("attachment_id") + variant := r.URL.Query().Get("variant") + + contentType, stream, err := h.attachmentService.GetAttachmentContent(r.Context(), userID, attachmentID, variant) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + defer stream.Close() + + w.Header().Set("Content-Type", contentType) + + _, err = io.Copy(w, stream) + + if err != nil { + slog.Error("failed to copy attachment content to w", "error", err, "request_id", requestID) + } + +} diff --git a/internal/transport/http/handler/attachment_types.go b/internal/transport/http/handler/attachment_types.go new file mode 100644 index 00000000..0931a828 --- /dev/null +++ b/internal/transport/http/handler/attachment_types.go @@ -0,0 +1,54 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type attachmentResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []attachmentVariantResponse `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +type attachmentVariantResponse struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +func convertAttachmentModelToResponse(attachment *model.Attachment) *attachmentResponse { + variants := make([]attachmentVariantResponse, 0, len(attachment.Variants)) + for _, v := range attachment.Variants { + variants = append(variants, attachmentVariantResponse{ + Name: v.Name, ContentType: v.ContentType, SizeBytes: v.SizeBytes, + Width: v.Width, Height: v.Height, SHA256: v.SHA256, + }) + } + return &attachmentResponse{ + ID: attachment.ID, + ChatID: attachment.ChatID, + OwnerID: attachment.OwnerID, + Filename: attachment.Filename, + ContentType: attachment.ContentType, + SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, + Status: attachment.Status, + Variants: variants, + ErrorCode: attachment.ErrorCode, + CreatedAt: httputils.FormatTime(attachment.CreatedAt), + UpdatedAt: httputils.FormatTime(attachment.UpdatedAt), + } +} diff --git a/internal/transport/http/handler/auth.go b/internal/transport/http/handler/auth.go new file mode 100644 index 00000000..78d46e5a --- /dev/null +++ b/internal/transport/http/handler/auth.go @@ -0,0 +1,94 @@ +package handler + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "io" + "net/http" +) + +type AuthHandler struct { + authService *service.AuthService +} + +func NewAuthHandler(authService *service.AuthService) *AuthHandler { + return &AuthHandler{authService: authService} +} + +func (h *AuthHandler) CreateSession(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + username, password, ok := r.BasicAuth() + + if !ok { + httputils.WriteBasicUnauthorized(w, requestID) + return + } + + var probe [1]byte + + n, err := r.Body.Read(probe[:]) + + if n > 0 || (err != nil && !errors.Is(err, io.EOF)) { + _ = httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "Request body must be empty", + RequestID: requestID, + }) + return + } + + createdSession, err := h.authService.CreateSession(r.Context(), username, password) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBasicUnauthorized(w, requestID) + return + } + + httputils.WriteServiceError(w, err, requestID) + return + } + + response := createSessionResponse{ + Token: createdSession.Token, + TokenType: createdSession.TokenType, + ExpiresAt: httputils.FormatTime(createdSession.ExpiresAt), + User: UserResponse{ + Username: createdSession.User.Username, + ID: createdSession.User.ID, + DisplayName: createdSession.User.DisplayName, + CreatedAt: httputils.FormatTime(createdSession.User.CreatedAt), + }, + } + + httputils.WriteJSON[createSessionResponse](w, http.StatusCreated, response) + +} + +func (h *AuthHandler) DeleteSession(w http.ResponseWriter, r *http.Request) { + + requestID := middleware.RequestIDFromContext(r.Context()) + + token, ok := httputils.ExtractBearerToken(r) + + if !ok { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + + err := h.authService.RevokeSession(r.Context(), token) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + httputils.WriteServiceError(w, err, requestID) + return + } + + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/transport/http/handler/auth_types.go b/internal/transport/http/handler/auth_types.go new file mode 100644 index 00000000..a1b0b2b8 --- /dev/null +++ b/internal/transport/http/handler/auth_types.go @@ -0,0 +1,15 @@ +package handler + +type UserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type createSessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User UserResponse `json:"user"` +} diff --git a/internal/transport/http/handler/chat.go b/internal/transport/http/handler/chat.go new file mode 100644 index 00000000..9b98eac0 --- /dev/null +++ b/internal/transport/http/handler/chat.go @@ -0,0 +1,102 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type ChatHandler struct { + chatService *service.ChatService +} + +func NewChatHandler(chatService *service.ChatService) *ChatHandler { + return &ChatHandler{chatService: chatService} +} + +func (h *ChatHandler) CreateChat(w http.ResponseWriter, r *http.Request) { + + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + req, code := httputils.Decode[chatCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + result, err := h.chatService.Create(r.Context(), req.Title, req.MemberIDs, userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := chatResponse{ + Title: result.Title, + MemberIDs: result.MemberIDs, + ID: result.ID, + CreatedAt: httputils.FormatTime(result.CreatedAt), + } + + _ = httputils.WriteJSON[chatResponse](w, http.StatusCreated, response) + +} + +func (h *ChatHandler) GetChatsByUserID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + userID := middleware.UserIDFromContext(r.Context()) + + limit, cursor, ok := httputils.GetLimitAndCursor(r) + + if !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + chats, nextCursor, err := h.chatService.GetChatsByUserID(r.Context(), userID, limit, cursor) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + chatResponses := make([]chatResponse, len(chats)) + for index, chat := range chats { + chatResponses[index] = *convertChatModelToResponse(chat) + } + + response := chatGetByUserIDResponse{ + Items: chatResponses, + NextCursor: nextCursor, + } + + _ = httputils.WriteJSON[chatGetByUserIDResponse](w, http.StatusOK, response) + +} + +func (h *ChatHandler) GetChatByChatID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + userID := middleware.UserIDFromContext(r.Context()) + + chatID := r.PathValue("chat_id") + + chat, err := h.chatService.GetChatByID(r.Context(), chatID, userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertChatModelToResponse(chat) + _ = httputils.WriteJSON[chatResponse](w, http.StatusOK, *response) + +} diff --git a/internal/transport/http/handler/chat_types.go b/internal/transport/http/handler/chat_types.go new file mode 100644 index 00000000..0580ee11 --- /dev/null +++ b/internal/transport/http/handler/chat_types.go @@ -0,0 +1,32 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type chatCreateRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type chatResponse struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + ID string `json:"id"` + CreatedAt string `json:"created_at"` +} + +type chatGetByUserIDResponse struct { + Items []chatResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +func convertChatModelToResponse(model *model.Chat) *chatResponse { + return &chatResponse{ + Title: model.Title, + ID: model.ID, + MemberIDs: model.MemberIDs, + CreatedAt: httputils.FormatTime(model.CreatedAt), + } +} diff --git a/internal/transport/http/handler/health.go b/internal/transport/http/handler/health.go new file mode 100644 index 00000000..85aa5c19 --- /dev/null +++ b/internal/transport/http/handler/health.go @@ -0,0 +1,86 @@ +package handler + +import ( + "context" + "highload/internal/storage/s3" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "log/slog" + "net/http" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +type HealthHandler struct { + pool *pgxpool.Pool + redisClient *redis.Client + s3Storage *s3.Storage +} + +func NewHealthHandler(pool *pgxpool.Pool, redisClient *redis.Client, s3Storage *s3.Storage) *HealthHandler { + return &HealthHandler{pool: pool, redisClient: redisClient, s3Storage: s3Storage} +} + +func (h *HealthHandler) Live(w http.ResponseWriter, r *http.Request) { + response := HealthResponse{Status: "ok"} + err := httputils.WriteJSON[HealthResponse](w, http.StatusOK, response) + if err != nil { + slog.Error("failed to write health response", "error", err) + } +} + +func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + requestID := middleware.RequestIDFromContext(r.Context()) + if err := h.pool.Ping(ctx); err != nil { + slog.Error("database readiness check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + if err := h.redisClient.Ping(ctx).Err(); err != nil { + slog.Error("redis readiness check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + if err := h.s3Storage.Check(ctx); err != nil { + slog.Error("s3 storage check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + response := HealthResponse{Status: "ok"} + err := httputils.WriteJSON[HealthResponse](w, http.StatusOK, response) + if err != nil { + slog.Error("failed to write health response", "error", err) + } + +} + +func (h *HealthHandler) NotFoundRoute(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + err := httputils.WriteError( + w, + http.StatusNotFound, + httputils.ErrorDetails{ + Code: "not_found", + Message: "Not Found", + RequestID: requestID, + }, + ) + if err != nil { + slog.Error("failed to write not found response", "error", err) + } +} diff --git a/internal/transport/http/handler/health_types.go b/internal/transport/http/handler/health_types.go new file mode 100644 index 00000000..e0b38e89 --- /dev/null +++ b/internal/transport/http/handler/health_types.go @@ -0,0 +1,5 @@ +package handler + +type HealthResponse struct { + Status string `json:"status"` +} diff --git a/internal/transport/http/handler/message.go b/internal/transport/http/handler/message.go new file mode 100644 index 00000000..5574faa7 --- /dev/null +++ b/internal/transport/http/handler/message.go @@ -0,0 +1,90 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type MessageHandler struct { + messageService *service.MessageService +} + +func NewMessageHandler(messageService *service.MessageService) *MessageHandler { + return &MessageHandler{messageService: messageService} +} + +func (h *MessageHandler) CreateMessage(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + chatID := r.PathValue("chat_id") + + req, code := httputils.Decode[messageCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + key := r.Header.Get("Idempotency-Key") + + if ok := httputils.CheckIdempotencyKeyPattern(key); !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "wrong Idempotency-Key", + RequestID: requestID, + }) + return + } + + message, err := h.messageService.CreateMessage(r.Context(), req.Text, chatID, userID, key, req.AttachmentIDs) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertMessageModelToResponse(message) + + _ = httputils.WriteJSON[messageResponse](w, http.StatusCreated, *response) + +} + +func (h *MessageHandler) GetMessagesByChatID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + chatID := r.PathValue("chat_id") + + limit, cursor, ok := httputils.GetLimitAndCursor(r) + + if !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + messages, nextCursor, err := h.messageService.GetMessagesByChatID(r.Context(), chatID, userID, limit, cursor) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + messageResponses := make([]messageResponse, len(messages)) + for index, message := range messages { + messageResponses[index] = *convertMessageModelToResponse(message) + } + + response := messageGetResponse{ + Items: messageResponses, + NextCursor: nextCursor, + } + + _ = httputils.WriteJSON[messageGetResponse](w, http.StatusOK, response) + +} diff --git a/internal/transport/http/handler/message_types.go b/internal/transport/http/handler/message_types.go new file mode 100644 index 00000000..09a431ab --- /dev/null +++ b/internal/transport/http/handler/message_types.go @@ -0,0 +1,42 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type messageCreateRequest struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type messageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type messageGetResponse struct { + Items []messageResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +func convertMessageModelToResponse(message *model.Message) *messageResponse { + attachmentIDs := message.AttachmentIDs + + if attachmentIDs == nil { + attachmentIDs = []string{} + } + + return &messageResponse{ + ID: message.ID, + ChatID: message.ChatID, + SenderID: message.SenderID, + Text: message.Text, + AttachmentIDs: attachmentIDs, + CreatedAt: httputils.FormatTime(message.CreatedAt), + } +} diff --git a/internal/transport/http/handler/user.go b/internal/transport/http/handler/user.go new file mode 100644 index 00000000..517a48e7 --- /dev/null +++ b/internal/transport/http/handler/user.go @@ -0,0 +1,60 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type UserHandler struct { + userService *service.UserService +} + +func NewUserHandler(service *service.UserService) *UserHandler { + return &UserHandler{userService: service} +} + +func (h *UserHandler) CreateUser(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + req, code := httputils.Decode[userCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + user, err := h.userService.CreateUser(r.Context(), req.Username, req.DisplayName, req.Password) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertUserModelToResponse(user) + + _ = httputils.WriteJSON[userResponse](w, http.StatusCreated, *response) + +} + +func (h *UserHandler) GetCurrentUser(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + user, err := h.userService.GetUser(r.Context(), userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertUserModelToResponse(user) + + _ = httputils.WriteJSON[userResponse](w, http.StatusOK, *response) + +} diff --git a/internal/transport/http/handler/user_types.go b/internal/transport/http/handler/user_types.go new file mode 100644 index 00000000..5a404e6d --- /dev/null +++ b/internal/transport/http/handler/user_types.go @@ -0,0 +1,28 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type userCreateRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type userResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +func convertUserModelToResponse(user *model.User) *userResponse { + return &userResponse{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + CreatedAt: httputils.FormatTime(user.CreatedAt), + } +} diff --git a/internal/transport/http/httputils/helper.go b/internal/transport/http/httputils/helper.go new file mode 100644 index 00000000..73ca6462 --- /dev/null +++ b/internal/transport/http/httputils/helper.go @@ -0,0 +1,207 @@ +package httputils + +import ( + "encoding/json/v2" + "errors" + "highload/internal/service" + "io" + "log/slog" + "mime" + "net/http" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type ErrorResponse struct { + Error ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +func Decode[T any](r *http.Request) (*T, int) { + contentType := r.Header.Get("Content-Type") + + mediaType, _, err := mime.ParseMediaType(contentType) + + if err != nil || mediaType != "application/json" { + return nil, http.StatusUnsupportedMediaType + } + + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, http.StatusBadRequest + } + + if !utf8.Valid(body) { + return nil, http.StatusBadRequest + } + + req := new(T) + + err = json.Unmarshal(body, req, json.RejectUnknownMembers(true)) + if err != nil { + return nil, http.StatusBadRequest + } + return req, http.StatusOK + +} + +func GetLimitAndCursor(r *http.Request) (limit *int, cursor *string, ok bool) { + query := r.URL.Query() + + var currentLimit *int + + if query.Has("limit") { + limitStr := query.Get("limit") + + if limitStr == "" { + return nil, nil, false + } + limitInt, err := strconv.Atoi(limitStr) + + if err != nil { + return nil, nil, false + } + + currentLimit = &limitInt + + } + + var currentCursor *string + if query.Has("cursor") { + cursor := query.Get("cursor") + currentCursor = &cursor + } + + return currentLimit, currentCursor, true +} + +func WriteJSON[T any](w http.ResponseWriter, status int, response T) error { + data, err := json.Marshal(response) + if err != nil { + return err + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, err = w.Write(data) + return err +} + +func ErrorFromService(err error) (int, string) { + + switch { + case errors.Is(err, service.ErrInvalidInput): + return http.StatusBadRequest, "invalid_request" + + case errors.Is(err, service.ErrNotFound): + return http.StatusNotFound, "not_found" + + case errors.Is(err, service.ErrAlreadyExists): + return http.StatusConflict, "conflict" + case errors.Is(err, service.ErrUnauthorized): + return http.StatusUnauthorized, "unauthorized" + case errors.Is(err, service.ErrServiceUnavailable): + return http.StatusServiceUnavailable, "unavailable" + case errors.Is(err, service.ErrConflict): + return http.StatusConflict, "conflict" + case errors.Is(err, service.ErrPayloadTooLarge): + return http.StatusRequestEntityTooLarge, "payload_too_large" + case errors.Is(err, service.ErrUnsupportMediaType): + return http.StatusUnsupportedMediaType, "unsupported_media_type" + default: + return http.StatusInternalServerError, "internal_error" + } +} + +func WriteBasicUnauthorized(w http.ResponseWriter, requestID string) error { + w.Header().Set( + "WWW-Authenticate", + `Basic realm="messenger", charset="UTF-8"`, + ) + + return WriteError(w, http.StatusUnauthorized, ErrorDetails{ + Code: "unauthorized", + Message: "Unauthorized", + RequestID: requestID, + }) +} + +func WriteBearerUnauthorized(w http.ResponseWriter, requestID string) error { + w.Header().Set( + "WWW-Authenticate", + `Bearer realm="messenger"`, + ) + + return WriteError(w, http.StatusUnauthorized, ErrorDetails{ + Code: "unauthorized", + Message: "Unauthorized", + RequestID: requestID, + }) +} + +func WriteError(w http.ResponseWriter, code int, errorDetails ErrorDetails) error { + return WriteJSON(w, code, ErrorResponse{Error: errorDetails}) + +} + +func WriteServiceUnavailable(w http.ResponseWriter, requestID string) error { + return WriteError( + w, + http.StatusServiceUnavailable, + ErrorDetails{ + Message: "unavailable", + Code: "unavailable", + RequestID: requestID, + }, + ) +} + +func WriteServiceError(w http.ResponseWriter, err error, requestID string) error { + status, code := ErrorFromService(err) + + if status >= http.StatusInternalServerError { + slog.Error("request failed", + "error", err, + "request_id", requestID, + "status", status, + "code", code, + ) + } + + return WriteJSON(w, status, ErrorResponse{ + Error: ErrorDetails{ + Code: code, + Message: http.StatusText(status), + RequestID: requestID, + }, + }) +} + +func ExtractBearerToken(r *http.Request) (string, bool) { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || + !strings.EqualFold(scheme, "Bearer") || + token == "" || + strings.ContainsAny(token, " \t\r\n") { + + return "", false + } + return token, true +} + +func FormatTime(t time.Time) string { + return t.UTC().Format(time.RFC3339Nano) +} + +func CheckIdempotencyKeyPattern(key string) bool { + patternReg := regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + return patternReg.MatchString(key) +} diff --git a/internal/transport/http/middleware/auth.go b/internal/transport/http/middleware/auth.go new file mode 100644 index 00000000..69bbec7d --- /dev/null +++ b/internal/transport/http/middleware/auth.go @@ -0,0 +1,71 @@ +package middleware + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "net/http" + + "github.com/google/uuid" +) + +type Middleware struct { + authService *service.AuthService + instanceID string +} + +func NewMiddleware(authService *service.AuthService, instanceID string) *Middleware { + + return &Middleware{ + authService: authService, + instanceID: instanceID, + } + +} + +func (m *Middleware) Auth(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := RequestIDFromContext(r.Context()) + + token, ok := httputils.ExtractBearerToken(r) + + if !ok { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + + userID, err := m.authService.ValidateSession(r.Context(), token) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + httputils.WriteServiceError(w, err, requestID) + return + } + + ctx := WithUserID(r.Context(), userID) + next.ServeHTTP(w, r.WithContext(ctx)) + + }) +} + +func (m *Middleware) RequestID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := uuid.NewString() + + w.Header().Set("X-Request-Id", requestID) + + ctx := WithRequestID(r.Context(), requestID) + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func (m *Middleware) InstanceID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", m.instanceID) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/transport/http/middleware/context.go b/internal/transport/http/middleware/context.go new file mode 100644 index 00000000..6419c4ac --- /dev/null +++ b/internal/transport/http/middleware/context.go @@ -0,0 +1,29 @@ +package middleware + +import "context" + +// UserID +type contextKey string + +const userIDKey contextKey = "userID" + +func WithUserID(ctx context.Context, userID string) context.Context { + return context.WithValue(ctx, userIDKey, userID) +} + +func UserIDFromContext(ctx context.Context) string { + userID, _ := ctx.Value(userIDKey).(string) + return userID +} + +// RequestIDKey +const requestIDKey contextKey = "requestID" + +func RequestIDFromContext(ctx context.Context) string { + requestID, _ := ctx.Value(requestIDKey).(string) + return requestID +} + +func WithRequestID(ctx context.Context, requestID string) context.Context { + return context.WithValue(ctx, requestIDKey, requestID) +} diff --git a/internal/transport/http/middleware/logging.go b/internal/transport/http/middleware/logging.go new file mode 100644 index 00000000..a3675566 --- /dev/null +++ b/internal/transport/http/middleware/logging.go @@ -0,0 +1,39 @@ +package middleware + +import ( + "log/slog" + "net/http" + "time" +) + +type responseWriter struct { + http.ResponseWriter + status int +} + +func (w *responseWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func (m *Middleware) Logging(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + rw := &responseWriter{ + ResponseWriter: w, + status: http.StatusOK, + } + + next.ServeHTTP(rw, r) + + slog.Info( + "http request", + "method", r.Method, + "path", r.URL.Path, + "status", rw.status, + "duration", time.Since(start), + "request_id", RequestIDFromContext(r.Context()), + ) + }) +} diff --git a/internal/transport/http/router.go b/internal/transport/http/router.go new file mode 100644 index 00000000..634a5acd --- /dev/null +++ b/internal/transport/http/router.go @@ -0,0 +1,57 @@ +package http + +import ( + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + "net/http" +) + +type Router struct { + middleware *middleware.Middleware + authHandler *handler.AuthHandler + healthHandler *handler.HealthHandler + chatHandler *handler.ChatHandler + userHandler *handler.UserHandler + messageHandler *handler.MessageHandler + attachmentHandler *handler.AttachmentHandler +} + +func NewRouter(middleware *middleware.Middleware, authHandler *handler.AuthHandler, healthHandler *handler.HealthHandler, chatHandler *handler.ChatHandler, userHandler *handler.UserHandler, messageHandler *handler.MessageHandler, attachmentHandler *handler.AttachmentHandler) *Router { + return &Router{chatHandler: chatHandler, authHandler: authHandler, healthHandler: healthHandler, middleware: middleware, userHandler: userHandler, messageHandler: messageHandler, attachmentHandler: attachmentHandler} +} + +func (r *Router) Handler() http.Handler { + mux := http.NewServeMux() + protectedMux := http.NewServeMux() + + protectedMux.HandleFunc("/", r.healthHandler.NotFoundRoute) + mux.HandleFunc("GET /health/live", r.healthHandler.Live) + mux.HandleFunc("GET /health/ready", r.healthHandler.Ready) + + //Auth + mux.HandleFunc("POST /api/v1/auth/sessions", r.authHandler.CreateSession) + protectedMux.HandleFunc("DELETE /api/v1/auth/sessions/current", r.authHandler.DeleteSession) + + //User + mux.HandleFunc("POST /api/v1/users", r.userHandler.CreateUser) + protectedMux.HandleFunc("GET /api/v1/users/me", r.userHandler.GetCurrentUser) + + //Chat + protectedMux.HandleFunc("POST /api/v1/chats", r.chatHandler.CreateChat) + protectedMux.HandleFunc("GET /api/v1/chats", r.chatHandler.GetChatsByUserID) + protectedMux.HandleFunc("GET /api/v1/chats/{chat_id}", r.chatHandler.GetChatByChatID) + + //Message + protectedMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", r.messageHandler.CreateMessage) + protectedMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", r.messageHandler.GetMessagesByChatID) + + //Attachment + protectedMux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", r.attachmentHandler.CreateAttachment) + protectedMux.HandleFunc("GET /api/v1/attachments/{attachment_id}", r.attachmentHandler.GetAttachment) + protectedMux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", r.attachmentHandler.GetAttachmentContent) + + mux.Handle("/api/v1/", r.middleware.Auth(protectedMux)) + return r.middleware.InstanceID(r.middleware.RequestID( + r.middleware.Logging(mux), + )) +} diff --git a/scripts/init-local.sh b/scripts/init-local.sh new file mode 100755 index 00000000..994e319a --- /dev/null +++ b/scripts/init-local.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu +umask 077 + +cd "$(dirname "$0")/.." +command -v openssl >/dev/null 2>&1 || { echo 'openssl is required' >&2; exit 1; } +mkdir -p secrets + +if [ ! -e secrets/postgres_password ] && [ ! -e secrets/database_url ]; then + password=$(openssl rand -hex 24) + printf '%s' "$password" > secrets/postgres_password + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$password" > secrets/database_url +elif [ ! -s secrets/postgres_password ] || [ ! -s secrets/database_url ]; then + echo 'incomplete PostgreSQL secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e secrets/rustfs_root_access_key ] && [ ! -e secrets/rustfs_root_secret_key ]; then + openssl rand -hex 16 > secrets/rustfs_root_access_key + openssl rand -hex 32 > secrets/rustfs_root_secret_key +elif [ ! -s secrets/rustfs_root_access_key ] || [ ! -s secrets/rustfs_root_secret_key ]; then + echo 'incomplete RustFS root secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e secrets/s3_access_key_id ] && [ ! -e secrets/s3_secret_access_key ]; then + openssl rand -hex 16 > secrets/s3_access_key_id + openssl rand -hex 32 > secrets/s3_secret_access_key +elif [ ! -s secrets/s3_access_key_id ] || [ ! -s secrets/s3_secret_access_key ]; then + echo 'incomplete API S3 secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e secrets/rabbitmq_password ] && [ ! -e secrets/broker_url ]; then + broker_password=$(openssl rand -hex 24) + printf '%s' "$broker_password" > secrets/rabbitmq_password + printf 'amqp://messenger:%s@rabbitmq:5672/messenger' "$broker_password" > secrets/broker_url +elif [ ! -s secrets/rabbitmq_password ] || [ ! -s secrets/broker_url ]; then + echo 'incomplete RabbitMQ secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e .env ]; then + cp .env.example .env +fi +echo 'Local .env and secret files are present. Existing values were kept.' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a964c566 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if [ -f .env ]; then + compose() { docker compose --env-file .env "$@"; } +else + : "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}" + compose() { docker compose "$@"; } +fi + +wait_healthy() { + for service in "$@"; do + containers=$(compose ps -q "$service") + [ -n "$containers" ] || { echo "$service has no running containers" >&2; return 1; } + for container in $containers; do + attempts=0 + until [ "$(docker inspect --format '{{.State.Health.Status}}' "$container")" = healthy ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 30 ] || { echo "$service did not become healthy" >&2; return 1; } + sleep 1 + done + done + done +} + +restart_services() { + for service in "$@"; do + containers=$(compose ps -a -q "$service") + [ -n "$containers" ] || { echo "$service has no containers" >&2; return 1; } + docker restart --timeout 20 $containers >/dev/null + done +} + +# Restart only long-running services and preserve their containers and volumes. +compose stop worker relay attachment-cleanup >/dev/null +restart_services postgres redis s3 rabbitmq +wait_healthy postgres redis s3 rabbitmq +restart_services api +wait_healthy api +restart_services lb +restart_services relay worker attachment-cleanup diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..deca2b10 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +cd "$(dirname "$0")/.." + +for container_id in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --timeout 5 "$container_id" >/dev/null + exit 0 + fi +done + +printf 'API replica %s not found\n' "$TARGET_INSTANCE_ID" >&2 +exit 1 \ No newline at end of file diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 00000000..65192b2d --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,11 @@ +version: "2" + +sql: + - engine: "postgresql" + schema: "db/migrations" + queries: "db/queries" + gen: + go: + package: "sqlc" + out: "internal/database/sqlc" + sql_package: "pgx/v5" \ No newline at end of file diff --git a/student-tests/auth_contract.py b/student-tests/auth_contract.py new file mode 100644 index 00000000..adc88b83 --- /dev/null +++ b/student-tests/auth_contract.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Lab2 auth acceptance tests. Uses the unchanged teacher HTTP client. + +No server startup, automatic Docker actions, retries of writes or secret output. +Infrastructure hooks run only in explicitly selected suites. +""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import unittest +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, LAB, check, timestamp # noqa: E402 + +LOGIN = "/api/v1/auth/sessions" +LOGOUT = LOGIN + "/current" +ME = "/api/v1/users/me" +ARGS = None + + +def basic(username, password): + value = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + value} + + +def registration(password=None): + return { + "username": "auth_" + uuid.uuid4().hex[:20], + "display_name": "Проверка auth", + "password": password if password is not None else "Aa:" + uuid.uuid4().hex, + } + + +def new_client(timeout=None): + return Client(ARGS.base_url, ARGS.ca_file, timeout=timeout or ARGS.timeout) + + +def error(c, method, path, status, code, **kwargs): + if LAB >= 3 and method == "POST" and path.endswith("/messages"): + kwargs["headers"] = {**kwargs.get("headers", {}), "Idempotency-Key": uuid.uuid4().hex} + response = c.call(method, path, status, **kwargs) + check(response["json"]["error"]["code"] == code, "incorrect contract error code") + if status == 401: + challenge = ('Basic realm="messenger", charset="UTF-8"' + if path == LOGIN else 'Bearer realm="messenger"') + check(response["headers"].get("WWW-Authenticate") == challenge, + "incorrect authentication challenge") + return response + + +def create_actor(c, password=None): + body = registration(password) + user = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + session = c.call("POST", LOGIN, 201, + headers=basic(body["username"], body["password"]), schema="Session")["json"] + check(session["user"] == user, "login returned a different user") + check(timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc), + "new session is already expired") + return {"id": user["id"], "user": user, "registration": body, + "token": session["token"], "expires_at": session["expires_at"]} + + +def login(c, actor): + return c.call("POST", LOGIN, 201, schema="Session", + headers=basic(actor["registration"]["username"], + actor["registration"]["password"]))["json"] + + +def chat(c, owner, members=None): + members = members if members is not None else [owner["id"]] + return c.call("POST", "/api/v1/chats", 201, actor=owner, + body={"title": "Auth regression", "member_ids": members}, schema="Chat")["json"] + + +class AuthContract(unittest.TestCase): + """Independent cases: a logout or rejection never poisons another test.""" + + def setUp(self): + self.c = new_client() + + def test_public_health_and_registration(self): + for path in ("/health/live", "/health/ready"): + with self.subTest(path=path): + self.c.call("GET", path, schema="Health") + body = registration() + # Registration is public even when a stale credential is supplied. + user = self.c.call("POST", "/api/v1/users", 201, body=body, + headers={"Authorization": "Bearer invalid"}, schema="User")["json"] + check(user["username"] == body["username"], "username changed") + check(user["display_name"] == body["display_name"], "display_name changed") + + def test_password_unicode_boundaries_and_colon(self): + for label, password in ( + ("ascii_min", "a" * 12), ("ascii_max", "b" * 128), + ("unicode_min", "😀" * 12), ("unicode_max", "😀" * 128), + ("colon", "начало:пароль:конец"), + ("spaces_are_significant", " significant-password "), + ): + with self.subTest(case=label): + actor = create_actor(self.c, password) + me = self.c.call("GET", ME, actor=actor, schema="User")["json"] + check(me == actor["user"], "current user differs from registration") + altered = password.strip() if password != password.strip() else password + "x" + error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(actor["registration"]["username"], altered)) + + def test_invalid_password_length_and_blank(self): + for label, value in ( + ("empty", ""), ("ascii_short", "a" * 11), ("ascii_long", "a" * 129), + ("unicode_short", "😀" * 11), ("unicode_long", "😀" * 129), + ("blank_ascii", " " * 12), ("blank_unicode", "\u2003" * 12), + ): + with self.subTest(case=label): + body = registration(value) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_registration_required_fields_types_and_names(self): + for field in ("username", "display_name", "password"): + for label, value in (("missing", None), ("null", None), ("integer", 1), + ("boolean", True), ("array", []), ("object", {})): + with self.subTest(field=field, case=label): + body = registration() + if label == "missing": + body.pop(field) + else: + body[field] = value + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for alias in (field.upper(), field[0].upper() + field[1:]): + for extra in (False, True): + with self.subTest(field=field, alias=alias, extra=extra): + body = registration() + body[alias] = body[field] if extra else body.pop(field) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for root in (None, True, 1, "string", []): + with self.subTest(root_type=type(root).__name__): + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=json.dumps(root).encode(), headers={"Content-Type": "application/json"}) + body = registration() + body["sender_id"] = str(uuid.uuid4()) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_registration_bytes_json_and_content_type(self): + for label, raw in ( + ("empty", b""), ("malformed", b'{"username":'), + ("second_json", json.dumps(registration()).encode() + b"{}"), + ("trailing_garbage", json.dumps(registration()).encode() + b" garbage"), + ): + with self.subTest(case=label): + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=raw, headers={"Content-Type": "application/json"}) + for field in ("username", "display_name", "password"): + for label, bad in (("ff", b"\xff"), ("overlong", b"\xc0\xaf"), + ("truncated", b"\xe2\x82"), ("surrogate", b"\xed\xa0\x80")): + with self.subTest(field=field, encoding=label): + body = registration() + body[field] = "BYTE_MARKER" + raw = json.dumps(body).encode().replace(b"BYTE_MARKER", b"prefix" + bad + b"suffix") + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=raw, headers={"Content-Type": "application/json"}) + for media in ("text/plain", "application/octet-stream"): + with self.subTest(media=media): + error(self.c, "POST", "/api/v1/users", 415, "unsupported_media_type", + raw=json.dumps(registration()).encode(), headers={"Content-Type": media}) + + def test_username_and_display_name_constraints(self): + for username in ("ab", "a" * 33, "ABC", "имя", "bad-name", "bad name", " bad"): + with self.subTest(case="username", length=len(username)): + body = registration() + body["username"] = username + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for name in ("", " \t\n", "\u00a0\u2003", "😀" * 101): + with self.subTest(case="display_name", length=len(name)): + body = registration() + body["display_name"] = name + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for length in (1, 100): + with self.subTest(case="valid_display_name", length=length): + body = registration() + body["display_name"] = "😀" * length + user = self.c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["display_name"] == body["display_name"], "Unicode display name changed") + body = registration() + body["username"] = uuid.uuid4().hex + user = self.c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["username"] == body["username"], "maximum-length username changed") + + def test_wrong_credentials_have_same_public_error(self): + actor = create_actor(self.c) + responses = [] + for label, username, password in ( + ("wrong_password", actor["registration"]["username"], "wrong-password"), + ("unknown_user", registration()["username"], actor["registration"]["password"]), + ): + with self.subTest(case=label): + responses.append(error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(username, password))) + if len(responses) == 2: + check(responses[0]["json"]["error"]["message"] == + responses[1]["json"]["error"]["message"], + "credentials errors reveal whether username exists") + self.c.call("GET", ME, actor=actor, schema="User") + + def test_login_rejects_missing_or_malformed_basic(self): + actor = create_actor(self.c) + encoded_no_colon = base64.b64encode(b"username-only").decode() + for label, headers in ( + ("absent", {}), ("legacy_id", {"X-User-Id": actor["id"]}), + ("bearer", {"Authorization": "Bearer " + actor["token"]}), + ("empty_basic", {"Authorization": "Basic"}), + ("invalid_base64", {"Authorization": "Basic !!!"}), + ("missing_separator", {"Authorization": "Basic " + encoded_no_colon}), + ("invalid_username_utf8", {"Authorization": "Basic " + base64.b64encode(b"\xffuser:synthetic-password").decode("ascii")}), + ("username_contains_nul", {"Authorization": "Basic " + base64.b64encode(b"bad\x00user:synthetic-password").decode("ascii")}), + ("wrong_scheme", {"Authorization": "Digest anything"}), + ("empty_password", basic(actor["registration"]["username"], "")), + ): + with self.subTest(case=label): + error(self.c, "POST", LOGIN, 401, "unauthorized", headers=headers) + + def test_login_does_not_accept_json_credentials(self): + actor = create_actor(self.c) + body = {"username": actor["registration"]["username"], + "password": actor["registration"]["password"]} + # Authentication precedes payload validation. + error(self.c, "POST", LOGIN, 401, "unauthorized", body=body) + error(self.c, "POST", LOGIN, 400, "invalid_request", body=body, + headers=basic(body["username"], body["password"])) + + def test_protected_routes_require_bearer_before_validation(self): + actor = create_actor(self.c) + basic_headers = basic(actor["registration"]["username"], actor["registration"]["password"]) + unknown = str(uuid.uuid4()) + routes = ( + ("GET", ME, None), ("GET", "/api/v1/chats", None), + ("POST", "/api/v1/chats", {"title": "", "member_ids": []}), + ("GET", "/api/v1/chats/" + unknown, None), + ("GET", "/api/v1/chats/not-a-uuid/messages?limit=0", None), + ("POST", "/api/v1/chats/" + unknown + "/messages", {"text": ""}), + ("DELETE", LOGOUT, None), + ) + for method, path, body in routes: + for label, headers in ( + ("absent", {}), ("basic", basic_headers), ("legacy_id", {"X-User-Id": actor["id"]}), + ("empty_bearer", {"Authorization": "Bearer"}), + ("unknown_token", {"Authorization": "Bearer " + uuid.uuid4().hex * 2}), + ("space_in_token", {"Authorization": "Bearer invalid token"}), + ): + with self.subTest(method=method, path=path, auth=label): + error(self.c, method, path, 401, "unauthorized", body=body, headers=headers) + + def test_identity_acl_and_sender_ignore_legacy_header(self): + alice, bob, eve = (create_actor(self.c) for _ in range(3)) + room = chat(self.c, alice, [alice["id"], bob["id"]]) + path = "/api/v1/chats/" + room["id"] + me = self.c.call("GET", ME, actor=alice, headers={"X-User-Id": eve["id"]}, + schema="User")["json"] + check(me == alice["user"], "legacy ID overrode session identity") + result = self.c.call("POST", path + "/messages", 201, actor=alice, + headers={"X-User-Id": bob["id"], + **({"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {})}, + body={"text": "Auth sender"}, schema="Message")["json"] + check(result["sender_id"] == alice["id"], "sender not derived from session") + for suffix in ("", "/messages"): + error(self.c, "GET", path + suffix, 404, "not_found", actor=eve, + headers={"X-User-Id": alice["id"]}) + error(self.c, "POST", path + "/messages", 404, "not_found", actor=eve, + body={"text": "Forbidden"}, headers={"X-User-Id": alice["id"]}) + hidden = self.c.call("GET", "/api/v1/chats?limit=100", actor=eve, + schema="ChatPage")["json"] + check(all(item["id"] != room["id"] for item in hidden["items"]), "foreign chat leaked") + page = self.c.call("GET", path + "/messages", actor=bob, schema="MessagePage")["json"] + check(page["items"] == [result], "confirmed message is not immediately visible") + error(self.c, "POST", path + "/messages", 400, "invalid_request", actor=alice, + body={"text": "Forged sender", "sender_id": bob["id"]}) + + def test_invalid_chat_does_not_leave_visible_partial_chat(self): + actor = create_actor(self.c) + before = self.c.call("GET", "/api/v1/chats?limit=100", actor=actor, schema="ChatPage")["json"] + for members in ([actor["id"], str(uuid.uuid4())], + [actor["id"], actor["id"]], + [actor["id"], actor["id"].upper()], [str(uuid.uuid4())]): + with self.subTest(member_count=len(members)): + error(self.c, "POST", "/api/v1/chats", 400, "invalid_request", actor=actor, + body={"title": "Must not exist", "member_ids": members}) + after = self.c.call("GET", "/api/v1/chats?limit=100", actor=actor, schema="ChatPage")["json"] + check(after == before, "rejected chat request left visible partial data") + + def test_logout_revokes_only_current_session(self): + actor = create_actor(self.c) + other = login(self.c, actor) + check(other["token"] != actor["token"], "separate logins reused a token") + self.c.call("DELETE", LOGOUT, 204, actor=actor) + error(self.c, "GET", ME, 401, "unauthorized", actor=actor) + error(self.c, "DELETE", LOGOUT, 401, "unauthorized", actor=actor) + second = dict(actor, token=other["token"]) + me = self.c.call("GET", ME, actor=second, schema="User")["json"] + check(me == actor["user"], "logout revoked another session") + fresh = login(self.c, actor) + check(fresh["token"] not in (actor["token"], other["token"]), "login reused an earlier token") + self.c.call("GET", ME, actor=dict(actor, token=fresh["token"]), schema="User") + + def test_session_ttl_matches_configuration(self): + actor = create_actor(self.c) + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(ARGS.ttl - 5 <= remaining <= ARGS.ttl + 5, "SESSION_TTL_SECONDS not respected") + for _ in range(3): + self.c.call("GET", ME, actor=actor, schema="User") + # Exact non-sliding deadline is checked by the storage and short-TTL suites. + + def test_tokens_distinct_opaque_and_tamper_rejected(self): + actor = create_actor(self.c) + issued = [actor["token"]] + [login(self.c, actor)["token"] for _ in range(3)] + check(len(set(issued)) == len(issued), "independent logins reused tokens") + for token in issued: + check(token != actor["id"] and token != actor["registration"]["username"], + "token contains identity instead of an opaque credential") + check(token != actor["registration"]["password"], "password used as session token") + token = actor["token"] + changed = token[:-1] + ("A" if token[-1] != "A" else "B") + error(self.c, "GET", ME, 401, "unauthorized", actor=dict(actor, token=changed)) + self.c.call("GET", ME, actor=actor, schema="User") + # Length/uniqueness is not proof of entropy. RNG is reviewed separately. + + def test_duplicate_username_concurrent_registration(self): + body = registration() + barrier = threading.Barrier(4) + + def send(index): + c = new_client() + candidate = dict(body, display_name="Race " + str(index), password=body["password"] + str(index)) + barrier.wait(timeout=ARGS.timeout) + result = c.call("POST", "/api/v1/users", (201, 409), body=candidate, schema="User") + if result["status"] == 409: + check(result["json"]["error"]["code"] == "conflict", "race loser did not receive conflict") + return candidate, result + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(send, range(4))) + winners = [(body, result) for body, result in results if result["status"] == 201] + check(len(winners) == 1, "username race must have exactly one winner") + winner_body, result = winners[0] + session = self.c.call("POST", LOGIN, 201, headers=basic(winner_body["username"], + winner_body["password"]), schema="Session")["json"] + check(session["user"] == result["json"], "race winner was overwritten") + for candidate, response in results: + if response["status"] == 409: + error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(candidate["username"], candidate["password"])) + + +def hook(path): + script = Path(path).expanduser().resolve() + check(script.is_file() and os.access(script, os.X_OK), "hook must exist and be executable") + check(".example." not in script.name, "implement a real hook first") + try: + result = subprocess.run([str(script)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=ARGS.hook_timeout) + except (OSError, subprocess.TimeoutExpired): + raise Failure("infrastructure hook failed or timed out; inspect it locally") from None + check(result.returncode == 0, "infrastructure hook returned an error") + + +def wait_ready(c): + deadline = time.monotonic() + ARGS.hook_timeout + while True: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "API readiness did not recover") + time.sleep(0.25) + + +def expiry(): + c = new_client() + actor = create_actor(c) + start = time.monotonic() + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(0 < remaining <= ARGS.ttl + 5, "short TTL not applied") + c.call("GET", ME, actor=actor, schema="User") + # Reading halfway must not renew the original deadline. + time.sleep(max(0, remaining / 2)) + c.call("GET", ME, actor=actor, schema="User") + time.sleep(max(0, remaining + 0.25 - (time.monotonic() - start))) + error(c, "GET", ME, 401, "unauthorized", actor=actor) + error(c, "DELETE", LOGOUT, 401, "unauthorized", actor=actor) + fresh = login(c, actor) + check(fresh["token"] != actor["token"], "expired token reused") + c.call("GET", ME, actor=dict(actor, token=fresh["token"]), schema="User") + + +def persistence(): + c = new_client(timeout=2) + actor = create_actor(c) + revoked = dict(actor, token=login(c, actor)["token"]) + c.call("DELETE", LOGOUT, 204, actor=revoked) + room = chat(c, actor) + original = c.message(actor, room, "Auth persistence")["json"] + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(remaining > 2 * ARGS.hook_timeout + 10, "TTL too short for restart bounds") + hook(ARGS.restart_hook) + wait_ready(c) + # No re-login before checking the original token. + check(c.call("GET", ME, actor=actor, schema="User")["json"] == actor["user"], + "original session/user did not survive restart") + check(c.call("GET", "/api/v1/chats/" + room["id"], actor=actor, schema="Chat")["json"] == room, + "chat/membership changed after restart") + page = c.call("GET", "/api/v1/chats/" + room["id"] + "/messages", + actor=actor, schema="MessagePage")["json"] + check(page["items"] == [original], "confirmed message changed after restart") + error(c, "GET", ME, 401, "unauthorized", actor=revoked) + error(c, "DELETE", LOGOUT, 401, "unauthorized", actor=revoked) + c.call("DELETE", LOGOUT, 204, actor=actor) + error(c, "GET", ME, 401, "unauthorized", actor=actor) + check(login(c, actor)["user"] == actor["user"], "password/user not persistent") + # HTTP exposes expiry only at login. Storage test verifies deadline stability. + + +def dependency(): + c = new_client() + actor = create_actor(c) + failures = [] + fallback_session = None + revoked_during_outage = False + sql_with_redis = ARGS.dependency == "postgres" and getattr(ARGS, "session_store", "redis") == "redis" + if sql_with_redis: + fallback_session = dict(actor, token=login(c, actor)["token"]) + + def observe(name, operation): + try: + operation() + print("PASS:", name) + return True + except Failure as exc: + failures.append(name + ": " + str(exc)) + print("FAIL:", name + ": " + str(exc)) + return False + + try: + hook(ARGS.failure_hook) + observe("liveness during outage", lambda: c.call("GET", "/health/live", schema="Health")) + critical = not (ARGS.dependency == "redis" and ARGS.cache_policy == "fallback") + if critical: + observe("readiness during outage", lambda: error(c, "GET", "/health/ready", 503, "unavailable")) + observe("valid session during outage", lambda: error(c, "GET", ME, 503, "unavailable", actor=actor)) + else: + observe("fallback readiness", lambda: c.call("GET", "/health/ready", schema="Health")) + observe("fallback identity", lambda: c.call("GET", ME, actor=actor, schema="User")) + for name, headers in (("missing auth", {}), ("legacy identity", {"X-User-Id": actor["id"]})): + observe(name, lambda headers=headers: error(c, "GET", ME, 401, "unauthorized", headers=headers)) + # A syntactically valid Bearer cannot be declared unknown while the + # sole session store is unavailable. The lookup failed, not identity. + unknown_status = 503 if ARGS.dependency == "redis" and critical else 401 + observe("unknown bearer during outage", lambda: error(c, "GET", ME, unknown_status, + "unavailable" if unknown_status == 503 else "unauthorized", + headers={"Authorization": "Bearer " + uuid.uuid4().hex * 2})) + if critical: + observe("login during outage", lambda: error(c, "POST", LOGIN, 503, "unavailable", + headers=basic(actor["registration"]["username"], actor["registration"]["password"]))) + if not critical: + fallback_session = dict(actor, token=login(c, actor)["token"]) + if sql_with_redis or not critical: + revoked_during_outage = observe("Redis logout independent of SQL", lambda: c.call("DELETE", LOGOUT, 204, actor=actor)) + else: + observe("logout during outage", lambda: error(c, "DELETE", LOGOUT, 503, "unavailable", actor=actor)) + if ARGS.dependency == "postgres": + body = registration() + observe("registration during SQL outage", lambda: error(c, "POST", "/api/v1/users", 503, "unavailable", body=body)) + finally: + hook(ARGS.recovery_hook) + observe("readiness recovers", lambda: wait_ready(c)) + observe("liveness recovers", lambda: c.call("GET", "/health/live", schema="Health")) + survivor = fallback_session or actor + observe("independent session after recovery", lambda: c.call("GET", ME, actor=survivor, schema="User")) + if revoked_during_outage: + observe("logout remains revoked after recovery", lambda: error(c, "GET", ME, 401, "unauthorized", actor=actor)) + if ARGS.dependency == "postgres": + observe("failed registration left no user", lambda: c.call("POST", "/api/v1/users", 201, body=body, schema="User")) + check(not failures, "; ".join(failures)) + + +def main(): + global ARGS + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--base-url", default=None) + p.add_argument("--ca-file", default=None) + p.add_argument("--timeout", type=float, default=15) + p.add_argument("--ttl", type=int, default=3600, help="actual server SESSION_TTL_SECONDS") + p.add_argument("--suite", choices=("normal", "expiry", "persistence", "dependency"), default="normal") + p.add_argument("--restart-hook") + p.add_argument("--failure-hook") + p.add_argument("--recovery-hook") + p.add_argument("--dependency", choices=("postgres", "redis")) + p.add_argument("--cache-policy", choices=("critical", "fallback")) + p.add_argument("--session-store", choices=("postgres", "redis"), default="redis", + help="declared session backend; controls logout expectation during a SQL outage") + p.add_argument("--hook-timeout", type=float, default=90) + ARGS = p.parse_args() + if LAB < 2: + p.error("this suite requires the Bearer session contract introduced in lab2") + if ARGS.timeout <= 0 or ARGS.hook_timeout <= 0: + p.error("timeouts must be positive") + if ARGS.suite == "expiry": + if not 2 <= ARGS.ttl <= 10: + p.error("expiry needs a separately configured server with TTL 2..10") + elif ARGS.ttl < 300: + p.error("normal, restart and dependency suites require TTL >= 300") + if ARGS.suite == "persistence" and not ARGS.restart_hook: + p.error("persistence requires an explicit --restart-hook") + if ARGS.suite == "dependency": + if not (ARGS.failure_hook and ARGS.recovery_hook and ARGS.dependency): + p.error("dependency suite needs failure/recovery hooks and the dependency name") + if ARGS.dependency == "redis" and not ARGS.cache_policy: + p.error("declare whether Redis is critical or has a safe fallback") + if ARGS.suite == "normal": + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(AuthContract)) + raise SystemExit(0 if result.wasSuccessful() else 1) + try: + {"expiry": expiry, "persistence": persistence, "dependency": dependency}[ARGS.suite]() + except (Failure, ValueError, OSError) as exc: + print("FAIL:", str(exc)) + raise SystemExit(1) + print("PASS:", ARGS.suite, "(hook scope and runtime settings need separate evidence)") + + +if __name__ == "__main__": + main() diff --git a/student-tests/auth_tests/attachment_cleanup_test.go b/student-tests/auth_tests/attachment_cleanup_test.go new file mode 100644 index 00000000..5febe9ac --- /dev/null +++ b/student-tests/auth_tests/attachment_cleanup_test.go @@ -0,0 +1,221 @@ +package auth_tests + +import ( + "context" + "errors" + "io" + "net/http" + "slices" + "strings" + "testing" + "testing/synctest" + "time" + + "highload/internal/model" + "highload/internal/repository" + "highload/internal/service" + "highload/internal/storage" + "highload/internal/storage/s3" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type cleanupRepo struct { + repository.AttachmentRepository + referenced map[string]bool + busy string + existsErr error + checked []string + locked string +} + +type cleanupTx struct{ repo *cleanupRepo } + +func (tx cleanupTx) CreateQueued(context.Context, string, string, string, string, string, string, string, int64) (*model.Attachment, error) { + return nil, errors.New("cleanup fixture must not create queued attachments") +} + +func (tx cleanupTx) CreateAttachment(context.Context, repository.CreateAttachmentParams) (*model.Attachment, error) { + return nil, errors.New("unexpected INSERT") +} +func (tx cleanupTx) GetAttachmentByID(context.Context, string) (*model.Attachment, error) { + return nil, errors.New("unexpected attachment lookup") +} +func (tx cleanupTx) ExistsByObjectKey(ctx context.Context, key string) (bool, error) { + r := tx.repo + if r.locked != key { + return false, errors.New("EXISTS outside locked transaction") + } + r.checked = append(r.checked, key) + return r.referenced[key], r.existsErr +} +func (r *cleanupRepo) WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, repository.AttachmentTx) error) (bool, error) { + if key == r.busy { + return false, nil + } + r.locked = key + defer func() { r.locked = "" }() + return true, fn(ctx, cleanupTx{repo: r}) +} + +type cleanupStore struct { + pages map[string][]storage.ObjectInfo + next map[string]string + listed, deleted []string + repo *cleanupRepo + listErr, deleteErr error + waitList bool +} + +func (s *cleanupStore) ListPage(ctx context.Context, token *string, prefix string) ([]storage.ObjectInfo, *string, error) { + if prefix != "attachments/" { + return nil, nil, errors.New("unexpected prefix") + } + if _, ok := ctx.Deadline(); !ok { + return nil, nil, errors.New("LIST did not receive deadline") + } + if s.waitList { + <-ctx.Done() + return nil, nil, ctx.Err() + } + key := aws.ToString(token) + s.listed = append(s.listed, key) + if s.listErr != nil { + return nil, nil, s.listErr + } + if next, ok := s.next[key]; ok { + return s.pages[key], &next, nil + } + return s.pages[key], nil, nil +} +func (s *cleanupStore) Delete(_ context.Context, key string) error { + if s.repo.locked != key { + return errors.New("DELETE outside locked transaction") + } + s.deleted = append(s.deleted, key) + return s.deleteErr +} + +func TestAttachmentCleanupPagesAndCutoff(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + now := time.Now() + old := now.Add(-time.Minute - time.Nanosecond) + orphan := "attachments/orphan" + opts := &service.AttachmentCleanupOptions{GracePeriod: time.Minute, ListTimeout: time.Second} + repo := &cleanupRepo{referenced: map[string]bool{"attachments/referenced": true}, busy: "attachments/busy"} + objects := &cleanupStore{repo: repo, pages: map[string][]storage.ObjectInfo{ + "": {{Key: "attachments/equal", LastModified: now.Add(-time.Minute)}, + {Key: "attachments/fresh", LastModified: now.Add(-time.Second)}, + {Key: "attachments/future", LastModified: now.Add(time.Hour)}, + {Key: "attachments/zero"}, + {Key: "other/orphan", LastModified: old}, + {Key: "attachments/busy", LastModified: old}}, + "last": {{Key: "attachments/referenced", LastModified: old}, {Key: orphan, LastModified: old}}, + }, next: map[string]string{"": "empty", "empty": "last"}} + svc := service.NewAttachmentCleanupService(repo, objects, opts) + opts.DryRun = true + if err := svc.RunOnce(context.Background()); err != nil || len(objects.deleted) != 0 || + !slices.Equal(repo.checked, []string{"attachments/referenced", orphan}) { + t.Fatalf("dry-run must check old unlocked keys without deleting: err=%v checked=%v deletes=%v", err, repo.checked, objects.deleted) + } + opts.DryRun = false + objects.listed = nil + if err := svc.RunOnce(context.Background()); err != nil || !slices.Equal(objects.listed, []string{"", "empty", "last"}) || + !slices.Equal(objects.deleted, []string{orphan}) { + t.Fatalf("wrong cutoff or page walk: err=%v pages=%v deletes=%v", err, objects.listed, objects.deleted) + } + }) +} +func TestAttachmentCleanupFaults(t *testing.T) { + for _, name := range []string{"list", "exists", "delete", "list_timeout", "pass_timeout", "canceled"} { + t.Run(name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := &cleanupRepo{} + objects := &cleanupStore{repo: repo, pages: map[string][]storage.ObjectInfo{"": {{Key: "attachments/old", LastModified: time.Now().Add(-time.Hour)}}}} + svc := service.NewAttachmentCleanupService(repo, objects, &service.AttachmentCleanupOptions{GracePeriod: time.Second, ListTimeout: time.Second}) + sentinel := errors.New("synthetic " + name + " error") + var want error = sentinel + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch name { + case "list": + objects.listErr = sentinel + case "exists": + repo.existsErr = sentinel + case "delete": + objects.deleteErr = sentinel + case "list_timeout": + objects.waitList = true + want = context.DeadlineExceeded + case "pass_timeout": + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, 100*time.Millisecond) + defer stop() + objects.waitList = true + want = context.DeadlineExceeded + case "canceled": + cancel() + want = context.Canceled + } + start := time.Now() + got := svc.RunOnce(ctx) + if !errors.Is(got, want) { + t.Fatalf("want %v, got %v", want, got) + } + if (name == "list_timeout" && time.Since(start) != time.Second) || + (name == "pass_timeout" && time.Since(start) != 100*time.Millisecond) { + t.Fatal("wrong cancellation budget") + } + if name != "delete" && len(objects.deleted) > 0 { + t.Fatal("DELETE ran after unsafe precondition") + } + if name == "delete" && len(objects.deleted) != 1 { + t.Fatal("DELETE fault not reached") + } + repo.existsErr, objects.listErr, objects.deleteErr = nil, nil, nil + objects.waitList = false + objects.deleted = nil + if err := svc.RunOnce(context.Background()); err != nil || !slices.Equal(objects.deleted, []string{"attachments/old"}) { + t.Fatalf("cleanup did not recover: %v %v", err, objects.deleted) + } + }) + }) + } +} + +type cleanupListTransport func(*http.Request) (*http.Response, error) + +func (fn cleanupListTransport) Do(req *http.Request) (*http.Response, error) { return fn(req) } + +func TestAttachmentCleanupS3InvalidNextToken(t *testing.T) { + for _, tc := range []struct { + name string + token *string + xml string + }{ + {"missing", nil, ""}, + {"empty", nil, ""}, + {"repeat", aws.String("same"), "same"}, + } { + t.Run(tc.name, func(t *testing.T) { + n := 0 + obj := &s3.Storage{Bucket: "test", Client: awss3.New(awss3.Options{ + Region: "us-east-1", UsePathStyle: true, BaseEndpoint: aws.String("http://cleanup.invalid"), + Credentials: aws.AnonymousCredentials{}, Retryer: aws.NopRetryer{}, + HTTPClient: cleanupListTransport(func(req *http.Request) (*http.Response, error) { + n++ + if req.URL.Query().Get("continuation-token") != aws.ToString(tc.token) { + return nil, errors.New("wrong request token") + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader( + "true" + tc.xml + ""))}, nil + }), + })} + _, _, err := obj.ListPage(context.Background(), tc.token, "attachments/") + if !errors.Is(err, storage.ErrStorage) || n != 1 { + t.Fatalf("invalid token was not rejected: %v, calls=%d", err, n) + } + }) + } +} diff --git a/student-tests/auth_tests/attachment_tx_test.go b/student-tests/auth_tests/attachment_tx_test.go new file mode 100644 index 00000000..f53c28ce --- /dev/null +++ b/student-tests/auth_tests/attachment_tx_test.go @@ -0,0 +1,340 @@ +package auth_tests + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + repo "highload/internal/repository" + "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage" + "highload/internal/storage/s3" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func newAttachmentTxFixture(t *testing.T) (*fixture, repo.CreateAttachmentParams) { + t.Helper() + f := newDatabaseFixture(t, "LAB4_TEST_DATABASE_URL", 4) + owner := uuid.NewString() + if _, err := f.pool.Exec(f.ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,'tx_fixture','TX fixture','unused')", owner); err != nil { + t.Fatal(err) + } + chat, err := postgres.NewChatRepository(f.queries, f.pool).CreateChat(f.ctx, "Attachment transaction", []string{owner}) + if err != nil { + t.Fatal(err) + } + return f, repo.CreateAttachmentParams{ + ChatID: chat.ID, OwnerID: owner, ObjectKey: "attachments/" + uuid.NewString(), + Filename: "fixture.png", ContentType: "image/png", SizeBytes: 1, Sha256: strings.Repeat("a", 64), + } +} + +func assertAttachmentCount(t *testing.T, f *fixture, want int) { + t.Helper() + var attachments, jobs, outbox int + err := f.pool.QueryRow(f.ctx, `SELECT (SELECT count(*) FROM attachments), + (SELECT count(*) FROM attachment_jobs), (SELECT count(*) FROM outbox)`).Scan(&attachments, &jobs, &outbox) + if err != nil || attachments != want || jobs != want || outbox != want { + t.Fatalf("expected %d rows each, got attachments=%d jobs=%d outbox=%d (%v)", want, attachments, jobs, outbox, err) + } +} + +func TestAttachmentTxLockAndRollback(t *testing.T) { + f, params := newAttachmentTxFixture(t) + // Run the isolation check on the very connection doing the callback INSERT. + // A query through the pool would only establish the observer's isolation. + if _, err := f.pool.Exec(f.ctx, ` + CREATE FUNCTION assert_attachment_isolation() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'expected read committed during attachment INSERT'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER assert_attachment_isolation BEFORE INSERT ON attachments + FOR EACH ROW EXECUTE FUNCTION assert_attachment_isolation(); + `, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal(err) + } + attachments := postgres.NewAttachmentRepository(f.queries, f.pool) + id := uuid.NewString() + key := "staging/" + id + "/original" + entered := make(chan struct{}) + release := make(chan struct{}) + type outcome struct { + locked bool + err error + } + done := make(chan outcome, 1) + holdCtx, cancelHold := context.WithCancel(f.ctx) + defer cancelHold() + go func() { + locked, err := attachments.WithObjectKeyLock(holdCtx, key, func(ctx context.Context, _ repo.AttachmentTx) error { + close(entered) + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + done <- outcome{locked, err} + }() + select { + case <-entered: + case result := <-done: + t.Fatalf("holder never entered callback: %+v", result) + case <-time.After(3 * time.Second): + t.Fatal("holder did not acquire the lock") + } + unexpected := func(context.Context, repo.AttachmentTx) error { + return errors.New("contending callback ran") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key, unexpected); locked || err != nil { + t.Fatalf("same key must return false, nil without callback: %t, %v", locked, err) + } + noop := func(context.Context, repo.AttachmentTx) error { return nil } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key+"-other", noop); !locked || err != nil { + t.Fatalf("another key was blocked: %t, %v", locked, err) + } + close(release) + select { + case result := <-done: + if !result.locked || result.err != nil { + t.Fatalf("holder did not complete: %+v", result) + } + case <-time.After(3 * time.Second): + t.Fatal("holder did not close its transaction") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key, noop); !locked || err != nil { + t.Fatalf("committed lock was not released: %t, %v", locked, err) + } + + for _, name := range []string{"callback_error", "context_canceled"} { + t.Run(name, func(t *testing.T) { + sentinel := errors.New("synthetic callback failure") + ctx, cancel := context.WithCancel(f.ctx) + defer cancel() + want := error(sentinel) + if name == "context_canceled" { + want = context.Canceled + } + locked, err := attachments.WithObjectKeyLock(ctx, key, func(txCtx context.Context, tx repo.AttachmentTx) error { + if _, err := tx.CreateQueued(txCtx, id, params.ChatID, params.OwnerID, params.ContentType, + params.Sha256, key, params.Filename, params.SizeBytes); err != nil { + return err + } + exists, err := tx.ExistsByObjectKey(txCtx, key) + if err != nil || !exists { + return fmt.Errorf("transaction did not see its INSERT: %w", err) + } + assertAttachmentCount(t, f, 0) + if name == "context_canceled" { + cancel() + return txCtx.Err() + } + return sentinel + }) + if !locked || err != want { + t.Fatalf("callback error must pass through unchanged with true: %t, %v", locked, err) + } + assertAttachmentCount(t, f, 0) + if locked, err = attachments.WithObjectKeyLock(f.ctx, key, noop); !locked || err != nil { + t.Fatalf("rollback failed to free lock/pool: %t, %v", locked, err) + } + t.Log("INSERT ran at read committed, rollback left zero rows and released the lock") + }) + } +} + +// txS3Transport observes the real SDK requests and retains synthetic bytes for GET. +// Before accepting a PUT it challenges the lock from a separate PostgreSQL session. +type txS3Transport struct { + fixture *fixture + key string + data []byte + put, get, delete int + failPut bool +} + +func (s *txS3Transport) Do(request *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: 200, Header: make(http.Header), Body: http.NoBody} + key := strings.TrimPrefix(request.URL.Path, "/tx-test/") + parts := strings.Split(key, "/") + if len(parts) != 3 || parts[0] != "staging" || parts[2] != "original" { + return nil, errors.New("unexpected synthetic S3 staging key") + } + if _, err := uuid.Parse(parts[1]); err != nil { + return nil, errors.New("staging key lacks an attachment UUID") + } + switch request.Method { + case http.MethodPut: + s.put++ + s.key = key + var acquired bool + err := s.fixture.pool.QueryRow(request.Context(), + "SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0))", key).Scan(&acquired) + if err != nil || acquired { + return nil, fmt.Errorf("PUT ran without held object lock: acquired=%t, error=%v", acquired, err) + } + if s.failPut { + resp.StatusCode = 503 + resp.Header.Set("Content-Type", "application/xml") + resp.Body = io.NopCloser(strings.NewReader("SlowDownSynthetic outage")) + return resp, nil + } + defer request.Body.Close() + s.data, err = io.ReadAll(request.Body) + if err != nil { + return nil, err + } + case http.MethodGet: + s.get++ + if key != s.key || s.data == nil { + return nil, errors.New("object unavailable in fixture") + } + resp.Body = io.NopCloser(bytes.NewReader(s.data)) + resp.ContentLength = int64(len(s.data)) + case http.MethodDelete: + s.delete++ + s.data = nil + default: + return nil, fmt.Errorf("unexpected S3 method %s", request.Method) + } + return resp, nil +} + +func TestAttachmentTxHTTP(t *testing.T) { + _, source, _, _ := runtime.Caller(0) + png, err := os.ReadFile(filepath.Join(filepath.Dir(source), "../../tests/fixtures/sample.png")) + if err != nil { + t.Fatal(err) + } + hash := fmt.Sprintf("%x", sha256.Sum256(png)) + for _, scenario := range []struct { + name string + faultSQL string + http int + errorCode string + }{ + {name: "s3_unavailable", http: 503, errorCode: "unavailable"}, + {name: "insert_after_put", faultSQL: "ALTER TABLE attachments ADD CHECK (filename <> 'fixture.png')", http: 500, errorCode: "internal_error"}, + {name: "commit_after_insert", faultSQL: ` + CREATE FUNCTION reject_attachment_commit() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic deferred failure'; END $$; + CREATE CONSTRAINT TRIGGER reject_attachment_commit AFTER INSERT ON attachments + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reject_attachment_commit(); + `, http: 500, errorCode: "internal_error"}, + {name: "committed_upload_is_queued", http: http.StatusAccepted}, + } { + t.Run(scenario.name, func(t *testing.T) { + f, params := newAttachmentTxFixture(t) + if scenario.faultSQL != "" { + if _, err := f.pool.Exec(f.ctx, scenario.faultSQL, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal(err) + } + } + transport := &txS3Transport{fixture: f, failPut: scenario.http == 503} + objects := &s3.Storage{Bucket: "tx-test", Client: awss3.New(awss3.Options{ + Region: "us-east-1", UsePathStyle: true, BaseEndpoint: aws.String("http://tx-s3.invalid"), + Credentials: aws.AnonymousCredentials{}, Retryer: aws.NopRetryer{}, HTTPClient: transport, + })} + chats := service.NewChatService(postgres.NewChatRepository(f.queries, f.pool), nil, nil) + attachments := postgres.NewAttachmentRepository(f.queries, f.pool) + svc := service.NewAttachmentService(attachments, chats, objects) + h := handler.NewAttachmentHandler(svc) + call := func(method, path, param, value string, data []byte, next http.HandlerFunc) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, bytes.NewReader(data)) + r = r.WithContext(middleware.WithUserID(f.ctx, params.OwnerID)) + r.SetPathValue(param, value) + r.Header.Set("Content-Type", "image/png") + w := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "attachment-tx") + mid.InstanceID(mid.RequestID(next)).ServeHTTP(w, r) + return w + } + w := call(http.MethodPost, "/attachments?filename=fixture.png", "chat_id", params.ChatID, png, h.CreateAttachment) + if transport.put != 1 || transport.delete != 0 { + t.Fatalf("expected one PUT and zero DELETE, got %d/%d", transport.put, transport.delete) + } + if scenario.http != http.StatusAccepted { + assertContractError(t, w, scenario.http, scenario.errorCode) + assertAttachmentCount(t, f, 0) + if scenario.http == 503 { + if transport.data != nil { + t.Fatal("failed S3 PUT stored an object") + } + _, err := svc.CreateAttachment(f.ctx, params.ChatID, params.OwnerID, "fixture.png", "image/png", bytes.NewReader(png), int64(len(png))) + if !errors.Is(err, service.ErrServiceUnavailable) || errors.Is(err, storage.ErrStorage) { + t.Fatalf("storage category did not reach service as unavailable: %v", err) + } + } else if !bytes.Equal(transport.data, png) { + t.Fatal("failed database write did not leave the already stored original") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, transport.key, func(context.Context, repo.AttachmentTx) error { return nil }); !locked || err != nil { + t.Fatalf("failed upload left key locked: %t, %v", locked, err) + } + t.Logf("direct handler HTTP %d; committed rows=0; DELETE=0", w.Code) + return + } + if w.Code != http.StatusAccepted { + t.Fatalf("expected 202 after commit, got %d", w.Code) + } + assertAttachmentCount(t, f, 1) + var response struct { + ID string `json:"id"` + SHA string `json:"sha256"` + Status string `json:"status"` + Variants []json.RawMessage `json:"variants"` + ErrorCode *string `json:"error_code"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || response.SHA != hash || + response.Status != "queued" || response.Variants == nil || len(response.Variants) != 0 || response.ErrorCode != nil { + t.Fatal("invalid queued upload response or SHA-256") + } + row, err := attachments.GetAttachmentForUser(f.ctx, params.OwnerID, response.ID) + if err != nil || row.SHA256 != hash || row.ObjectKey != "" || row.StagingKey != transport.key || + transport.key != "staging/"+response.ID+"/original" || row.SizeBytes != int64(len(png)) || + !bytes.Equal(transport.data, png) { + t.Fatal("committed metadata/staging does not match uploaded file") + } + var linked bool + err = f.pool.QueryRow(f.ctx, `SELECT EXISTS(SELECT 1 FROM attachment_jobs j + JOIN outbox o ON o.job_id=j.id WHERE j.attachment_id=$1 + AND o.payload->>'attachment_id' = $1::uuid::text AND o.payload->>'job_id'=j.id::text + AND o.sent_at IS NULL)`, response.ID).Scan(&linked) + if err != nil || !linked { + t.Fatal("202 lacks a pending outbox record for the same durable job") + } + get := call(http.MethodGet, "/attachments/"+response.ID, "attachment_id", response.ID, nil, h.GetAttachment) + if get.Code != 200 || !bytes.Equal(bytes.TrimSpace(get.Body.Bytes()), bytes.TrimSpace(w.Body.Bytes())) { + t.Fatal("metadata GET changed") + } + body := call(http.MethodGet, "/attachments/"+response.ID+"/content", "attachment_id", response.ID, nil, h.GetAttachmentContent) + assertContractError(t, body, http.StatusConflict, "conflict") + if transport.get != 0 { + t.Fatal("content GET attempted to read S3 for a queued attachment") + } + t.Logf("HTTP 202, attachment/job/outbox committed, metadata=200, queued content=409, SHA-256=%s", hash) + }) + } +} diff --git a/student-tests/auth_tests/auth_handler_test.go b/student-tests/auth_tests/auth_handler_test.go new file mode 100644 index 00000000..51b67fa7 --- /dev/null +++ b/student-tests/auth_tests/auth_handler_test.go @@ -0,0 +1,32 @@ +package auth_tests + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" +) + +func TestStorageAuthHandlerRejectsJSONBody(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + writes := f.redis.audit.writeCount() + auth := handler.NewAuthHandler(f.auth) + mid := middleware.NewMiddleware(f.auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + request.SetBasicAuth(user.Username, password) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + // Exercise the real handler independently of Router wiring. + // This component test does not replace the end-to-end login acceptance test. + mid.RequestID(http.HandlerFunc(auth.CreateSession)).ServeHTTP(recorder, request) + if recorder.Code != http.StatusBadRequest { + t.Errorf("login with JSON body must return 400; received %d", recorder.Code) + } + if f.redis.audit.writeCount() != writes { + t.Error("login with forbidden JSON body attempted to create a session") + } +} diff --git a/student-tests/auth_tests/budgets_test.go b/student-tests/auth_tests/budgets_test.go new file mode 100644 index 00000000..1818c906 --- /dev/null +++ b/student-tests/auth_tests/budgets_test.go @@ -0,0 +1,209 @@ +package auth_tests + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository" + "highload/internal/repository/postgres" +) + +func operationBudget(t *testing.T) time.Duration { + t.Helper() + value := os.Getenv("AUTH_TEST_OPERATION_BUDGET_MS") + if value == "" { + t.Log("budget_policy=provisional_3000ms; teacher has no numeric threshold; approval remains pending") + return 3 * time.Second + } + n, err := strconv.Atoi(value) + if err != nil || n < 500 || n > 10000 { + t.Fatal("AUTH_TEST_OPERATION_BUDGET_MS must be 500..10000") + } + return time.Duration(n) * time.Millisecond +} + +func applicationPool(t *testing.T, f *fixture) *pgxpool.Pool { + t.Helper() + u, err := url.Parse(os.Getenv("AUTH_TEST_DATABASE_URL")) + if err != nil { + t.Fatal("invalid test URL") + } + q := u.Query() + q.Set("search_path", f.config.ConnConfig.RuntimeParams["search_path"]) + u.RawQuery = q.Encode() + // Actual production constructor: never inherit fixture statement_timeout. + pool, err := database.NewPostgres(f.ctx, u.String()) + if err != nil { + t.Fatal("application pool unavailable") + } + f.pool.Close() + f.pool = pool + f.queries = sqlc.New(pool) + return pool +} + +func blockOperation(t *testing.T, f *fixture, mode string) func() { + t.Helper() + if mode == "blocked_query" { + tx, err := f.admin.Begin(f.ctx) + if err != nil { + t.Fatal("lock transaction failed") + } + table := pgx.Identifier{f.config.ConnConfig.RuntimeParams["search_path"], "users"}.Sanitize() + if _, err = tx.Exec(f.ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal("isolated table lock failed") + } + return func() { + ctx, c := context.WithTimeout(context.Background(), 2*time.Second) + defer c() + _ = tx.Rollback(ctx) + } + } + max := f.pool.Config().MaxConns + if max <= 0 || max > 128 { + t.Fatal("test refuses unbounded/oversized pool") + } + held := make([]*pgxpool.Conn, 0, max) + for range max { + conn, err := f.pool.Acquire(f.ctx) + if err != nil { + for _, h := range held { + h.Release() + } + t.Fatal("pool saturation failed") + } + held = append(held, conn) + } + return func() { + for _, h := range held { + h.Release() + } + } +} + +func TestPostgresOperationsHaveBoundedWaits(t *testing.T) { + budget := operationBudget(t) + for _, mode := range []string{"blocked_query", "exhausted_pool"} { + for _, scenario := range []string{"application_budget", "shorter_caller_deadline", "released_before_budget"} { + t.Run(mode+"/"+scenario, func(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + applicationPool(t, f) + release := blockOperation(t, f, mode) + released := false + defer func() { + if !released { + release() + } + }() + safety := budget + 2*time.Second + if scenario == "shorter_caller_deadline" { + safety = 150 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), safety) + defer cancel() + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil).WithContext(ctx) + request.Header.Set("Authorization", "Bearer "+session.Token) + recorder := httptest.NewRecorder() + done := make(chan struct{}) + started := time.Now() + go func() { defer close(done); redisRouter(f, f.redis.client).ServeHTTP(recorder, request) }() + if scenario == "released_before_budget" { + time.Sleep(100 * time.Millisecond) + release() + released = true + } + <-done + elapsed := time.Since(started) + t.Logf("mode=%s scenario=%s elapsed_ms=%.3f caller_deadline_ms=%d budget_ms=%d caller_canceled=%t", mode, scenario, float64(elapsed)/float64(time.Millisecond), safety.Milliseconds(), budget.Milliseconds(), ctx.Err() != nil) + switch scenario { + case "application_budget": + if ctx.Err() != nil || elapsed > budget+250*time.Millisecond { + t.Error("application did not independently enforce its operation budget before test safety cancellation") + } + assertContractError(t, recorder, 503, "unavailable") + case "shorter_caller_deadline": + if elapsed > 750*time.Millisecond { + t.Error("short caller deadline was not honored") + } + assertContractError(t, recorder, 503, "unavailable") + case "released_before_budget": + if recorder.Code != 200 || elapsed >= budget { + t.Error("unblocked operation failed or missed the declared budget") + } + } + if !released { + release() + released = true + } + recovery := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/v1/users/me", nil) + req.Header.Set("Authorization", "Bearer "+session.Token) + recoveryCtx, c := context.WithTimeout(context.Background(), 2*time.Second) + defer c() + redisRouter(f, f.redis.client).ServeHTTP(recovery, req.WithContext(recoveryCtx)) + if recovery.Code != 200 { + t.Error("service did not recover after releasing lock/pool") + } + if f.pool.Stat().AcquiredConns() != 0 { + t.Error("application pool leaked an acquired connection") + } + }) + } + } +} + +func TestPostgresChatCancellationBeforeCommitIsAtomic(t *testing.T) { + f := newFixture(t) + user, _ := f.user(t) + applicationPool(t, f) + tx, err := f.admin.Begin(f.ctx) + if err != nil { + t.Fatal("cannot start membership blocker") + } + released := false + defer func() { + if !released { + _ = tx.Rollback(context.Background()) + } + }() + table := pgx.Identifier{f.config.ConnConfig.RuntimeParams["search_path"], "chat_members"}.Sanitize() + if _, err = tx.Exec(f.ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal("cannot block membership insertion") + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + repo := postgres.NewChatRepository(f.queries, f.pool) + _, err = repo.CreateChat(ctx, "Canceled before membership", []string{user.ID}) + if !errors.Is(err, repository.ErrServiceUnavailable) { + t.Errorf("want unavailable after controlled pre-commit cancellation, got category %v", err) + } + // The membership INSERT cannot finish while this lock is held, so COMMIT + // has not been reached. No ambiguous-commit inference is made here. + _ = tx.Rollback(context.Background()) + released = true + for _, name := range []string{"chats", "chat_members"} { + var count int + if f.pool.QueryRow(f.ctx, "SELECT count(*) FROM "+name).Scan(&count) != nil || count != 0 { + t.Error("pre-commit cancellation left partial chat data") + } + } + if _, err = repo.CreateChat(f.ctx, "Works after cancellation", []string{user.ID}); err != nil { + t.Error("chat repository failed to recover") + } + if f.pool.Stat().AcquiredConns() != 0 { + t.Error("chat cancellation leaked a connection") + } +} diff --git a/student-tests/auth_tests/errors_test.go b/student-tests/auth_tests/errors_test.go new file mode 100644 index 00000000..8bad516a --- /dev/null +++ b/student-tests/auth_tests/errors_test.go @@ -0,0 +1,273 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "highload/internal/database/sqlc" + repository "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +func TestErrorDiagnosticsRecordedOnceAndPublicResponseIsSafe(t *testing.T) { + var output bytes.Buffer + original := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, nil))) + defer slog.SetDefault(original) + users := service.NewUserService(repository.NewUserRepository(sqlc.New(faultDB{cause: errors.New("synthetic-storage-diagnostic")}))) + _, cause := users.GetUser(context.Background(), uuid.NewString()) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", "Bearer synthetic-secret-not-for-logs") + recorder := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "auth-test") + mid.RequestID(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = httputils.WriteServiceError(w, cause, middleware.RequestIDFromContext(r.Context())) + })).ServeHTTP(recorder, request) + assertContractError(t, recorder, 500, "internal_error") + if strings.Contains(recorder.Body.String(), "synthetic-storage-diagnostic") { + t.Error("public response exposed internal cause") + } + if strings.Contains(output.String(), "synthetic-secret-not-for-logs") { + t.Error("server log exposed a Bearer credential") + } + var records []map[string]any + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + if line == "" { + continue + } + var record map[string]any + if json.Unmarshal([]byte(line), &record) != nil { + t.Error("diagnostic log is not structured JSON") + continue + } + // Access records are not detailed duplicate error records. + if record["msg"] != "http request" { + records = append(records, record) + } + } + if len(records) != 1 { + t.Errorf("want exactly one detailed error at response boundary, got %d", len(records)) + return + } + record := records[0] + if _, malformed := record["!BADKEY"]; malformed { + t.Error("structured error log contains !BADKEY: slog arguments are not valid key/value pairs") + } + if record["request_id"] != recorder.Header().Get("X-Request-Id") || record["status"] != float64(500) || + (record["code"] != "internal_error" && record["error_code"] != "internal_error") { + t.Error("diagnostic is missing matching request_id/status/error code") + } + encoded, _ := json.Marshal(record) + if !bytes.Contains(encoded, []byte("synthetic-storage-diagnostic")) { + t.Error("diagnostic lost storage cause") + } +} + +// Faults enter through sqlc's database interface, then pass through the real +// repository, service and HTTP error mapper. Driver causes are deliberately +// not required to survive: the public category and response are the contract. +type errorCase struct { + name string + cause error + want error + status int + code string +} + +func databaseErrorCases() []errorCase { + cases := []errorCase{ + {"no_rows", pgx.ErrNoRows, service.ErrNotFound, 404, "not_found"}, + {"duplicate", &pgconn.PgError{Code: "23505"}, service.ErrAlreadyExists, 409, "conflict"}, + {"eof", io.EOF, service.ErrServiceUnavailable, 503, "unavailable"}, + {"deadline", context.DeadlineExceeded, service.ErrServiceUnavailable, 503, "unavailable"}, + {"unexpected_eof", io.ErrUnexpectedEOF, service.ErrServiceUnavailable, 503, "unavailable"}, + {"closed_connection", pgconn.ErrConnClosed, service.ErrServiceUnavailable, 503, "unavailable"}, + {"connection_refused", unavailableRow{}.Scan(), service.ErrServiceUnavailable, 503, "unavailable"}, + // Docker removes the stopped service from DNS. The real outage probe + // returned a ConnectError containing this error, without a net.OpError. + {"database_dns_missing", &net.DNSError{Err: "no such host", Name: "postgres", IsNotFound: true}, service.ErrServiceUnavailable, 503, "unavailable"}, + {"sql_syntax", &pgconn.PgError{Code: "42601", Message: "private SQL diagnostic"}, service.ErrServerError, 500, "internal_error"}, + {"unexpected", errors.New("private database diagnostic"), service.ErrServerError, 500, "internal_error"}, + } + for _, code := range []string{"53300", "57P03", "08006", "08001", "57P01", "57P02"} { + cases = append(cases, errorCase{"sqlstate_" + code, &pgconn.PgError{Code: code}, service.ErrServiceUnavailable, 503, "unavailable"}) + } + // Wrapping must not change how an existing error category is recognized. + originals := append([]errorCase(nil), cases...) + for _, tc := range originals { + tc.name = "wrapped_" + tc.name + tc.cause = fmt.Errorf("database operation: %w", tc.cause) + cases = append(cases, tc) + } + return cases +} + +type faultRow struct{ cause error } + +func (r faultRow) Scan(...any) error { return r.cause } + +type faultDB struct { + unavailableDB + cause error +} + +func (d faultDB) QueryRow(context.Context, string, ...any) pgx.Row { + return faultRow{d.cause} +} + +func assertContractError(t *testing.T, recorder *httptest.ResponseRecorder, status int, code string) { + t.Helper() + if recorder.Code != status { + t.Fatalf("want HTTP %d, got %d", status, recorder.Code) + } + if recorder.Header().Get("Content-Type") != "application/json" { + t.Fatal("error response must have application/json content type") + } + var payload struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + } + decoder := json.NewDecoder(recorder.Body) + if err := decoder.Decode(&payload); err != nil { + t.Fatal("error response is not valid JSON") + } + if decoder.Decode(new(any)) != io.EOF { + t.Fatal("error response contains more than one JSON value") + } + if payload.Error.Code != code || payload.Error.RequestID == "" || + payload.Error.RequestID != recorder.Header().Get("X-Request-Id") { + t.Fatal("error category or request ID violates the API contract") + } + if payload.Error.Message == "" || strings.Contains(payload.Error.Message, "private") { + t.Fatal("error message is empty or discloses internal diagnostics") + } +} + +func TestErrorDatabaseCategoryReachesHTTP(t *testing.T) { + for _, tc := range databaseErrorCases() { + t.Run(tc.name, func(t *testing.T) { + users := service.NewUserService(repository.NewUserRepository(sqlc.New(faultDB{cause: tc.cause}))) + _, err := users.GetUser(context.Background(), uuid.NewString()) + if !errors.Is(err, tc.want) { + t.Fatalf("want category %v, got %v", tc.want, err) + } + if errors.Is(tc.cause, context.DeadlineExceeded) && !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Error("deadline diagnostic context lost between repository and service") + } + recorder := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "auth-test") + mid.RequestID(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = httputils.WriteServiceError(w, err, middleware.RequestIDFromContext(r.Context())) + })).ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)) + assertContractError(t, recorder, tc.status, tc.code) + }) + } +} + +func TestErrorUnavailableSQLLoginKeepsCategory(t *testing.T) { + for _, tc := range databaseErrorCases() { + if tc.status != 503 { + continue + } + t.Run(tc.name, func(t *testing.T) { + queries := sqlc.New(faultDB{cause: tc.cause}) + auth := service.NewAuthService(repository.NewUserRepository(queries), nil, time.Hour) + mid := middleware.NewMiddleware(auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("synthetic_user", "Synthetic:password") + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(handler.NewAuthHandler(auth).CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 503, "unavailable") + }) + } +} + +// The caller exists; the other member lookup fails. This isolates the service +// branch that used to replace the received err with ErrServerError. +type memberFaultDB struct { + unavailableDB + caller uuid.UUID + cause error +} + +func (d memberFaultDB) QueryRow(_ context.Context, _ string, args ...any) pgx.Row { + id := uuid.UUID(args[0].(pgtype.UUID).Bytes) + if id == d.caller { + return memberUserRow{id} + } + return faultRow{d.cause} +} + +type memberUserRow struct{ id uuid.UUID } + +func (r memberUserRow) Scan(dst ...any) error { + *dst[0].(*pgtype.UUID) = pgtype.UUID{Bytes: [16]byte(r.id), Valid: true} + *dst[1].(*string) = "error_test_caller" + *dst[2].(*string) = "Error test caller" + *dst[3].(*string) = "unused password hash" + *dst[4].(*pgtype.Timestamptz) = pgtype.Timestamptz{Time: time.Unix(1, 0), Valid: true} + return nil +} + +func TestErrorChatMemberLookupKeepsOperationMeaning(t *testing.T) { + for _, tc := range databaseErrorCases() { + if tc.status != 503 && tc.status != 404 && tc.status != 500 { + continue + } + t.Run(tc.name, func(t *testing.T) { + caller := uuid.New() + queries := sqlc.New(memberFaultDB{caller: caller, cause: tc.cause}) + users := repository.NewUserRepository(queries) + // No writable repository: an invalid member list must never reach + // chat insertion. Accidentally continuing is a test failure. + chats := service.NewChatService(nil, users, service.NewUserService(users)) + _, err := chats.Create(context.Background(), "Error regression", []string{caller.String(), uuid.NewString()}, caller.String()) + want, status, code := tc.want, tc.status, tc.code + if tc.status == 404 { + // contracts/README.md:17: unknown members make POST /chats invalid. + want, status, code = service.ErrInvalidInput, 400, "invalid_request" + } + if !errors.Is(err, want) { + t.Fatalf("want category %v, got %v", want, err) + } + gotStatus, gotCode := httputils.ErrorFromService(err) + if gotStatus != status || gotCode != code { + t.Fatalf("want %d/%s, got %d/%s", status, code, gotStatus, gotCode) + } + }) + } +} + +func TestErrorMissingBasicPrecedesForbiddenBody(t *testing.T) { + mid := middleware.NewMiddleware(nil, "auth-test") + auth := handler.NewAuthHandler(nil) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(auth.CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 401, "unauthorized") + if recorder.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatal("missing Basic challenge") + } +} diff --git a/student-tests/auth_tests/logout_diagnostic_test.go b/student-tests/auth_tests/logout_diagnostic_test.go new file mode 100644 index 00000000..33217a99 --- /dev/null +++ b/student-tests/auth_tests/logout_diagnostic_test.go @@ -0,0 +1,202 @@ +package auth_tests + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/google/uuid" + redisclient "github.com/redis/go-redis/v9" + "highload/internal/repository" +) + +type logoutObservationKey struct{} +type logoutObservation struct { + Group int `json:"clients"` + Round int `json:"round"` + PauseMS int `json:"pause_ms"` + ElapsedMS float64 `json:"elapsed_ms"` + WaitMS float64 `json:"waitaof_ms"` + DelID int64 `json:"del_connection_id"` + WaitID int64 `json:"wait_connection_id"` + Local *int64 `json:"local_fsynced"` + Replicas *int64 `json:"replicas_fsynced"` + Kind string `json:"kind"` + CommandError string `json:"command_error,omitempty"` + RepositoryError string `json:"repository_error,omitempty"` +} +type logoutAOFObservation struct{} + +func (h logoutAOFObservation) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (h logoutAOFObservation) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (h logoutAOFObservation) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + started := time.Now() + err := next(ctx, cmd) + row, _ := ctx.Value(logoutObservationKey{}).(*logoutObservation) + if row == nil { + return err + } + name := cmd.Name() + if name != "del" && name != "waitaof" { + return err + } + if name == "waitaof" { + row.WaitMS = float64(time.Since(started)) / float64(time.Millisecond) + if err != nil { + row.CommandError = err.Error() + row.Kind = "command_error" + } else { + if value, ok := cmd.(*redisclient.Cmd); ok { + if reply, ok := value.Val().([]any); ok && len(reply) == 2 { + local, lok := reply[0].(int64) + replicas, rok := reply[1].(int64) + if lok && rok { + row.Local = &local + row.Replicas = &replicas + row.Kind = "acknowledged" + if local != 1 { + row.Kind = "unacknowledged" + } + } + } + } + if row.Local == nil { + row.Kind = "unexpected_reply" + } + } + } + if err == nil { + // next is bound to this Conn's processing path. The observation adds a + // read-only CLIENT ID on that same path; it never gets a second pool slot. + id := redisclient.NewIntCmd(ctx, "CLIENT", "ID") + if e := next(ctx, id); e == nil { + if name == "del" { + row.DelID = id.Val() + } else { + row.WaitID = id.Val() + } + } + } + return err + } +} + +func TestRedisLogoutPersistenceDiagnostic(t *testing.T) { + f := newRedisFixture(t) + f.client.AddHook(logoutAOFObservation{}) + before := f.client.PoolStats() + rows := []logoutObservation{} + var mu sync.Mutex + stop := make(chan struct{}) + sampled := make(chan struct{}) + poolSamples := []map[string]any{} + go func() { + defer close(sampled) + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + poolSamples = append(poolSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "stats": f.client.PoolStats()}) + } + } + }() + defer func() { + close(stop) + <-sampled + deadline := time.Now().Add(time.Second) + for f.client.PoolStats().TotalConns != f.client.PoolStats().IdleConns && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + after := f.client.PoolStats() + if after.TotalConns != after.IdleConns || after.PendingRequests != 0 { + t.Error("logout left acquired/pending pool connections") + } + report := map[string]any{"pool_before": before, "pool_after": after, "pool_samples": poolSamples, "samples": rows, "production_options": map[string]any{"read_timeout_ms": f.client.Options().ReadTimeout.Milliseconds(), "pool_size": f.client.Options().PoolSize, "pool_timeout_ms": f.client.Options().PoolTimeout.Milliseconds()}, "instrumentation": "read-only CLIENT ID after DEL and WAITAOF, on the pinned processing path"} + if directory := os.Getenv("AUTH_TEST_DIAGNOSTICS_DIR"); directory != "" { + _ = os.MkdirAll(directory, 0700) + data, _ := json.MarshalIndent(report, "", " ") + if os.WriteFile(filepath.Join(directory, "redis-logout.json"), data, 0600) != nil { + t.Error("cannot save Redis diagnostics") + } + } + t.Logf("logout samples=%d pool_wait_count=%d wait_ns=%d pool_timeouts=%d final_busy=%d", len(rows), after.WaitCount-before.WaitCount, after.WaitDurationNs-before.WaitDurationNs, after.Timeouts-before.Timeouts, after.TotalConns-after.IdleConns) + }() + run := func(clients, round, pause int) { + var workers sync.WaitGroup + start := make(chan struct{}) + for range clients { + workers.Add(1) + go func() { + defer workers.Done() + // Two sessions for one synthetic identity: revoking one must leave the + // other valid, even while other workers revoke their own sessions. + first, other := f.trackSafe(uuid.NewString()), f.trackSafe(uuid.NewString()) + user := uuid.NewString() + if _, err := f.repo.Create(f.ctx, first[:], user, time.Minute); err != nil { + t.Error("create logout fixture failed") + return + } + if _, err := f.repo.Create(f.ctx, other[:], user, time.Minute); err != nil { + t.Error("create independent fixture failed") + return + } + <-start + time.Sleep(time.Duration(pause) * time.Millisecond) + row := logoutObservation{Group: clients, Round: round, PauseMS: pause} + ctx := context.WithValue(f.ctx, logoutObservationKey{}, &row) + began := time.Now() + _, err := f.repo.RevokeSession(ctx, first[:]) + row.ElapsedMS = float64(time.Since(began)) / float64(time.Millisecond) + if err != nil { + row.RepositoryError = err.Error() + t.Errorf("logout clients=%d kind=%s category=%v", clients, row.Kind, err) + } + if row.Kind == "unacknowledged" && !errors.Is(err, repository.ErrServiceUnavailable) { + t.Error("unacknowledged local fsync must preserve unavailable category") + } + if row.DelID == 0 || row.DelID != row.WaitID { + t.Error("DEL/WAITAOF connection identity not preserved") + } + if _, err = f.repo.GetSessionByTokenHash(f.ctx, other[:]); err != nil { + t.Error("concurrent logout affected independent session") + } + if _, err = f.repo.GetSessionByTokenHash(f.ctx, first[:]); !errors.Is(err, repository.ErrNotFound) { + t.Error("revoked session still available") + } + mu.Lock() + rows = append(rows, row) + mu.Unlock() + }() + } + close(start) + workers.Wait() + } + for i, pause := range []int{0, 137, 359, 613, 887, 1109} { + run(1, i+1, pause) + } + for _, clients := range []int{5, 10} { + for round := 1; round <= 3; round++ { + run(clients, round, 137*round) + } + } +} + +// track modifies fixture cleanup bookkeeping; concurrent probes synchronize it. +var diagnosticTrackMu sync.Mutex + +func (f *redisFixture) trackSafe(token string) [32]byte { + diagnosticTrackMu.Lock() + defer diagnosticTrackMu.Unlock() + return f.track(token) +} diff --git a/student-tests/auth_tests/message_attachments_test.go b/student-tests/auth_tests/message_attachments_test.go new file mode 100644 index 00000000..4090ab5d --- /dev/null +++ b/student-tests/auth_tests/message_attachments_test.go @@ -0,0 +1,570 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "highload/internal/model" + domainrepo "highload/internal/repository" + postgresrepo "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" +) + +// Uses the real handler/service/PostgreSQL repository and metadata-only fixtures. +// Authentication is already covered by the existing auth tests: only the current +// user context is injected here. No S3 server, upload endpoint or Redis is needed. +type messageAttachmentFixture struct { + *fixture + http http.Handler + alice, bob string + chat, other *model.Chat + ids, foreign []string +} + +type attachmentMessage struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +// Small dependencies for the real service's one-attachment regression. They +// compare UUID identity without prescribing where normalization takes place. +// They do not implement persistence/idempotency: PostgreSQL cases prove that. +type singleAttachmentChatRepo struct { + domainrepo.ChatRepository + chat *model.Chat +} + +func (r *singleAttachmentChatRepo) GetChatByID(_ context.Context, id string) (*model.Chat, error) { + if id != r.chat.ID { + return nil, domainrepo.ErrNotFound + } + return r.chat, nil +} + +type singleAttachmentMessageRepo struct { + domainrepo.MessageRepository + message *model.Message + key string + calls [][]string +} + +func (r *singleAttachmentMessageRepo) CreateMessage(_ context.Context, chatID, userID, text, key string, ids []string) (*model.Message, error) { + r.calls = append(r.calls, append([]string{}, ids...)) + if chatID != r.message.ChatID || userID != r.message.SenderID || text != r.message.Text || key != r.key { + return nil, domainrepo.ErrServerError + } + if len(ids) != len(r.message.AttachmentIDs) { + return nil, domainrepo.ErrInvalidInput + } + for index, raw := range ids { + id, err := uuid.Parse(raw) + if err != nil || id.String() != r.message.AttachmentIDs[index] { + return nil, domainrepo.ErrInvalidInput + } + } + return r.message, nil +} + +func TestMessageAttachmentsSingleIDServiceRegression(t *testing.T) { + const attachmentID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + for _, scenario := range []struct { + name, inputID string + requests int + }{ + {name: "single_attachment_without_panic", inputID: attachmentID, requests: 1}, + {name: "identical_uppercase_body_replay", inputID: strings.ToUpper(attachmentID), requests: 2}, + } { + t.Run(scenario.name, func(t *testing.T) { + defer func() { + if recovered := recover(); recovered != nil { + t.Errorf("CreateMessage with one valid attachment panicked: %v", recovered) + } + }() + userID, chatID, key := uuid.NewString(), uuid.NewString(), uuid.NewString() + original := &model.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: userID, + Text: "", AttachmentIDs: []string{attachmentID}, CreatedAt: time.Unix(1700000000, 0).UTC()} + repo := &singleAttachmentMessageRepo{message: original, key: key} + chatRepo := &singleAttachmentChatRepo{chat: &model.Chat{ID: chatID, MemberIDs: []string{userID}}} + chats := service.NewChatService(chatRepo, nil, nil) + messages := service.NewMessageService(repo, chats, nil) + for range scenario.requests { + // The uppercase case starts uppercase and repeats EXACTLY the same + // values. It never assumes different string bodies must be equivalent. + input := []string{scenario.inputID} + message, err := messages.CreateMessage(context.Background(), "", chatID, userID, key, input) + if err != nil || !reflect.DeepEqual(message, original) { + t.Fatalf("one attachment / identical uppercase replay: expected original Message without conflict, got %v", err) + } + } + if len(repo.calls) == 0 { + t.Fatal("service did not create the message through its repository") + } + for _, ids := range repo.calls { + if len(ids) != 1 { + t.Fatal("service lost the single attachment") + } + id, err := uuid.Parse(ids[0]) + if err != nil || id.String() != attachmentID { + t.Fatal("service changed the attachment UUID identity") + } + } + }) + } +} + +func newMessageAttachmentFixture(t *testing.T) *messageAttachmentFixture { + t.Helper() + f := newDatabaseFixture(t, "LAB4_TEST_DATABASE_URL", 12) + result := &messageAttachmentFixture{fixture: f, alice: uuid.NewString(), bob: uuid.NewString()} + for _, id := range []string{result.alice, result.bob} { + _, err := f.pool.Exec(f.ctx, `INSERT INTO users(id,username,display_name,password_hash) + VALUES($1,$2,'Synthetic attachment test','fixture-only-not-used-for-login')`, + id, "attach_"+strings.ReplaceAll(id, "-", "")[:18]) + if err != nil { + t.Fatal("cannot prepare synthetic users") + } + } + chatRepo := postgresrepo.NewChatRepository(f.queries, f.pool) + var err error + result.chat, err = chatRepo.CreateChat(f.ctx, "Message attachments", []string{result.alice, result.bob}) + if err != nil { + t.Fatal("cannot prepare primary chat") + } + result.other, err = chatRepo.CreateChat(f.ctx, "Other attachment scope", []string{result.alice}) + if err != nil { + t.Fatal("cannot prepare other chat") + } + attachments := postgresrepo.NewAttachmentRepository(f.queries, f.pool) + for _, chat := range []*model.Chat{result.chat, result.other} { + for i := 0; i < 11; i++ { + att, err := attachments.CreateAttachment(f.ctx, domainrepo.CreateAttachmentParams{ + ChatID: chat.ID, OwnerID: result.alice, Filename: fmt.Sprintf("synthetic-%d.png", i), + ObjectKey: "metadata-only/" + uuid.NewString(), ContentType: "image/png", + Sha256: strings.Repeat("a", 64), SizeBytes: 1, + }) + if err != nil { + t.Fatal("cannot seed attachment metadata through its PostgreSQL repository") + } + if chat.ID == result.chat.ID { + result.ids = append(result.ids, att.ID) + } else { + result.foreign = append(result.foreign, att.ID) + } + } + } + // Deliberately disagree with UUID ordering to catch accidental ORDER BY id. + slices.Sort(result.ids) + slices.Reverse(result.ids) + users := service.NewUserService(postgresrepo.NewUserRepository(f.queries)) + chats := service.NewChatService(chatRepo, postgresrepo.NewUserRepository(f.queries), users) + messages := service.NewMessageService(postgresrepo.NewMessageRepository(f.queries, f.pool), chats, users) + h := handler.NewMessageHandler(messages) + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.GetMessagesByChatID) + mid := middleware.NewMiddleware(nil, "message-pg-test") + result.http = mid.InstanceID(mid.RequestID(mux)) + return result +} + +func (f *messageAttachmentFixture) call(actor, method, chatID, key string, body any) *httptest.ResponseRecorder { + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + request := httptest.NewRequest(method, "/api/v1/chats/"+chatID+"/messages?limit=100", bytes.NewReader(raw)) + request = request.WithContext(middleware.WithUserID(f.ctx, actor)) + request.Header.Set("Content-Type", "application/json") + if key != "" { + request.Header.Set("Idempotency-Key", key) + } + recorder := httptest.NewRecorder() + f.http.ServeHTTP(recorder, request) + return recorder +} + +func attachmentBody(text string, ids []string) map[string]any { + return map[string]any{"text": text, "attachment_ids": ids} +} + +func decodeAttachmentMessage(t *testing.T, response *httptest.ResponseRecorder) attachmentMessage { + t.Helper() + if response.Code != http.StatusCreated { + t.Fatalf("POST message: expected HTTP 201, got %d", response.Code) + } + var value map[string]json.RawMessage + if err := json.Unmarshal(response.Body.Bytes(), &value); err != nil || len(value) != 6 { + t.Fatal("Message must contain exactly the six lab4 contract fields, including attachment_ids") + } + var message attachmentMessage + if err := json.Unmarshal(response.Body.Bytes(), &message); err != nil || message.AttachmentIDs == nil { + t.Fatal("attachment_ids must be an array, including [] for a text-only message") + } + for _, id := range []string{message.ID, message.ChatID, message.SenderID} { + if _, err := uuid.Parse(id); err != nil { + t.Fatal("Message contains an invalid UUID") + } + } + if _, err := time.Parse(time.RFC3339Nano, message.CreatedAt); err != nil || !strings.HasSuffix(message.CreatedAt, "Z") { + t.Fatal("Message must preserve a server UTC timestamp") + } + if response.Header().Get("X-Request-Id") == "" || response.Header().Get("X-Instance-Id") != "message-pg-test" { + t.Fatal("message response is missing request/instance identity") + } + return message +} + +func (f *messageAttachmentFixture) history(t *testing.T) []attachmentMessage { + t.Helper() + r := f.call(f.bob, "GET", f.chat.ID, "", nil) + var page struct { + Items []attachmentMessage `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if r.Code != 200 || json.Unmarshal(r.Body.Bytes(), &page) != nil || page.NextCursor != nil { + t.Fatal("cannot read the complete synthetic history") + } + for _, message := range page.Items { + if message.AttachmentIDs == nil { + t.Fatal("history must include attachment_ids arrays") + } + } + return page.Items +} + +func (f *messageAttachmentFixture) assertLinks(t *testing.T, message attachmentMessage, expected []string) { + t.Helper() + if !sameOrderedAttachmentUUIDs(message.AttachmentIDs, expected) { + t.Fatal("Message attachment order or contents differ from request") + } + rows, err := f.pool.Query(f.ctx, `SELECT attachment_id::text, position, chat_id::text + FROM messages_attachments WHERE message_id=$1 ORDER BY position`, message.ID) + if err != nil { + t.Fatal("cannot inspect attachment links") + } + defer rows.Close() + var actual []string + for rows.Next() { + var id, chatID string + var position int + if rows.Scan(&id, &position, &chatID) != nil || position != len(actual) || chatID != f.chat.ID { + t.Fatal("attachment link has incorrect position or chat scope") + } + actual = append(actual, id) + } + if rows.Err() != nil || !slices.Equal(actual, expected) { + t.Fatal("database links do not match exactly the submitted attachment order") + } +} + +func sameOrderedAttachmentUUIDs(actual, expected []string) bool { + if actual == nil || len(actual) != len(expected) { + return false + } + for index, raw := range actual { + id, err := uuid.Parse(raw) + want, wantErr := uuid.Parse(expected[index]) + if err != nil || wantErr != nil || id != want { + return false + } + } + return true +} + +func sameHistoryMessage(actual, expected attachmentMessage) bool { + if !sameOrderedAttachmentUUIDs(actual.AttachmentIDs, expected.AttachmentIDs) { + return false + } + actual.AttachmentIDs = expected.AttachmentIDs + return reflect.DeepEqual(actual, expected) +} + +func (f *messageAttachmentFixture) snapshot(t *testing.T) map[string][]string { + t.Helper() + state := make(map[string][]string) + for _, table := range []string{"messages", "messages_attachments", "message_idempotency", "attachments"} { + // Table names are fixed test constants, never request input. + rows, err := f.pool.Query(f.ctx, "SELECT to_jsonb(item)::text FROM "+table+" AS item ORDER BY to_jsonb(item)::text") + if err != nil { + t.Fatal("cannot inspect transaction state") + } + for rows.Next() { + var row string + if rows.Scan(&row) != nil { + t.Fatal("cannot decode transaction state") + } + state[table] = append(state[table], row) + } + err = rows.Err() + rows.Close() + if err != nil { + t.Fatal("cannot finish inspecting transaction state") + } + } + return state +} + +func (f *messageAttachmentFixture) assertUnchanged(t *testing.T, before map[string][]string) { + t.Helper() + if !reflect.DeepEqual(before, f.snapshot(t)) { + t.Fatal("failed/conflicting request changed messages, links, idempotency or metadata") + } +} + +func TestMessageAttachmentsTextCompatibility(t *testing.T) { + f := newMessageAttachmentFixture(t) + key := uuid.NewString() + first := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, map[string]any{"text": "Legacy text 👋"})) + second := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody(first.Text, []string{}))) + if first.ChatID != f.chat.ID || first.SenderID != f.alice || first.Text != "Legacy text 👋" || !reflect.DeepEqual(first, second) { + t.Fatal("absent attachment_ids and [] must preserve the same text-only Message") + } + f.assertLinks(t, first, []string{}) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("text-only replay changed or duplicated history") + } +} + +func TestMessageAttachmentsOnlyAndOrderZeroToTen(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for count := 0; count <= 10; count++ { + t.Run(fmt.Sprintf("count_%d", count), func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids := append([]string{}, f.ids[:count]...) + text := "" + if count == 0 { + text = "Text without attachments" + } + key := uuid.NewString() + body := attachmentBody(text, ids) + if count == 1 { + body = attachmentBody(text, []string{strings.ToUpper(ids[0])}) + } + first := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)) + if count == 1 { + before := f.snapshot(t) + if replay := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)); !reflect.DeepEqual(replay, first) { + t.Fatal("identical JSON with uppercase attachment UUID must replay the same 201/Message") + } + f.assertUnchanged(t, before) + } + if first.Text != text || first.SenderID != f.bob || first.ChatID != f.chat.ID { + t.Fatal("message-only/attachment-only content or sender changed") + } + f.assertLinks(t, first, ids) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("history lost the message or attachment order") + } + }) + } +} + +func TestMessageAttachmentsInvalidReferencesAreAtomic(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for _, kind := range []string{"unknown_id", "attachment_from_other_chat", "foreign_chat"} { + t.Run(kind, func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids, actor, chatID := []string{f.ids[0], uuid.NewString()}, f.alice, f.chat.ID + if kind == "attachment_from_other_chat" { + ids[1] = f.foreign[0] + } else if kind == "foreign_chat" { + actor, chatID, ids = f.bob, f.other.ID, []string{f.foreign[0]} + } + before, key := f.snapshot(t), uuid.NewString() + response := f.call(actor, "POST", chatID, key, attachmentBody("Must not be written", ids)) + assertContractError(t, response, 404, "not_found") + f.assertUnchanged(t, before) + // A rejected attempt must not reserve this user's key as a final result. + valid := decodeAttachmentMessage(t, f.call(actor, "POST", f.chat.ID, key, attachmentBody("Valid after 404", f.ids[:1]))) + f.assertLinks(t, valid, f.ids[:1]) + }) + } +} + +func TestMessageAttachmentsInvalidListLeavesNoRows(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for _, kind := range []string{"duplicate", "eleven"} { + t.Run(kind, func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids := f.ids + if kind == "duplicate" { + ids = []string{f.ids[0], f.ids[0]} + } + before := f.snapshot(t) + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, uuid.NewString(), attachmentBody("invalid list", ids)), 400, "invalid_request") + f.assertUnchanged(t, before) + }) + } +} + +func TestMessageAttachmentsIdempotencyIncludesListAndOrder(t *testing.T) { + f := newMessageAttachmentFixture(t) + key, ids := uuid.NewString(), f.ids[:2] + upperIDs := []string{strings.ToUpper(ids[0]), strings.ToUpper(ids[1])} + if slices.Equal(upperIDs, ids) { + t.Fatal("normalization fixture must contain a UUID with hexadecimal letters") + } + body := attachmentBody("Same text", upperIDs) + first := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, body)) + before := f.snapshot(t) + for range 3 { + if replay := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, body)); !reflect.DeepEqual(replay, first) { + t.Fatal("same key/body did not return the same Message") + } + f.assertUnchanged(t, before) + } + for _, changed := range [][]string{{ids[1], ids[0]}, {ids[0], f.ids[2]}, {ids[0]}, {}} { + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Same text", changed)), 409, "conflict") + f.assertUnchanged(t, before) + } + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Changed text", ids)), 409, "conflict") + f.assertUnchanged(t, before) + f.assertLinks(t, first, ids) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("idempotency conflict changed history") + } +} + +func TestMessageAttachmentsConcurrentReplayDoesNotMultiplyLinks(t *testing.T) { + f := newMessageAttachmentFixture(t) + const workers = 8 + key, ids := uuid.NewString(), f.ids[:3] + gate := make(chan struct{}) + results := make(chan *httptest.ResponseRecorder, workers) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + <-gate + results <- f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Concurrent attachment replay", ids)) + }() + } + close(gate) + group.Wait() + close(results) + var first attachmentMessage + for response := range results { + message := decodeAttachmentMessage(t, response) + if first.ID == "" { + first = message + } else if !reflect.DeepEqual(message, first) { + t.Fatal("concurrent replay returned a different Message") + } + } + f.assertLinks(t, first, ids) + state := f.snapshot(t) + if len(state["messages"]) != 1 || len(state["messages_attachments"]) != len(ids) || len(state["message_idempotency"]) != 1 { + t.Fatal("concurrent replay multiplied message, link or idempotency rows") + } + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("concurrent replay changed history") + } +} + +func TestMessageAttachmentsLinkFailureRollsBackEverything(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids, key := f.ids[:2], uuid.NewString() + // The sequence survives rollback. It proves that the fault was reached only + // after the first link was visible inside the actual application transaction. + _, err := f.pool.Exec(f.ctx, `CREATE SEQUENCE test_link_fault_reached; + CREATE FUNCTION fail_second_test_link() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF (SELECT count(*) FROM messages_attachments WHERE message_id=NEW.message_id) >= 1 THEN + PERFORM nextval('test_link_fault_reached'); + RAISE EXCEPTION 'synthetic link failure'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_link_fault BEFORE INSERT ON messages_attachments + FOR EACH ROW EXECUTE FUNCTION fail_second_test_link();`) + if err != nil { + t.Fatal("cannot prepare isolated link fault") + } + before := f.snapshot(t) + response := f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Rollback all writes", ids)) + assertContractError(t, response, 500, "internal_error") + var faultReached bool + if f.pool.QueryRow(f.ctx, "SELECT is_called FROM test_link_fault_reached").Scan(&faultReached) != nil || !faultReached { + t.Fatal("test did not reach an injected link failure after one real link insert") + } + f.assertUnchanged(t, before) + if strings.Contains(response.Body.String(), "synthetic link failure") { + t.Fatal("response exposed internal SQL fault details") + } + if _, err := f.pool.Exec(f.ctx, "DROP TRIGGER test_link_fault ON messages_attachments"); err != nil { + t.Fatal("cannot clear isolated link fault") + } + message := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Rollback all writes", ids))) + f.assertLinks(t, message, ids) + state := f.snapshot(t) + if len(state["messages"]) != 1 || len(state["messages_attachments"]) != 2 || len(state["message_idempotency"]) != 1 { + t.Fatal("retry after transient link failure did not commit exactly one complete message") + } +} + +// The public API has immutable membership. Change it only in the isolated +// PostgreSQL fixture to exercise the same sender/key after losing access. +func TestMessageReplayRechecksCurrentChatMembership(t *testing.T) { + f := newMessageAttachmentFixture(t) + const key = "membership-recheck" + body := attachmentBody("prior authorized write", []string{}) + original := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)) + before := f.snapshot(t) + // Membership is immutable in the real contract, including at the FK. + // Remove that FK only in this disposable schema to simulate a stale grant + // with historical messages left behind. + if _, err := f.pool.Exec(f.ctx, "ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey"); err != nil { + t.Fatal("cannot enable membership fault in the isolated fixture") + } + tag, err := f.pool.Exec(f.ctx, "DELETE FROM chat_members WHERE chat_id=$1 AND user_id=$2", + f.chat.ID, f.bob) + if err != nil || tag.RowsAffected() != 1 { + t.Fatal("cannot remove sender's membership in the isolated fixture") + } + denied := f.call(f.bob, "POST", f.chat.ID, key, body) + var payload struct { + Error struct { + Code string `json:"code"` + RequestID string `json:"request_id"` + } `json:"error"` + } + if denied.Code != http.StatusNotFound || json.Unmarshal(denied.Body.Bytes(), &payload) != nil || + payload.Error.Code != "not_found" || payload.Error.RequestID == "" || + payload.Error.RequestID != denied.Header().Get("X-Request-Id") { + t.Fatalf("replay after losing access must return contract 404, got %d", denied.Code) + } + f.assertUnchanged(t, before) + if _, err := f.pool.Exec(f.ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", f.chat.ID, f.bob); err != nil { + t.Fatal("cannot restore sender's membership in the isolated fixture") + } + if got := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)); !reflect.DeepEqual(got, original) { + t.Fatal("restored access must replay the original Message") + } + f.assertUnchanged(t, before) +} diff --git a/student-tests/auth_tests/password_test.go b/student-tests/auth_tests/password_test.go new file mode 100644 index 00000000..983cda77 --- /dev/null +++ b/student-tests/auth_tests/password_test.go @@ -0,0 +1,126 @@ +package auth_tests + +import ( + "errors" + "strings" + "testing" + + "highload/internal/security" +) + +// These are tests of the selected Argon2id implementation, not a requirement +// that lab2 use particular parameters or a particular encoded hash format. +func TestPasswordRoundTripAndRandomSalt(t *testing.T) { + password := "Синтетический:пароль:😀" + first, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + second, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + if first == password || second == password || first == second { + t.Fatal("hash must differ from plaintext and have a fresh random salt") + } + if !strings.HasPrefix(first, "$argon2id$") { + t.Fatal("selected password implementation did not encode Argon2id") + } + for _, encoded := range []string{first, second} { + ok, err := security.VerifyPassword(password, encoded) + if err != nil || !ok { + t.Fatal("stored hash does not verify the original password") + } + ok, err = security.VerifyPassword(password+"x", encoded) + if err != nil || ok { + t.Fatal("wrong password was accepted or verifier failed") + } + } +} + +func TestPasswordDoesNotTruncateLongUTF8(t *testing.T) { + // Both passwords share much more than bcrypt's 72-byte limit; each is + // within the lab's 128-code-point maximum. + password := strings.Repeat("😀", 127) + "A" + different := strings.Repeat("😀", 127) + "B" + encoded, err := security.HashPassword(password) + if err != nil { + t.Fatal("maximum-length UTF-8 password cannot be hashed") + } + ok, err := security.VerifyPassword(password, encoded) + if err != nil || !ok { + t.Fatal("maximum-length UTF-8 password cannot be verified") + } + ok, err = security.VerifyPassword(different, encoded) + if err != nil || ok { + t.Fatal("password suffix was truncated") + } +} + +func TestPasswordWhitespaceIsSignificant(t *testing.T) { + password := " synthetic-password " + encoded, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + ok, err := security.VerifyPassword(strings.TrimSpace(password), encoded) + if err != nil || ok { + t.Fatal("password whitespace was discarded") + } +} + +func TestMalformedPasswordHashRejectedWithoutPanic(t *testing.T) { + valid, err := security.HashPassword("synthetic-password") + if err != nil { + t.Fatal("password hashing failed") + } + parts := strings.Split(valid, "$") + if len(parts) != 6 { + t.Fatal("unexpected encoded hash format") + } + cases := map[string]string{ + "empty": "", + "plaintext": "synthetic-password", + "wrong_algorithm": strings.Replace(valid, "argon2id", "argon2i", 1), + "wrong_version": strings.Replace(valid, "$v=19$", "$v=999$", 1), + "missing_version": strings.Replace(valid, "$v=19$", "$19$", 1), + "extra_field": valid + "$extra", + } + for name, params := range map[string]string{ + "zero_iterations": "m=65536,t=0,p=2", + "zero_threads": "m=65536,t=3,p=0", + "tiny_memory": "m=1,t=3,p=2", + "huge_memory": "m=4294967295,t=3,p=2", + "huge_iterations": "m=65536,t=4294967295,p=2", + "huge_threads": "m=65536,t=3,p=255", + "negative": "m=-1,t=3,p=2", + "non_numeric": "m=no,t=3,p=2", + "missing_param": "m=65536,t=3", + } { + copyParts := append([]string(nil), parts...) + copyParts[3] = params + cases[name] = strings.Join(copyParts, "$") + } + for name, field := range map[string]int{"salt": 4, "hash": 5} { + for label, value := range map[string]string{ + "empty": "", "bad_base64": "!!!", "wrong_size": "YQ", + } { + copyParts := append([]string(nil), parts...) + copyParts[field] = value + cases[name+"_"+label] = strings.Join(copyParts, "$") + } + } + for name, encoded := range cases { + t.Run(name, func(t *testing.T) { + defer func() { + if recover() != nil { + t.Error("malformed stored hash caused a panic") + } + }() + ok, err := security.VerifyPassword("synthetic-password", encoded) + if ok || !errors.Is(err, security.ErrInvalidHash) { + t.Error("malformed stored hash was not rejected") + } + }) + } +} diff --git a/student-tests/auth_tests/redis_test.go b/student-tests/auth_tests/redis_test.go new file mode 100644 index 00000000..9d8fd4d4 --- /dev/null +++ b/student-tests/auth_tests/redis_test.go @@ -0,0 +1,455 @@ +package auth_tests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "testing" + "time" + + "highload/internal/config" + "highload/internal/database" + repoerrors "highload/internal/repository" + "highload/internal/repository/postgres" + redisrepo "highload/internal/repository/redis" + "highload/internal/service" + httptransport "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" + redisclient "github.com/redis/go-redis/v9" +) + +func TestRedisAddressIsAnAddressNotAFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "postgres_url") + if err := os.WriteFile(path, []byte("postgres://fixture:fixture@postgres/messenger"), 0600); err != nil { + t.Fatal("cannot create synthetic config fixture") + } + t.Setenv("DATABASE_URL_FILE", path) + t.Setenv("REDIS_ADDR", "redis:6379") + t.Setenv("SESSION_TTL_SECONDS", "3600") + t.Setenv("SESSION_STORE", "redis") + t.Setenv("S3_ENDPOINT", "http://s3-fixture.invalid") + t.Setenv("S3_REGION", "us-east-1") + t.Setenv("S3_BUCKET", "health-fixture") + for variable, value := range map[string]string{ + "S3_ACCESS_KEY_ID_FILE": "fixture-access-key", + "S3_SECRET_ACCESS_KEY_FILE": "fixture-secret-key", + } { + keyFile := filepath.Join(dir, variable) + if err := os.WriteFile(keyFile, []byte(value), 0600); err != nil { + t.Fatal("cannot create synthetic S3 key file") + } + t.Setenv(variable, keyFile) + } + cfg, err := config.Load() + if err != nil { + t.Fatalf("cannot load synthetic config with REDIS_ADDR=redis:6379: %v", err) + } + if cfg.Database.RedisAddr != "redis:6379" { + t.Fatal("REDIS_ADDR did not reach the Redis configuration") + } +} + +type redisFixture struct { + ctx context.Context + client *redisclient.Client + repo *redisrepo.SessionRepository + keys []string + audit *sessionWriteAudit +} + +// Only observe writes through this test's client. No global key counting or +// FLUSH commands: other clients' sessions must not affect assertions/cleanup. +type sessionWriteAudit struct { + sync.Mutex + keys map[string]bool + writes int +} + +func (a *sessionWriteAudit) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (a *sessionWriteAudit) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (a *sessionWriteAudit) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + if cmd.Name() == "set" && len(cmd.Args()) > 1 { + if key, ok := cmd.Args()[1].(string); ok { + a.Lock() + a.keys[key] = true + a.writes++ + a.Unlock() + } + } + return next(ctx, cmd) + } +} +func (a *sessionWriteAudit) writeCount() int { a.Lock(); defer a.Unlock(); return a.writes } +func (a *sessionWriteAudit) ownedKeys() []string { + a.Lock() + defer a.Unlock() + keys := make([]string, 0, len(a.keys)) + for key := range a.keys { + keys = append(keys, key) + } + return keys +} + +type aofReplyHook struct { + reply any + err error +} + +func (h aofReplyHook) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (h aofReplyHook) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (h aofReplyHook) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + if cmd.Name() == "waitaof" { + // DEL still runs on the real test Redis. Only its persistence + // acknowledgement is fault-injected deterministically. + value := cmd.(*redisclient.Cmd) + value.SetVal(h.reply) + value.SetErr(h.err) + return h.err + } + return next(ctx, cmd) + } +} + +func TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout(t *testing.T) { + for _, tc := range []struct { + name string + reply any + err error + }{ + {"local_not_fsynced", []any{int64(0), int64(0)}, nil}, + {"only_replica_fsynced", []any{int64(0), int64(1)}, nil}, + {"malformed_reply", []any{int64(1)}, nil}, + {"command_failure", nil, errors.New("synthetic acknowledgement failure")}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + f.redis.client.AddHook(aofReplyHook{tc.reply, tc.err}) + request := httptest.NewRequest(http.MethodDelete, "/api/v1/auth/sessions/current", nil) + request.Header.Set("Authorization", "Bearer "+session.Token) + recorder := httptest.NewRecorder() + redisRouter(f, f.redis.client).ServeHTTP(recorder, request) + if recorder.Code != 500 && recorder.Code != 503 { + t.Fatalf("unconfirmed persistence must fail logout, got HTTP %d", recorder.Code) + } + if (tc.name == "local_not_fsynced" || tc.name == "only_replica_fsynced") && recorder.Code != 503 { + t.Errorf("valid reply without local acknowledgement must be HTTP 503, got %d", recorder.Code) + } + code := "internal_error" + if recorder.Code == 503 { + code = "unavailable" + } + assertContractError(t, recorder, recorder.Code, code) + hash := sha256.Sum256([]byte(session.Token)) + if f.redis.client.Exists(f.ctx, "session:"+hex.EncodeToString(hash[:])).Val() != 0 { + t.Fatal("fault fixture did not actually execute DEL before WAITAOF") + } + }) + } +} + +func newRedisFixture(t *testing.T) *redisFixture { + t.Helper() + addr := os.Getenv("AUTH_TEST_REDIS_ADDR") + if addr == "" { + t.Skip("Redis not verified: set AUTH_TEST_REDIS_ADDR to a dedicated test Redis with AOF") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + t.Cleanup(cancel) + client, err := database.NewRedis(ctx, addr) + if err != nil { + t.Fatal("dedicated test Redis is unavailable") + } + audit := &sessionWriteAudit{keys: make(map[string]bool)} + client.AddHook(audit) + f := &redisFixture{ctx: ctx, client: client, repo: redisrepo.NewSessionRepository(client), audit: audit} + t.Cleanup(func() { + cleanup, done := context.WithTimeout(context.Background(), 3*time.Second) + defer done() + // No FLUSHDB/FLUSHALL: delete only this test's own random token keys. + keys := append(f.audit.ownedKeys(), f.keys...) + if len(keys) > 0 { + if err := client.Del(cleanup, keys...).Err(); err != nil { + t.Error("test-owned Redis key cleanup failed") + } + } + _ = client.Close() + }) + return f +} + +func (f *redisFixture) track(token string) [32]byte { + hash := sha256.Sum256([]byte(token)) + f.keys = append(f.keys, "session:"+hex.EncodeToString(hash[:])) + return hash +} + +func TestRedisSessionHasTTLAndReadsDoNotExtendIt(t *testing.T) { + f := newRedisFixture(t) + hash := f.track("synthetic-" + uuid.NewString()) + created, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 5*time.Second) + if err != nil { + t.Fatal("cannot create Redis session") + } + key := f.keys[0] + before := f.client.PTTL(f.ctx, key).Val() + if before <= 0 || before > 5*time.Second { + t.Fatal("session key has no bounded positive TTL") + } + time.Sleep(120 * time.Millisecond) + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || loaded == nil { + t.Fatal("cannot read Redis session") + } + if loaded.UserID != created.UserID || !loaded.ExpiresAt.Equal(created.ExpiresAt) || + !loaded.CreatedAt.Equal(created.CreatedAt) { + t.Fatal("session identity or original timestamps changed on read") + } + after := f.client.PTTL(f.ctx, key).Val() + if after <= 0 || after >= before { + t.Fatal("reading refreshed or removed the session TTL") + } +} + +func TestRedisExpiredAndUnknownSessionsAreNotFound(t *testing.T) { + f := newRedisFixture(t) + hash := f.track(uuid.NewString()) + if _, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("unknown session must be not found") + } + if _, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 200*time.Millisecond); err != nil { + t.Fatal("cannot create short-lived test session") + } + time.Sleep(350 * time.Millisecond) + if _, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("expired Redis session is still available") + } +} + +func TestRedisLogoutOnlyRemovesSelectedSession(t *testing.T) { + f := newRedisFixture(t) + first, second := f.track(uuid.NewString()), f.track(uuid.NewString()) + userID := uuid.NewString() + for _, hash := range [][32]byte{first, second} { + if _, err := f.repo.Create(f.ctx, hash[:], userID, time.Minute); err != nil { + t.Fatal("cannot create independent sessions") + } + } + count, err := f.repo.RevokeSession(f.ctx, first[:]) + if err != nil || count != 1 { + t.Fatal("logout failed to remove exactly one session or acknowledge persistence") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, first[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("revoked session is still readable") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, second[:]); err != nil { + t.Fatal("logout removed another session") + } +} + +func TestRedisSessionsSurviveActualRestartWithoutExtendingTTL(t *testing.T) { + hook := os.Getenv("AUTH_TEST_REDIS_RESTART_HOOK") + if hook == "" { + t.Skip("Redis restart not verified: set an explicit dedicated test restart hook") + } + f := newRedisFixture(t) + active, revoked := f.track(uuid.NewString()), f.track(uuid.NewString()) + created, err := f.repo.Create(f.ctx, active[:], uuid.NewString(), 30*time.Second) + if err != nil { + t.Fatal("cannot create active session") + } + if _, err := f.repo.Create(f.ctx, revoked[:], created.UserID, time.Minute); err != nil { + t.Fatal("cannot create revocation fixture") + } + if _, err := f.repo.RevokeSession(f.ctx, revoked[:]); err != nil { + t.Fatal("cannot persist logout before restart") + } + before := f.client.PTTL(f.ctx, f.keys[0]).Val() + if err := exec.CommandContext(f.ctx, hook).Run(); err != nil { + t.Fatal("explicit Redis restart hook failed (private output suppressed)") + } + deadline := time.Now().Add(5 * time.Second) + for f.client.Ping(f.ctx).Err() != nil && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, active[:]) + if err != nil || loaded == nil || !loaded.ExpiresAt.Equal(created.ExpiresAt) { + t.Fatal("restart lost active session or changed its original deadline") + } + if after := f.client.PTTL(f.ctx, f.keys[0]).Val(); after <= 0 || after >= before { + t.Fatal("restart extended or lost remaining TTL") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, revoked[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("restart revived a revoked session") + } +} + +func TestRedisActiveSessionSurvivesRestartIndependentlyOfLogout(t *testing.T) { + hook := os.Getenv("AUTH_TEST_REDIS_RESTART_HOOK") + if hook == "" { + t.Skip("Redis restart not verified: set an explicit dedicated test restart hook") + } + f := newRedisFixture(t) + hash := f.track(uuid.NewString()) + created, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 30*time.Second) + if err != nil { + t.Fatal("cannot create active session") + } + before := f.client.PTTL(f.ctx, f.keys[0]).Val() + if err := exec.CommandContext(f.ctx, hook).Run(); err != nil { + t.Fatal("explicit Redis restart hook failed (private output suppressed)") + } + deadline := time.Now().Add(5 * time.Second) + for f.client.Ping(f.ctx).Err() != nil && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || loaded == nil || !loaded.ExpiresAt.Equal(created.ExpiresAt) || loaded.UserID != created.UserID { + t.Fatal("restart lost active session identity or changed its deadline") + } + if after := f.client.PTTL(f.ctx, f.keys[0]).Val(); after <= 0 || after >= before { + t.Fatal("restart extended or removed the session TTL") + } +} + +func refusedRedis(t *testing.T) *redisclient.Client { + t.Helper() + client := redisclient.NewClient(&redisclient.Options{ + Addr: "synthetic-unavailable:6379", MaxRetries: -1, + Dialer: func(context.Context, string, string) (net.Conn, error) { + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} + }, + }) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func TestRedisUnavailableKeepsUnavailableCategory(t *testing.T) { + repo := redisrepo.NewSessionRepository(refusedRedis(t)) + hash := sha256.Sum256([]byte("synthetic-failed-token")) + t.Run("create", func(t *testing.T) { + _, err := repo.Create(context.Background(), hash[:], uuid.NewString(), time.Minute) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + t.Run("read", func(t *testing.T) { + _, err := repo.GetSessionByTokenHash(context.Background(), hash[:]) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + t.Run("logout", func(t *testing.T) { + _, err := repo.RevokeSession(context.Background(), hash[:]) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + auth := service.NewAuthService(nil, repo, time.Hour) + _, err := auth.ValidateSession(context.Background(), "synthetic-error-context") + if !errors.Is(err, service.ErrServiceUnavailable) || !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("service lost Redis unavailable category or diagnostic context") + } + +} + +func redisRouter(f *fixture, client *redisclient.Client) http.Handler { + users := postgres.NewUserRepository(f.queries) + auth := service.NewAuthService(users, redisrepo.NewSessionRepository(client), time.Hour) + userService := service.NewUserService(users) + chats := service.NewChatService(postgres.NewChatRepository(f.queries, f.pool), users, userService) + messages := service.NewMessageService(postgres.NewMessageRepository(f.queries, f.pool), chats, userService) + objects := readyS3Storage() + attachments := service.NewAttachmentService(postgres.NewAttachmentRepository(f.queries, f.pool), chats, objects) + return httptransport.NewRouter(middleware.NewMiddleware(auth, "auth-test"), handler.NewAuthHandler(auth), + handler.NewHealthHandler(f.pool, client, objects), handler.NewChatHandler(chats), handler.NewUserHandler(userService), + handler.NewMessageHandler(messages), handler.NewAttachmentHandler(attachments)).Handler() +} + +func TestRedisCriticalOutageHTTPAndReadiness(t *testing.T) { + f := newFixture(t) + router := redisRouter(f, refusedRedis(t)) + for _, tc := range []struct { + name, path string + status int + }{ + {"liveness", "/health/live", 200}, + {"readiness", "/health/ready", 503}, + {"protected_request", "/api/v1/users/me", 503}, + } { + t.Run(tc.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, tc.path, nil) + request.Header.Set("Authorization", "Bearer synthetic-outage-token") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + if recorder.Code != tc.status { + t.Fatalf("want HTTP %d, got %d", tc.status, recorder.Code) + } + if tc.status == 503 { + assertContractError(t, recorder, 503, "unavailable") + } + }) + } +} + +func TestRedisHTTPAuthContract(t *testing.T) { + f := newFixture(t) + r := newRedisFixture(t) + server := httptest.NewServer(redisRouter(f, r.client)) + t.Cleanup(server.Close) + _, source, _, _ := runtime.Caller(0) + root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", "..")) + // Test the auth contract in this fixture. The public smoke now needs a + // durable broker and worker and runs against the full stand in lab5/acceptance.py. + acceptanceCtx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + cmd := exec.CommandContext(acceptanceCtx, "python3", "-B", filepath.Join(root, "student-tests/auth_contract.py"), + "--base-url", server.URL, "--ttl", "3600") + cmd.Dir = root + output, err := cmd.CombinedOutput() + if err == nil { + return + } + if acceptanceCtx.Err() != nil { + t.Fatal("HTTP auth acceptance subprocess timed out") + } + for _, line := range strings.Split(string(output), "\n") { + if strings.HasPrefix(line, "FAIL: test_") || strings.HasPrefix(line, "ERROR: test_") { + // Print only the method name, never credentials or subtest values. + name := strings.Fields(line)[1] + t.Log("failed auth case: " + name) + } + } + t.Fatal("HTTP auth acceptance failed with Redis-backed sessions") +} diff --git a/student-tests/auth_tests/restart_http_test.go b/student-tests/auth_tests/restart_http_test.go new file mode 100644 index 00000000..c85b1825 --- /dev/null +++ b/student-tests/auth_tests/restart_http_test.go @@ -0,0 +1,121 @@ +package auth_tests + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestRedisHTTPFullRestartKeepsDeadlineAndRevocation(t *testing.T) { + base, hook := os.Getenv("AUTH_TEST_BASE_URL"), os.Getenv("AUTH_TEST_STACK_RESTART_HOOK") + if base == "" || hook == "" { + t.Skip("set AUTH_TEST_BASE_URL and AUTH_TEST_STACK_RESTART_HOOK for a disposable API+stores stack") + } + for _, withLogout := range []bool{false, true} { + name := "active_only" + if withLogout { + name = "active_and_revoked" + } + t.Run(name, func(t *testing.T) { + f := newRedisFixture(t) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + client := &http.Client{Timeout: 5 * time.Second} + username := "restart_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:18] + password := "Synthetic:" + uuid.NewString() + call := func(method, path, token string, body any, basic bool, want int, dst any) { + t.Helper() + var data bytes.Buffer + if body != nil { + if json.NewEncoder(&data).Encode(body) != nil { + t.Fatal("cannot encode test request") + } + } + req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(base, "/")+path, &data) + if err != nil { + t.Fatal("cannot construct test request") + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if basic { + req.SetBasicAuth(username, password) + } else if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + response, err := client.Do(req) + if err != nil { + t.Fatal("restart HTTP request failed") + } + defer response.Body.Close() + if response.StatusCode != want { + t.Fatalf("%s %s: want %d, got %d", method, path, want, response.StatusCode) + } + if dst != nil && json.NewDecoder(response.Body).Decode(dst) != nil { + t.Fatal("invalid restart response JSON") + } + } + call("POST", "/api/v1/users", "", map[string]string{"username": username, "password": password, "display_name": "Restart fixture"}, false, 201, nil) + var active struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + call("POST", "/api/v1/auth/sessions", "", nil, true, 201, &active) + hash := f.track(active.Token) + before, err := f.repo.GetSessionByTokenHash(ctx, hash[:]) + if err != nil { + t.Fatal("cannot inspect issued session") + } + if !before.ExpiresAt.Equal(active.ExpiresAt) { + t.Fatal("HTTP and Redis expiry differ") + } + remaining := f.client.PTTL(ctx, f.keys[0]).Val() + if remaining < 2*time.Minute { + t.Fatal("normal restart suite requires long session TTL") + } + revoked := "" + if withLogout { + var other struct { + Token string `json:"token"` + } + call("POST", "/api/v1/auth/sessions", "", nil, true, 201, &other) + revoked = other.Token + f.track(revoked) + call("DELETE", "/api/v1/auth/sessions/current", revoked, nil, false, 204, nil) + call("GET", "/api/v1/users/me", revoked, nil, false, 401, nil) + } + if exec.CommandContext(ctx, hook).Run() != nil { + t.Fatal("real API+stores restart hook failed") + } + call("GET", "/health/live", "", nil, false, 200, nil) + call("GET", "/health/ready", "", nil, false, 200, nil) + call("GET", "/api/v1/users/me", active.Token, nil, false, 200, nil) + after, err := f.repo.GetSessionByTokenHash(ctx, hash[:]) + if err != nil { + t.Fatal("restart lost session") + } + if !after.ExpiresAt.Equal(before.ExpiresAt) || after.UserID != before.UserID || !after.CreatedAt.Equal(before.CreatedAt) { + t.Fatal("actual API+Redis restart changed session identity/deadline") + } + if ttl := f.client.PTTL(ctx, f.keys[0]).Val(); ttl <= 0 || ttl >= remaining { + t.Fatal("restart reset/removed original TTL") + } + if revoked != "" { + call("GET", "/api/v1/users/me", revoked, nil, false, 401, nil) + revokedHash := sha256.Sum256([]byte(revoked)) + if _, err = f.repo.GetSessionByTokenHash(ctx, revokedHash[:]); err == nil { + t.Fatal("revoked credential reappeared after restart") + } + } + }) + } +} diff --git a/student-tests/auth_tests/storage_test.go b/student-tests/auth_tests/storage_test.go new file mode 100644 index 00000000..87dca4a2 --- /dev/null +++ b/student-tests/auth_tests/storage_test.go @@ -0,0 +1,345 @@ +package auth_tests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "highload/internal/database/sqlc" + "highload/internal/model" + repository "highload/internal/repository/postgres" + redisrepo "highload/internal/repository/redis" + "highload/internal/security" + "highload/internal/service" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type fixture struct { + admin *pgxpool.Pool + pool *pgxpool.Pool + config *pgxpool.Config + queries *sqlc.Queries + users *service.UserService + auth *service.AuthService + redis *redisFixture + ctx context.Context +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + f := newDatabaseFixture(t, "AUTH_TEST_DATABASE_URL", 4) + f.redis = newRedisFixture(t) + userRepo := repository.NewUserRepository(f.queries) + f.users = service.NewUserService(userRepo) + f.auth = service.NewAuthService(userRepo, redisrepo.NewSessionRepository(f.redis.client), time.Hour) + return f +} + +// Shared PostgreSQL fixture: attachment transaction checks do not require Redis/S3. +func newDatabaseFixture(t *testing.T, environment string, maxConnections int32) *fixture { + t.Helper() + url := os.Getenv(environment) + if url == "" { + t.Skip("PENDING_INFRA: set " + environment + " for a dedicated test PostgreSQL") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + config, err := pgxpool.ParseConfig(url) + if err != nil { + t.Fatal("cannot parse test PostgreSQL configuration (credentials suppressed)") + } + config.MaxConns = maxConnections + config.MinConns = 0 + config.ConnConfig.ConnectTimeout = 3 * time.Second + config.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + admin, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + t.Fatal("cannot open test PostgreSQL pool") + } + t.Cleanup(admin.Close) + if err := admin.Ping(ctx); err != nil { + t.Fatal("test PostgreSQL is unavailable") + } + // Only this uniquely named schema is created and dropped. Existing tables + // and volumes are never removed. The role needs CREATE SCHEMA permission. + schema := "auth_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal("cannot create isolated test schema") + } + t.Cleanup(func() { + cleanupCtx, done := context.WithTimeout(context.Background(), 5*time.Second) + defer done() + if _, err := admin.Exec(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error("isolated test schema cleanup failed") + } + }) + config = config.Copy() + config.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, config.Copy()) + if err != nil { + t.Fatal("cannot connect to isolated test schema") + } + f := &fixture{admin: admin, config: config, ctx: ctx, pool: pool, queries: sqlc.New(pool)} + t.Cleanup(func() { f.pool.Close() }) + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate migration files") + } + migrations, err := filepath.Glob(filepath.Join(filepath.Dir(source), "..", "..", "db", "migrations", "*.sql")) + if err != nil || len(migrations) == 0 { + t.Fatal("migration SQL is missing") + } + sort.Strings(migrations) + for _, path := range migrations { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal("cannot read migration SQL") + } + // Current migrations are ordinary SQL with Goose Up/Down markers. + // This applies schema fixtures; it is NOT a test of Goose version tracking. + up, _, _ := strings.Cut(string(data), "-- +goose Down") + if !strings.Contains(up, "-- +goose Up") { + t.Fatal("test fixture expects SQL migrations with Goose Up/Down markers") + } + if _, err := f.pool.Exec(ctx, up, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal("migration cannot initialize isolated schema") + } + } + return f +} + +func (f *fixture) connect(t *testing.T, ttl time.Duration) { + t.Helper() + pool, err := pgxpool.NewWithConfig(f.ctx, f.config.Copy()) + if err != nil { + t.Fatal("cannot connect to isolated schema") + } + f.pool = pool + f.queries = sqlc.New(pool) + userRepo := repository.NewUserRepository(f.queries) + sessionRepo := redisrepo.NewSessionRepository(f.redis.client) + f.users = service.NewUserService(userRepo) + f.auth = service.NewAuthService(userRepo, sessionRepo, ttl) +} + +func (f *fixture) user(t *testing.T) (*model.User, string) { + t.Helper() + password := "Synthetic:" + uuid.NewString() + username := "auth_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:18] + user, err := f.users.CreateUser(f.ctx, username, "Auth storage", password) + if err != nil { + t.Fatal("cannot create synthetic user") + } + return user, password +} + +func (f *fixture) session(t *testing.T, user *model.User, password string) *service.CreatedSession { + t.Helper() + session, err := f.auth.CreateSession(f.ctx, user.Username, password) + if err != nil { + t.Fatal("cannot create synthetic session") + } + return session +} + +func TestStoragePasswordIsKDFAndDuplicateCannotOverwrite(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + stored, err := f.queries.GetUserByUsername(f.ctx, user.Username) + if err != nil || stored.PasswordHash == password || !strings.HasPrefix(stored.PasswordHash, "$argon2id$") { + t.Fatal("database did not preserve the selected password KDF") + } + ok, err := security.VerifyPassword(password, stored.PasswordHash) + if err != nil || !ok { + t.Fatal("stored password does not verify") + } + _, err = f.users.CreateUser(f.ctx, user.Username, "Overwrite", password+"x") + if !errors.Is(err, service.ErrAlreadyExists) { + t.Fatal("duplicate username was not rejected") + } + after, err := f.queries.GetUserByUsername(f.ctx, user.Username) + if err != nil || after.ID != stored.ID || after.PasswordHash != stored.PasswordHash || + after.DisplayName != stored.DisplayName { + t.Fatal("failed duplicate registration changed the original user") + } +} + +func TestStorageSessionHashAndReconnectPreserveDeadline(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + hash := sha256.Sum256([]byte(session.Token)) + key := "session:" + hex.EncodeToString(hash[:]) + row, err := f.redis.repo.GetSessionByTokenHash(f.ctx, hash[:]) + value, getErr := f.redis.client.Get(f.ctx, key).Result() + if err != nil || getErr != nil || strings.Contains(key, session.Token) || strings.Contains(value, session.Token) { + t.Fatal("selected session storage did not use SHA-256 of the opaque token") + } + for _, ownedKey := range f.redis.audit.ownedKeys() { + storedValue, err := f.redis.client.Get(f.ctx, ownedKey).Result() + if err != nil || strings.Contains(ownedKey, session.Token) || strings.Contains(storedValue, session.Token) { + t.Fatal("raw session token was persisted in Redis") + } + } + if f.redis.client.Exists(f.ctx, "session:"+session.Token, session.Token).Val() != 0 { + t.Fatal("raw token was stored as a Redis key") + } + if ttl := f.redis.client.PTTL(f.ctx, key).Val(); ttl <= 0 || ttl > time.Hour { + t.Fatal("session must have a positive bounded Redis TTL") + } + if session.TokenType != "Bearer" || len(session.Token) < 32 || row.RevokedAt != nil || + !row.ExpiresAt.Equal(session.ExpiresAt) || !row.ExpiresAt.After(row.CreatedAt) { + t.Fatal("invalid stored session metadata") + } + beforeDeadline := row.ExpiresAt + for range 3 { + id, err := f.auth.ValidateSession(f.ctx, session.Token) + if err != nil || id != user.ID { + t.Fatal("valid session rejected or mapped to another user") + } + } + f.pool.Close() + // A different process configuration must not reset already stored expiry. + f.connect(t, 24*time.Hour) + id, err := f.auth.ValidateSession(f.ctx, session.Token) + if err != nil || id != user.ID { + t.Fatal("new pool/services lost the existing session") + } + after, err := f.redis.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || !after.ExpiresAt.Equal(beforeDeadline) { + t.Fatal("reads or reconnection extended session TTL") + } +} + +func TestStorageWrongCredentialsNeverCreateSession(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + writes := f.redis.audit.writeCount() + for _, credentials := range [][2]string{ + {user.Username, password + "x"}, {"unknown_auth_user", password}, + } { + _, err := f.auth.CreateSession(f.ctx, credentials[0], credentials[1]) + if !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("wrong credentials did not yield the same unauthorized error") + } + } + if f.redis.audit.writeCount() != writes { + t.Fatal("failed login created a session") + } + for _, token := range []string{"", uuid.NewString()} { + if _, err := f.auth.ValidateSession(f.ctx, token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("empty or unknown token accepted") + } + } +} + +func TestStorageRevocationIsPersistentAndSessionSpecific(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + first, second := f.session(t, user, password), f.session(t, user, password) + if first.Token == second.Token { + t.Fatal("two logins reused the same token") + } + if err := f.auth.RevokeSession(f.ctx, first.Token); err != nil { + t.Fatal("logout failed") + } + f.pool.Close() + f.connect(t, time.Hour) + if _, err := f.auth.ValidateSession(f.ctx, first.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("revoked session revived after reconnect") + } + id, err := f.auth.ValidateSession(f.ctx, second.Token) + if err != nil || id != user.ID { + t.Fatal("logout affected another session") + } +} + +func TestStorageExpiredSessionCannotRevive(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + hash := sha256.Sum256([]byte(session.Token)) + key := "session:" + hex.EncodeToString(hash[:]) + if err := f.redis.client.PExpire(f.ctx, key, time.Millisecond).Err(); err != nil { + t.Fatal("cannot set an expired fixture") + } + time.Sleep(20 * time.Millisecond) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("expired session accepted") + } + f.pool.Close() + f.connect(t, 24*time.Hour) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("new TTL configuration revived an expired session") + } +} + +func TestStorageShortTTLExpiresAndReLoginWorks(t *testing.T) { + f := newFixture(t) + f.auth = service.NewAuthService(repository.NewUserRepository(f.queries), + redisrepo.NewSessionRepository(f.redis.client), 2*time.Second) + user, password := f.user(t) + session := f.session(t, user, password) + if remaining := time.Until(session.ExpiresAt); remaining <= 0 || remaining > 3*time.Second { + t.Fatal("two-second TTL was not used for the stored session") + } + if id, err := f.auth.ValidateSession(f.ctx, session.Token); err != nil || id != user.ID { + t.Fatal("fresh short-lived session was rejected") + } + time.Sleep(max(0, time.Until(session.ExpiresAt)/2)) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); err != nil { + t.Fatal("session expired before its original deadline") + } + timer := time.NewTimer(max(0, time.Until(session.ExpiresAt)) + 100*time.Millisecond) + defer timer.Stop() + select { + case <-timer.C: + case <-f.ctx.Done(): + t.Fatal("expiry fixture exceeded its deadline") + } + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("short-lived session was accepted after its stored deadline") + } + fresh := f.session(t, user, password) + if fresh.Token == session.Token { + t.Fatal("re-login reused the expired token") + } + if id, err := f.auth.ValidateSession(f.ctx, fresh.Token); err != nil || id != user.ID { + t.Fatal("re-login did not produce a usable session") + } +} + +func TestStorageChatRollbackAfterMemberInsertFailure(t *testing.T) { + f := newFixture(t) + user, _ := f.user(t) + repo := repository.NewChatRepository(f.queries, f.pool) + // A valid first member is inserted before the nonexistent member fails. + // Calling the repository bypasses prevalidation and tests real rollback. + _, err := repo.CreateChat(f.ctx, "Must roll back", []string{user.ID, uuid.NewString()}) + if err == nil { + t.Fatal("chat with nonexistent member unexpectedly committed") + } + for _, table := range []string{"chats", "chat_members"} { + var count int + err := f.pool.QueryRow(f.ctx, "SELECT count(*) FROM "+table).Scan(&count) + if err != nil || count != 0 { + t.Fatal("failed transaction left partial chat or membership rows") + } + } + created, err := repo.CreateChat(f.ctx, "Valid after rollback", []string{user.ID}) + if err != nil || created == nil { + t.Fatal("rollback poisoned subsequent valid transaction") + } +} diff --git a/student-tests/auth_tests/transport_test.go b/student-tests/auth_tests/transport_test.go new file mode 100644 index 00000000..61132e95 --- /dev/null +++ b/student-tests/auth_tests/transport_test.go @@ -0,0 +1,226 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "syscall" + "testing" + "time" + + "highload/internal/database/sqlc" + repository "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage/s3" + httptransport "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +const healthFixtureBucket = "health-fixture" + +type readyS3Client struct { + mu sync.Mutex + objects map[string][]byte +} + +func (s *readyS3Client) Do(req *http.Request) (*http.Response, error) { + bucketPath := "/" + healthFixtureBucket + response := &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody} + if req.Method == http.MethodHead && req.URL.Path == bucketPath { + return response, nil + } + if !strings.HasPrefix(req.URL.Path, bucketPath+"/attachments/") { + return nil, fmt.Errorf("unexpected path in S3 fixture") + } + s.mu.Lock() + defer s.mu.Unlock() + switch req.Method { + case http.MethodPut: + defer req.Body.Close() + data, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + s.objects[req.URL.Path] = data + case http.MethodGet: + data, ok := s.objects[req.URL.Path] + if !ok { + response.StatusCode = http.StatusNotFound + return response, nil + } + response.Body = io.NopCloser(bytes.NewReader(data)) + response.ContentLength = int64(len(data)) + default: + return nil, fmt.Errorf("unexpected method in S3 fixture") + } + return response, nil +} + +// Supply local S3 responses for readiness and the attachment upload in the HTTP smoke test. +func readyS3Storage() *s3.Storage { + return &s3.Storage{ + Bucket: healthFixtureBucket, + Client: awss3.New(awss3.Options{ + Region: "us-east-1", + BaseEndpoint: aws.String("http://s3-fixture.invalid"), + UsePathStyle: true, + Credentials: aws.AnonymousCredentials{}, + HTTPClient: &readyS3Client{objects: make(map[string][]byte)}, + }), + } +} + +type unavailableDB struct{} +type unavailableRow struct{} + +func (unavailableRow) Scan(...any) error { + return &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} +} + +func (unavailableDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, unavailableRow{}.Scan() +} + +func (unavailableDB) Query(context.Context, string, ...any) (pgx.Rows, error) { + return nil, unavailableRow{}.Scan() +} + +func (unavailableDB) QueryRow(context.Context, string, ...any) pgx.Row { + return unavailableRow{} +} + +func TestSQLUnavailableLoginStopsBeforeSessionCreation(t *testing.T) { + queries := sqlc.New(unavailableDB{}) + auth := service.NewAuthService(repository.NewUserRepository(queries), nil, time.Hour) + mid := middleware.NewMiddleware(auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("synthetic_user", "Synthetic:password") + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(handler.NewAuthHandler(auth).CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 503, "unavailable") +} + +func TestMiddlewareMalformedAuthWritesExactlyOneError(t *testing.T) { + // Invalid syntax must be rejected before asking an auth service or DB. + // A nil service deliberately makes an accidental dependency call visible. + mid := middleware.NewMiddleware(nil, "auth-test") + for _, header := range []string{"", "Basic invalid", "Bearer", "Bearer ", + "Bearer invalid token", "Digest invalid"} { + t.Run("malformed", func(t *testing.T) { + nextCalled := false + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { nextCalled = true }) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", header) + recorder := httptest.NewRecorder() + mid.RequestID(mid.Auth(next)).ServeHTTP(recorder, request) + if nextCalled || recorder.Code != http.StatusUnauthorized { + t.Fatal("malformed auth reached business handler or did not yield 401") + } + if recorder.Header().Get("WWW-Authenticate") != "Bearer realm=\"messenger\"" { + t.Fatal("missing contract Bearer challenge") + } + var payload struct { + Error struct { + Code string `json:"code"` + RequestID string `json:"request_id"` + } `json:"error"` + } + decoder := json.NewDecoder(recorder.Body) + if err := decoder.Decode(&payload); err != nil { + t.Fatal("401 is not one valid JSON object") + } + if err := decoder.Decode(new(any)); err != io.EOF { + t.Fatal("middleware wrote more than one response after 401") + } + if payload.Error.Code != "unauthorized" || payload.Error.RequestID == "" || + payload.Error.RequestID != recorder.Header().Get("X-Request-Id") { + t.Fatal("error request ID or code violates contract") + } + }) + } +} + +func TestStorageHTTPAuthRoutingAndLogout(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + userRepo := repository.NewUserRepository(f.queries) + chatRepo := repository.NewChatRepository(f.queries, f.pool) + chatService := service.NewChatService(chatRepo, userRepo, f.users) + messageService := service.NewMessageService(repository.NewMessageRepository(f.queries, f.pool), chatService, f.users) + objects := readyS3Storage() + attachmentService := service.NewAttachmentService(repository.NewAttachmentRepository(f.queries, f.pool), chatService, objects) + router := httptransport.NewRouter( + middleware.NewMiddleware(f.auth, "auth-test"), handler.NewAuthHandler(f.auth), + handler.NewHealthHandler(f.pool, f.redis.client, objects), + handler.NewChatHandler(chatService), handler.NewUserHandler(f.users), + handler.NewMessageHandler(messageService), + handler.NewAttachmentHandler(attachmentService), + ).Handler() + + call := func(method, path, token string, login bool) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(method, path, nil) + if login { + request.SetBasicAuth(user.Username, password) + } else if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + if recorder.Header().Get("X-Request-Id") == "" { + t.Fatal("auth response missing X-Request-Id") + } + return recorder + } + defer func() { + if recover() != nil { + t.Error("auth router panicked; check constructor/handler wiring") + } + }() + response := call("POST", "/api/v1/auth/sessions", "", true) + if response.Code != 201 { + t.Fatal("Basic login route did not return 201") + } + var session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + } + if err := json.Unmarshal(response.Body.Bytes(), &session); err != nil || + session.Token == "" || session.TokenType != "Bearer" { + t.Fatal("login route returned invalid session") + } + other := f.session(t, user, password) + response = call("GET", "/api/v1/users/me", session.Token, false) + if response.Code != 200 || strings.Contains(response.Body.String(), "password") { + t.Fatal("Bearer user route failed or exposed password fields") + } + response = call("DELETE", "/api/v1/auth/sessions/current", session.Token, false) + if response.Code != 204 || response.Body.Len() != 0 { + t.Fatal("contract logout path must return empty 204") + } + for _, route := range [][2]string{ + {"GET", "/api/v1/users/me"}, {"DELETE", "/api/v1/auth/sessions/current"}, + } { + response = call(route[0], route[1], session.Token, false) + if response.Code != 401 || response.Header().Get("WWW-Authenticate") != "Bearer realm=\"messenger\"" { + t.Fatal("revoked token was accepted or challenged with wrong scheme") + } + } + if response := call("GET", "/api/v1/users/me", other.Token, false); response.Code != 200 { + t.Fatal("logout invalidated an independent session of the same user") + } + +} diff --git a/student-tests/lab5/FAULT-HOOKS.md b/student-tests/lab5/FAULT-HOOKS.md new file mode 100644 index 00000000..aee72cdb --- /dev/null +++ b/student-tests/lab5/FAULT-HOOKS.md @@ -0,0 +1,44 @@ +# Как воспроизводятся отказы ЛР 5 + +`compose.fault.test.yaml` подключается только к изолированному Compose-проекту. +В нём API, relay и worker получают общий тестовый каталог. Реализация атомарно +переименовывает `POINT.arm` в `POINT.used`, записывает идентификатор в `POINT.hit` +и ждёт `POINT.release` или своего дедлайна. Адаптер сверяет job ID с PostgreSQL +перед остановкой процесса. Job, версия и исходный upload остаются прежними. + +| Имя сценария | Marker | Что проверяем | +| --- | --- | --- | +| `commit-before-publish` | `before_publish.hit` | Принятая job и единственная unsent запись outbox переживают SIGKILL relay. После запуска исходная job доходит до ready. | +| `publish-before-mark` | `after_publish.hit` | RabbitMQ подтвердил publish, а outbox ещё не помечен. Первый и повторный publish не создают два эффекта. Worker остановлен до обоих действий. | +| `object-before-ack` | `after_objects.hit` | Файлы уже в S3, а job ещё processing. SIGKILL worker оставляет lease. После его истечения новая попытка завершает ту же job. | +| `ready-before-ack` | `before_ack.hit` | Ready и варианты уже закоммичены. SIGKILL worker до ack проверяет повторную доставку без новой попытки. | +| `bounded-processing-timeout-and-prefetch` | `after_objects.hit` | Точку оставляем удержанной до дедлайна. Проверка требует `processing_timeout`, новую попытку, тот же job ID и фактический prefetch из RabbitMQ. | +| `retry-transient` | Изоляция только worker от S3 | RabbitMQ и PostgreSQL остаются доступны. Ожидаются retry в заданных пределах jitter, терминальный failed, явный replay после восстановления S3. | +| `fail-permanent` | Тестовая порча SHA синтетического staging | Worker видит mismatch, пишет failed за одну попытку. Тест восстанавливает SHA и проверяет replay через настоящую операторскую команду. Ранее принятый Message по тому же ключу остаётся доступен, новый POST с failed-вложением возвращает 409. | +| `mismatched-broker-envelope-DLQ` | Доставки с неверным job ID, attachment ID или pipeline version | RabbitMQ принимает и доставляет сообщения. Все три уходят в DLQ. Опубликованный результат остаётся прежним. | + +Сценарий сохранности брокера останавливает worker, проверяет наличие доставки в +durable очереди, останавливает relay перед рестартом RabbitMQ и сверяет число +сообщений после запуска. Так relay не может скрыть утрату повторной публикацией. +Повтор идёт через настоящую очередь. S3 проверяется фактическим listing, а +PostgreSQL показывает опубликованное поколение, попытки, outbox и аудит replay. + +## Интерфейс адаптера + +`pipeline_action.py` использует текущую схему Compose и `/app/jobs` в +`attachment-cleanup`. Базовые переменные передаёт `acceptance.py`. Оператор +задаёт `LAB5_TEST_FAULT_HOST_DIR`, содержащий `.lab5-test-project` с именем +изолированного проекта. Аргументы запускаются как argv: + +- `reset`, `worker-stop`, `worker-start`, `relay-start`, `broker-restart`, `restart-apis`. +- `arm POINT`, `await POINT ATTACHMENT_ID`, `crash POINT ATTACHMENT_ID`. `await` возвращает + JSON с точкой, attachment ID и job ID из маркера. `crash` считывает + статус задания и посылает SIGKILL в пределах окна точки одним вызовом. +- `activate POINT ATTACHMENT_ID` и `clear POINT ATTACHMENT_ID` для retry/failed. +- `inspect ATTACHMENT_ID`, `worker-limits`, `drain`, `failed ATTACHMENT_ID`, + `replay ATTACHMENT_ID`, `redeliver ATTACHMENT_ID`, `invalid-job ATTACHMENT_ID`. + +Код 78 сообщает, что необходимый компонент ещё не подключён. Другой ненулевой +код показывает ошибку адаптера или невыполненное условие. Если сценарий остановлен +вручную, можно вызвать `reset` в том же контексте проекта. HTTP-клиенты и вывод +runner не сохраняют учётные данные. diff --git a/student-tests/lab5/README.md b/student-tests/lab5/README.md new file mode 100644 index 00000000..250c7341 --- /dev/null +++ b/student-tests/lab5/README.md @@ -0,0 +1,91 @@ +# Приёмка лабораторной 5 на «4» + +Это студенческие проверки. Обязательные документы: [рубрика ЛР 5](../../README.md), +[накопительность и совместимость](../../COURSE.md), [семантика HTTP](../../contracts/README.md), +[OpenAPI и примеры](../../contracts/), [инфраструктурный контракт](../../contracts/infrastructure.json), +[публичные проверки](../../tests/README.md). Порядок защиты и доказательства сверяются с +[REPORT.md](../../REPORT.md), [CONTRIBUTING.md](../../CONTRIBUTING.md), [MAINTAINERS.md](../../MAINTAINERS.md), +[infra](../../infra/README.md), [examples](../../examples/requests.http) и [.env.example](../../.env.example). + +| Сценарий | Что требует контракт или задание | Где доказательство | +| --- | --- | --- | +| Сохранённый HTTP и тексты | Ключ 1–128 ASCII, отсутствующий/неверный → 400, конфликт → 409 без эффекта. область пользователя/чата. семантическое равенство JSON. конкурентные повторы. отозванный токен. timeout после подтверждённого commit | `http` / `idempotency-lb`, история чата. timeout на реальном клиентском сокете, прокси пересылает POST в LB и не отдаёт ответ клиенту | +| Проверка отзыва доступа на replay | Публичный API не меняет membership. В отдельной PostgreSQL-схеме тест имитирует отзыв уже выданного доступа, повторяет тот же POST и проверяет 404, состояние БД и неизменный replay после восстановления доступа | `TestMessageReplayRechecksCurrentChatMembership`. временное снятие FK ограничено тестовой схемой | +| LB и отказ процесса | ≥2 живых API за опубликованным адресом LB, одинаковый токен. каждый API по очереди получает SIGTERM и остаётся остановленным до проверки. чтение подтверждённых данных и запись ≤10 с. старый Message воспроизводится другой репликой | `idempotency-lb`, `failover`. Docker state, X-Instance-Id, запросы и время восстановления | +| Сохранность | Сессия, запись и ключ переживают реальный перезапуск API и хранилищ | `persistence`. StartedAt каждого процесса и прежний токен | +| Readiness, hard limits, rolling | Live/ready с контрактными телами. неготовый API доступен из сети LB и отвечает 503, публичные запросы идут мимо него. CPU/RAM API заданы и нет обхода LB. мигратор один. обновление ровно одной реплики за шаг | `http`, `unready`, `resources`, `migrations`, `rolling`. Для rolling требуется операторский hook с образом кандидата | +| ЛР 5, уровень 3 | Upload 202 после коммита метаданных, job и outbox. queued можно прикрепить, download → 409, worker остановлен. API и broker действительно перезапускаются, дубли не размножают запись. thumbnail и original | `TestAttachmentTxHTTP`, `pipeline-http`, `durable-broker-worker-stop-and-redelivery`, публичные make-цели | +| ЛР 5, уровень 4 | Crop и WebP: измерения из данных, checksum, MIME, bytes. лимиты bytes/pixels и невалидный PNG. fault после COMMIT / до publish, после записи объекта / до ack, после ready / до ack. lease, retries/jitter/terminal failed и операторский replay. staging cleanup | `pipeline-http`, `pipeline-faults`. маркеры процесса, PostgreSQL, S3 list, аудит replay, RabbitMQ DLQ, HTTP | +| Доказательства и рекомендации | Прогрев, интенсивность, длительность, RPS, ошибки, p50/95/99 и CPU/RAM, временная шкала и число записей. полный decode через установленный ImageMagick можно добавить флагом `--magick` | JSONL + логи make. Closed-loop даёт результаты конкретного стенда. проходного RPS нет | + +Readiness и отсутствие опубликованных портов API обязательны по инфраструктурному контракту. +CPU/RAM API, проверка SIGTERM с clean exit, Go проверка устаревшего доступа и rolling относятся к +дополнительным условиям этого прогона. Для rolling установлен дополнительный критерий: запросы +под нагрузкой не теряют подтверждённые сообщения и не возвращают ошибок во время обоих шагов. +Сценарий нельзя объявить пройденным по одному заголовку. Исследовательский трек «5» не входит в прогон. + +## Запуск + +Для итогового прогона использовали Python 3.14.6, Go из `go.mod` и Docker Compose. +У публичного клиента на Python 3.9 возможен отказ разбора дробных секунд +RFC 3339, которые выдаёт Go. Для локального HTTP используйте +изолированный учебный стенд. Команды из отдельной копии репозитория: + +```sh +./scripts/init-local.sh +export LAB5_HTTP_PORT=18085 LAB5_POSTGRES_PORT=25485 +export LAB5_TEST_FAULT_HOST_DIR="$(mktemp -d)" +printf '%s\n' highload-lab5-student > "$LAB5_TEST_FAULT_HOST_DIR/.lab5-test-project" +chmod 777 "$LAB5_TEST_FAULT_HOST_DIR" # только новый тестовый каталог, чтобы UID контейнера мог писать маркеры +export COMPOSE_PROJECT_NAME=highload-lab5-student +cpfiles() { + docker compose -p "$COMPOSE_PROJECT_NAME" -f compose.yaml \ + -f student-tests/lab5/compose.readiness.test.yaml \ + -f student-tests/lab5/compose.fault.test.yaml "$@" +} +cpfiles config --quiet +cpfiles up -d --build +make check +go test ./... # интеграционные Go-тесты показывают SKIP без их переменных окружения + +python3 student-tests/lab5/acceptance.py --scenario all \ + --base-url "http://127.0.0.1:$LAB5_HTTP_PORT" --project "$COMPOSE_PROJECT_NAME" --exercise \ + --compose-file compose.yaml --compose-file student-tests/lab5/compose.readiness.test.yaml \ + --compose-file student-tests/lab5/compose.fault.test.yaml \ + --store redis --store s3 --store rabbitmq --output /tmp/lab5-acceptance.jsonl +# После прогона при необходимости: cpfiles stop +``` + +Для быстрого профиля сохраняйте в **отдельной копии** `.env`: +`JOB_TIMEOUT_SECONDS=14`, `JOB_LEASE_SECONDS=18`, `JOB_MAX_ATTEMPTS=3`, +`JOB_RETRY_BASE_SECONDS=2`, `CLEANUP_GRACE_SECONDS=1`, `CLEANUP_INTERVAL_SECONDS=1`, +`STAGING_RETENTION_SECONDS=600`, `WORKER_CONCURRENCY=1`, `WORKER_PREFETCH=1`. +Учитывайте задержку Docker Desktop при физических рестартах. Миграциям нужно пустое окружение +для проверки полного чистого bootstrap. Итоговый runner включает +`make test`, `make test-persistence`, `make test-failover`. Вывод каждой цели сохраняется отдельно. +Тестовый хук получает то же имя проекта и те же Compose-файлы. + +При одной реплике можно отдельно запустить `--scenario http --base-url http://127.0.0.1:8080 --output ...`: +Этот режим проверяет контрактный HTTP. Для LB и failover нужна топология из двух реплик. +Флаги `--scenario` и `--compose-file` можно повторять. Для точечного повтора +укажите `--case` с именем проверки из вывода runner. Смена сервисных имён, +портов и диагностического образа curl задаётся в `--help`. Приёмка заменила +временный `baseline.py` из начала ЛР 5, ожидавший upload `201 ready`. +Ожидаемая отсутствующая инфраструктура/хук → **PENDING**, системная ошибка/невыполненное условие → +**FAIL**, только наблюдавшееся условие → **PASS**. Коды запуска: 0 — все выбранные проверки +прошли, 1 — есть FAIL, 2 — есть PENDING и нет FAIL. Строки без результатов не засчитываются. + +`--roll-hook /abs/path/to/hook` вызывается один раз на каждый исходный контейнер, с +`TARGET_CONTAINER_ID`, `TARGET_INSTANCE_ID`, `COMPOSE_PROJECT_NAME`, `BASE_URL`. Хук сам заменяет +контейнер образом кандидата, сохраняет остальное и возвращает 0 после подключения кандидата; +runner проверяет новый digest, связь с тем же LB, выдерживает окно нагрузки и проверяет, +что мигратор не запускался снова. Пока такого хука нет, сценарий **PENDING**. +Подробнее о fault hooks: [FAULT-HOOKS.md](FAULT-HOOKS.md). + +Переменные Go-проверок: `LAB4_TEST_DATABASE_URL` (историческое имя fixture), +`AUTH_TEST_DATABASE_URL`, `AUTH_TEST_REDIS_ADDR`, `MIGRATE_TEST_DATABASE_URL`. используйте +изолированные БД/Redis. Для опциональных прежних сценариев auth/restart см. их SKIP-строки. +Повторный прогон должен писать в новый файл. Не сохраняйте токены, env, секреты, содержимое БД +или полные логи контейнеров. + +Эксперимент на «5» описан в [VERSIONED.md](VERSIONED.md). Для него есть отдельный сценарий `pipeline-versioned`. diff --git a/student-tests/lab5/VERSIONED.md b/student-tests/lab5/VERSIONED.md new file mode 100644 index 00000000..984465ee --- /dev/null +++ b/student-tests/lab5/VERSIONED.md @@ -0,0 +1,53 @@ +# ЛР 5. Трек 2: версия pipeline и повторная доставка + +Гипотеза. Если worker публикует результат только для текущей версии и +действующего поколения lease, валидные старые сообщения и опоздавшая попытка +не изменят готовый результат v2. В S3 версии и поколения имеют разные ключи. +Подтверждённое задание текущей версии переживает сбой worker и доходит до ready. + +| Сценарий | Воздействие и критерий | +| --- | --- | +| Дубль v2 и устаревшая v1 | Довести v1 и v2 до ready. По очереди повторно опубликовать байты сохранённых валидных envelopе v2 и v1 в RabbitMQ. Очередь опустела, DLQ не выросла, длина истории попыток, ключи и опубликованные SHA не изменились. | +| Reorder v2 → v1 | Остановить relay перед upload. Отложить исходную запись outbox v1 через `available_at`, поставить v2 штатным `jobs upgrade`. Дождаться ready v2 и только тогда отпустить исходную v1. Сверить идентичность envelope и порядок `sent_at`. У v1 ноль попыток. | +| Crash до и после S3 | На v2 удержать worker перед первым PUT или сразу после записи всех объектов. Снять фактический listing (0 или 4 ключа поколения), послать SIGKILL до дедлайна и вернуть контейнер. Первая попытка истекает, новое поколение той же job завершает обработку. Исторический набор v1 остаётся целым, сироты и staging исчезают. | +| Запоздавший worker v1 | Удержать живой процесс после записи объектов v1. Запустить второго worker и довести v2 до ready, затем отпустить прежний процесс до исходных deadline и lease. Получить его лог `job finalization rejected`, проверить ACK, ноль новых попыток v2, прежние ключи v2 и удаление четырёх неопубликованных объектов v1. | + +При каждом переходе повторный `jobs upgrade` обязан вернуть тот же job ID и +`queued=false`. Проверяется один outbox для новой job до инъекции дополнительных +доставок. Используется один и тот же исходный PNG. Варианты проверяются через HTTP +и повторным чтением S3 по сохранённым ключам. Сравниваются SHA-256, размеры, +метаданные, номера версий и поколения, исходные байты. Опубликованные результаты +v1 и v2 различимы по SHA, а v2 воспроизводится на одинаковом исходнике во всей серии. + +Автоматизация подтверждает идемпотентность эффекта в наблюдавшихся окнах отказа. +Брокер может доставить payload несколько раз, рабочий процесс может повторно +вычислить и записать объекты. Успешным считается один опубликованный набор +текущей версии. Измерения относятся к одному Docker-хосту и одному размеру файла. + +## Запуск + +Поднимите отдельный учебный проект с чистыми томами по [основной инструкции](README.md), +с `compose.readiness.test.yaml` и `compose.fault.test.yaml`. Установите миграцию +с `attachment_jobs.published_original_key` и текущий образ, поддерживающий `jobs upgrade`. +В начале нужен один worker с `WORKER_CONCURRENCY=1` и `WORKER_PREFETCH=1`. +Тест временно масштабирует его до двух процессов в последнем сценарии каждой серии. +Фиксируйте фактические лимиты в локальной копии `.env`; профиль прогона использует +`JOB_TIMEOUT_SECONDS=30`, `JOB_LEASE_SECONDS=36`, `CLEANUP_GRACE_SECONDS=1` и +`CLEANUP_INTERVAL_SECONDS=1`. На медленном хосте окно старого worker можно увеличить, +с теми же контрольными условиями по времени относительно исходного lease. + +```sh +python3 student-tests/lab5/acceptance.py --scenario pipeline-versioned --repetitions 3 \ + --base-url "http://127.0.0.1:$LAB5_HTTP_PORT" --project "$COMPOSE_PROJECT_NAME" --exercise \ + --compose-file compose.yaml --compose-file student-tests/lab5/compose.readiness.test.yaml \ + --compose-file student-tests/lab5/compose.fault.test.yaml --output /tmp/lab5-versioned.jsonl +``` + +Выбор `--scenario all` по-прежнему запускает приёмку уровня 4. Трек добавляется +явно через `--scenario pipeline-versioned`. При отсутствии миграции или +каталога fault ожидается PENDING. Провал отдельного повтора сохраняется в +строке `versioned_trial` и даёт ненулевой код запуска. Итоговая строка +`versioned_summary` хранит число попыток эксперимента, min/median/max +наблюдаемого времени до ready и стабильные SHA-256 результата v2. +Для crash секундомер начинается перед SIGKILL. В остальных случаях он +охватывает ожидание ready после ответа штатной команды upgrade или запуска relay. diff --git a/student-tests/lab5/acceptance.py b/student-tests/lab5/acceptance.py new file mode 100644 index 00000000..88285cd9 --- /dev/null +++ b/student-tests/lab5/acceptance.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Run Lab 5 checks against an isolated Compose stand and write JSONL evidence.""" +import argparse +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +from urllib.parse import urlsplit + +sys.dont_write_bytecode = True +from checks import ROOT, Evidence, Failure, Pending, Probe, check, health, idempotency +from pipeline import Pipeline, make_decoder, upload_rejects, variants +from stand import Stand + +HERE = Path(__file__).resolve().parent +TEST_HOOK = HERE / "compose_action.py" + + +def arguments(): + cases = ("http", "idempotency-lb", "resources", "persistence", "failover", + "unready", "rolling", "migrations", "pipeline-http", "pipeline-faults", "pipeline-versioned", "public", "all") + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--scenario", choices=cases, action="append", required=True) + p.add_argument("--case", action="append", help="run only the named case within the selected scenario(s)") + p.add_argument("--base-url", default=os.environ.get("BASE_URL")) + p.add_argument("--ca-file", default=os.environ.get("CA_FILE")) + p.add_argument("--output", required=True, help="new JSONL file; no overwrite") + p.add_argument("--project", help="explicit disposable Compose project required for fault scenarios") + p.add_argument("--exercise", action="store_true", help="allow fault actions on this isolated project") + p.add_argument("--compose-file", action="append", default=[]) + for k, default in (("api", "api"), ("lb", "lb"), ("db", "postgres"), ("migrate", "migrate"), + ("db-user", "messenger"), ("db-name", "messenger")): + p.add_argument("--" + k, default=default) + p.add_argument("--store", action="append", help="additional persistence stores; repeat per service") + p.add_argument("--api-port", type=int, default=8080) + p.add_argument("--network", help="shared API/LB Docker network for diagnostics") + p.add_argument("--probe-image", default="rustfs/rustfs:1.0.4", + help="local image with curl, for direct health diagnostics from the LB namespace") + p.add_argument("--workers", type=int, default=6) + p.add_argument("--timeout", type=float, default=2) + p.add_argument("--window", type=float, default=3) + p.add_argument("--warmup", type=float, default=2) + p.add_argument("--pipeline-timeout", type=int, default=120) + p.add_argument("--repetitions", type=int, default=3, help="number of complete versioned experiment series") + p.add_argument("--magick", help="optional path to ImageMagick 7 for full image decode") + for k in ("restart", "stop", "unready", "ready", "roll", "pipeline"): + p.add_argument("--" + k + "-hook", type=str) + a = p.parse_args() + if not a.base_url: + p.error("--base-url (or BASE_URL) is required") + url = urlsplit(a.base_url) + if (url.scheme not in ("http", "https") or not url.hostname or url.username or url.password + or url.query or url.fragment or url.path not in ("", "/")): + p.error("use an HTTP(S) origin without credentials, query, path, or fragment") + if not 2 <= a.workers <= 64 or not 0 < a.timeout <= 5 or not 2 <= a.window <= 30 or not 0 < a.warmup <= 30: + p.error("workers=2..64, timeout=(0,5], window=2..30, warmup=(0,30]") + if not 1 <= a.pipeline_timeout <= 600: + p.error("pipeline timeout must be 1..600 s") + if not 1 <= a.repetitions <= 10: + p.error("repetitions must be 1..10") + if a.store is None: + a.store = ["redis", "s3", "rabbitmq"] + if a.exercise and (not a.project or a.project == "highload-lab5"): + p.error("--exercise requires --project for a disposable stand (not highload-lab5)") + return a + + +def control_env(a): + import json + return dict(os.environ, COMPOSE_PROJECT_NAME=a.project or "", + LAB5_TEST_COMPOSE_FILES=json.dumps([str(Path(f).resolve()) for f in a.compose_file]), + LAB5_TEST_API=a.api, LAB5_TEST_DB=a.db, LAB5_TEST_LB=a.lb, + LAB5_TEST_DB_USER=a.db_user, LAB5_TEST_DB_NAME=a.db_name, + LAB5_TEST_STORES=" ".join(a.store), PYTHONDONTWRITEBYTECODE="1") + + +def safe_to_exercise(args): + if not args.exercise: + raise Pending("requires --exercise and a dedicated --project") + + +def public_case(stand, mode, out): + a, e = stand.a, stand.e + if mode != "test": + safe_to_exercise(a) + apis = stand.public_topology() + env = control_env(a) + cmd = ["make", mode, "BASE_URL=" + stand.c.base, "PYTHON=" + sys.executable] + if a.ca_file: + cmd.append("CA_FILE=" + a.ca_file) + if mode != "test": + hook = a.restart_hook if mode == "test-persistence" else a.stop_hook + # The default test-only hook is scoped to the explicit project and uses + # the same scope passed to the student scenarios. + hook = hook or str(TEST_HOOK) + check(Path(hook).is_file() and os.access(hook, os.X_OK), "public action hook is not executable") + cmd.append("ACTION_SCRIPT=" + str(Path(hook).resolve())) + env["LAB5_HOOK_MODE"] = "restart" if mode == "test-persistence" else "stop-one" + path = Path(out).with_suffix(f".{mode}.log") + start = time.monotonic() + try: + with path.open("xb") as log: + result = subprocess.run(cmd, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, timeout=300) + e.event("public_run", command=mode, exit_code=result.returncode, log=path.name, + elapsed_s=time.monotonic() - start) + check(result.returncode == 0, f"make {mode} returned {result.returncode}; see {path.name}") + except subprocess.TimeoutExpired: + raise Failure(f"make {mode} exceeded its timeout; see {path.name}") from None + finally: + if mode == "test-failover": + before = {r["id"] for r in apis} + rows = [r for r in stand.containers(a.api) if r["id"] in before and r["state"] == "exited"] + if rows: + stand.command("restore after teacher failover", ["docker", "start", *[r["id"] for r in rows]]) + + +def main(): + a = arguments() + e = Evidence(a.output) + e.event("environment", base=a.base_url, project=a.project, workers=a.workers, timeout=a.timeout, + fault_window=a.window, warmup=a.warmup, python=platform.python_version(), + host=platform.machine(), os=platform.system(), logical_cpus=os.cpu_count()) + c = Probe(a.base_url, a.ca_file, a.timeout, e) + s = Stand(a, c) + decoder = make_decoder(s, a.magick) + if not a.pipeline_hook and "compose.fault.test.yaml" in [Path(f).name for f in a.compose_file]: + a.pipeline_hook = str(HERE / "pipeline_action.py") + pipeline = Pipeline(s, decoder) + + def exercise(f): + def wrapper(): + safe_to_exercise(a) + f() + return wrapper + + def idem_lb(): + s.public_topology() + idempotency(c, a.workers, known_instances=s.a_ids) + + def readiness(): + # Use the bundled hook only with its explicit edge network override. + explicit = [Path(f).name for f in a.compose_file] + if "compose.readiness.test.yaml" in explicit: + a.unready_hook = a.unready_hook or str(TEST_HOOK) + a.ready_hook = a.ready_hook or str(TEST_HOOK) + s.unready() + + def persist(): + previous = a.restart_hook + a.restart_hook = previous or str(TEST_HOOK) + try: + s.persistence() + finally: + a.restart_hook = previous + + def versioned(): + from versioned import series + series(pipeline, a.repetitions) + + # The test-only restart hook uses the same Compose file/service selection. + os.environ.update(control_env(a)) + os.environ["LAB5_HOOK_MODE"] = "restart" + groups = { + "http": [("health", lambda: health(c)), ("idempotency", lambda: idempotency(c, a.workers))], + "idempotency-lb": [("idempotency-across-LB", idem_lb)], + "resources": [("CPU-RAM-ports", s.resources)], + "migrations": [("single-migration-start", lambda: (s.public_topology(), s.migration_snapshot()))], + "persistence": [("persistent-idempotency", exercise(persist))], + "failover": [("each-SIGTERM-read-write-with-load", exercise(s.failover))], + "unready": [("LB-excludes-unready-API", exercise(readiness))], + "rolling": [("one-API-at-a-time-under-load", exercise(s.rolling))], + "pipeline-http": [("upload-byte-pixel-media-limits", lambda: upload_rejects(c)), + ("async-output-variants", lambda: variants(c, min(60, a.pipeline_timeout), decoder))], + "pipeline-faults": [ + ("durable-broker-worker-stop-and-redelivery", exercise(pipeline.durable)), + ("crash-COMMIT-before-publish", exercise(lambda: pipeline.crash("commit-before-publish"))), + ("crash-relay-after-confirm-before-mark", exercise(lambda: pipeline.crash("publish-before-mark"))), + ("crash-object-before-ack-lease", exercise(lambda: pipeline.crash("object-before-ack"))), + ("crash-ready-before-ack", exercise(lambda: pipeline.crash("ready-before-ack"))), + ("bounded-processing-timeout-and-prefetch", exercise(pipeline.bounded_processing)), + ("mismatched-broker-envelope-DLQ", exercise(pipeline.malformed_job)), + ("backoff-jitter-permanent-failed-replay", exercise(pipeline.retry_replay)), + ], + "pipeline-versioned": [("pipeline-versioned-series", exercise(versioned))], + "public": [(f"make-{m}", lambda m=m: public_case(s, m, a.output)) + for m in ("test", "test-persistence", "test-failover")], + } + if "all" in a.scenario: + selectors = [name for name in groups if name != "pipeline-versioned" or name in a.scenario] + # The LB case covers HTTP idempotency as well; avoid a duplicate run. + groups["http"] = groups["http"][:1] + else: + selectors = list(dict.fromkeys(a.scenario)) + if "idempotency-lb" in selectors and "http" in selectors: + groups["http"] = groups["http"][:1] + selected = [(label, fn) for group in selectors for label, fn in groups[group]] + if a.case: + check(set(a.case) <= {label for label, _ in selected}, "--case is outside the selected scenarios") + selected = [(label, fn) for label, fn in selected if label in a.case] + results = [] + try: + for label, fn in selected: + results.append(e.case(label, fn)) + e.event("summary", **{k: results.count(k) for k in ("PASS", "FAIL", "PENDING")}) + return 1 if "FAIL" in results else (2 if "PENDING" in results else 0) + finally: + e.file.close() + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (Failure, OSError) as exc: + print("FAIL:", exc if isinstance(exc, Failure) else type(exc).__name__, file=sys.stderr) + raise SystemExit(1) from None diff --git a/student-tests/lab5/checks.py b/student-tests/lab5/checks.py new file mode 100644 index 00000000..94990da4 --- /dev/null +++ b/student-tests/lab5/checks.py @@ -0,0 +1,279 @@ +"""Shared assertions and safe evidence for Lab 5 acceptance.""" +import base64 +from collections import Counter +import concurrent.futures +import http.client +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import math +from pathlib import Path +import ssl +import sys +import threading +import time +from urllib.parse import urlsplit +import uuid + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check +from resilience import history + +IO_ERRORS = (OSError, http.client.HTTPException) + + +class Pending(Exception): + """A scenario has no configured infrastructure; exit status is not zero.""" + + +class Evidence: + def __init__(self, path): + self.started = time.monotonic() + self.lock = threading.Lock() + self.phase = "setup" + self.file = Path(path).open("x", encoding="utf-8", buffering=1) + + def event(self, event, **fields): + with self.lock: + line = {"event": event, "t_s": round(time.monotonic() - self.started, 6), + "phase": self.phase, **fields} + self.file.write(json.dumps(line, ensure_ascii=False) + "\n") + return line + + def case(self, name, fn): + self.phase = name + print(f"RUN: {name}", flush=True) + try: + fn() + except Pending as exc: + status, note = "PENDING", str(exc) + except (Failure, ValueError, OSError, http.client.HTTPException) as exc: + status, note = "FAIL", str(exc) if isinstance(exc, Failure) else type(exc).__name__ + else: + status, note = "PASS", "" + self.event("result", status=status, note=note) + print(f"{status}: {name}" + (f" — {note}" if note else ""), flush=True) + return status + + +class ObservedOpener: + def __init__(self, opener, capture): + self.opener = opener + self.capture = capture + + def open(self, request, **kwargs): + from urllib.error import HTTPError + try: + response = self.opener.open(request, **kwargs) + except HTTPError as response: + self.capture(response) + raise + self.capture(response) + return response + + +class Probe(Client): + def __init__(self, base, ca, timeout, evidence): + super().__init__(base, ca, timeout=timeout) + self.ca = ca + self.evidence = evidence + self.local = threading.local() + self.opener = ObservedOpener(self.opener, self.capture) + + def capture(self, response): + self.local.status = response.code + self.local.instance = response.headers.get("X-Instance-Id") + self.local.request_id = response.headers.get("X-Request-Id") + + def call(self, method, path, *args, **kwargs): + self.local.status = self.local.instance = self.local.request_id = None + started = time.monotonic() + result = None + try: + result = super().call(method, path, *args, **kwargs) + return result + except IO_ERRORS as exc: + raise Failure(f"{method}: {type(exc).__name__} while reading response") from None + finally: + self.evidence.event("http", method=method, path=path.split("?")[0], + status=self.local.status, instance=self.local.instance, + request_id=self.local.request_id, ok=result is not None, + elapsed_s=time.monotonic() - started) + + +def key(): + return uuid.uuid4().hex + + +def message_path(chat): + return f"/api/v1/chats/{chat['id']}/messages" + + +def post(c, actor, chat, k, text, *, status=201, body=None, code=None): + r = c.call("POST", message_path(chat), status, actor=actor, + headers={} if k is None else {"Idempotency-Key": k}, + body={"text": text} if body is None else body, schema="Message") + if status == 201: + m = r["json"] + check((m["chat_id"], m["sender_id"], m["text"], m["attachment_ids"]) == + (chat["id"], actor["id"], text, (body or {}).get("attachment_ids", [])), + "Message fields differ from POST") + elif code: + check(r["json"]["error"]["code"] == code, f"expected error code {code}") + if status == 401: + check(r["headers"].get("WWW-Authenticate") == 'Bearer realm="messenger"', + "incorrect Bearer challenge") + return r + + +def health(c): + for endpoint in ("live", "ready"): + r = c.call("GET", f"/health/{endpoint}", schema="Health") + check(r["json"] == {"status": "ok"}, f"/health/{endpoint}: unexpected payload") + + +def assert_history(c, actor, chat, expected): + actual = history(c, actor, chat) + check(len(actual) == len(expected) and {m["id"]: m for m in actual} == + {m["id"]: m for m in expected}, "history has a missing, changed or extra Message") + + +def idempotency(c, workers, known_instances=None): + alice, bob, eve, chat = c.bootstrap() + other = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat", + body={"title": "Independent key scope", "member_ids": [alice["id"]]})["json"] + for bad in (None, "", "a" * 129, "a b", "a/b", "a@b", "nonASCII-é"): + post(c, alice, chat, bad, "must not persist", status=400, code="invalid_request") + assert_history(c, bob, chat, []) + + k, upper = "A", ("AZaz09._:-" * 13)[:128] + check(len(upper) == 128, "length-128 fixture") + first = post(c, alice, chat, k, "Сообщение")["json"] + for body in (None, {"attachment_ids": [], "text": "Сообщение"}, + {"text": "Сообщение", "attachment_ids": []}): + check(post(c, alice, chat, k, "Сообщение", body=body)["json"] == first, + "same semantic JSON must return the same 201 Message") + post(c, alice, chat, k, "different text", status=409, code="conflict") + assert_history(c, bob, chat, [first]) + boundary = post(c, alice, chat, upper, "128-byte key")["json"] + check(post(c, alice, chat, upper, "128-byte key")["json"] == boundary, "max-key replay changed") + user_scope = post(c, bob, chat, k, "another user")["json"] + chat_scope = post(c, alice, other, k, "another chat")["json"] + check(len({first["id"], user_scope["id"], chat_scope["id"]}) == 3, "key leaked across scopes") + assert_history(c, alice, other, [chat_scope]) + post(c, eve, chat, k, "Сообщение", status=404, code="not_found") + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + post(c, alice, chat, k, "Сообщение", status=401, code="unauthorized") + reg = alice["registration"] + basic = base64.b64encode(f'{reg["username"]}:{reg["password"]}'.encode()).decode() + session = c.call("POST", "/api/v1/auth/sessions", 201, schema="Session", + headers={"Authorization": "Basic " + basic})["json"] + alice = {**alice, "token": session["token"]} + check(post(c, alice, chat, k, "Сообщение")["json"] == first, "key unexpectedly scoped to session") + + concurrent_key = key() + barrier = threading.Barrier(workers, timeout=10) + + def race(_): + barrier.wait() + return post(c, alice, chat, concurrent_key, "same concurrent request") + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + responses = list(pool.map(race, range(workers))) + m = responses[0]["json"] + check(all(r["json"] == m for r in responses), "concurrent requests returned different Messages") + instances = {r["headers"]["X-Instance-Id"] for r in responses} + c.evidence.event("concurrent_posts", attempts=workers, instances=sorted(instances), message=m["id"]) + assert_history(c, bob, chat, [first, boundary, user_scope, m]) + + lost_key, lost_text = key(), "commit with response unobserved by sender" + recorded = timeout_after_commit(c, alice, bob, chat, lost_key, lost_text) + check(post(c, alice, chat, lost_key, lost_text)["json"] == recorded, + "retry after loss of response changed Message") + assert_history(c, bob, chat, [first, boundary, user_scope, m, recorded]) + if known_instances is not None: + check(len(instances) >= 2 and instances <= known_instances, + "concurrent POSTs did not reach two API replicas of this project") + + +def timeout_after_commit(c, actor, reader, chat, k, text): + # One shot loopback relay. The caller times out waiting for a response + # which the relay intentionally withholds after forwarding through the LB. + target = urlsplit(c.base) + cls = http.client.HTTPConnection if target.scheme == "http" else http.client.HTTPSConnection + opts = {} if target.scheme == "http" else {"context": ssl.create_default_context(cafile=c.ca)} + received = {} + finished, release = threading.Event(), threading.Event() + + class DropResponse(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + upstream = cls(target.hostname, target.port, timeout=c.timeout, **opts) + try: + payload = self.rfile.read(int(self.headers["Content-Length"])) + headers = {h: self.headers[h] for h in ("Content-Type", "Idempotency-Key", "Authorization")} + upstream.request("POST", self.path, body=payload, headers=headers) + with upstream.getresponse() as response: + received["status"] = response.status + received["instance"] = response.getheader("X-Instance-Id") + if response.status == 201: + received["message"] = json.loads(response.read(1024 * 1024)) + except (ValueError, OSError, http.client.HTTPException): + received["error"] = "upstream connection or invalid response" + finally: + upstream.close() + finished.set() + release.wait(8) + + with HTTPServer(("127.0.0.1", 0), DropResponse) as relay: + relay.timeout = 3 + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as jobs: + served = jobs.submit(relay.handle_request) + downstream = http.client.HTTPConnection(*relay.server_address, timeout=0.5) + timed_out = False + try: + downstream.request("POST", message_path(chat), body=json.dumps({"text": text}), + headers={"Content-Type": "application/json", + "Idempotency-Key": k, "Authorization": "Bearer " + actor["token"]}) + try: + downstream.getresponse().close() + except TimeoutError: + timed_out = True + check(timed_out, "client did not actually time out on the dropped response") + check(finished.wait(6) and received.get("status") == 201 and received.get("instance"), + "relay did not observe committed upstream 201") + outcome = received["message"] + check(outcome in history(c, reader, chat), "original timed-out invocation did not durably commit") + c.evidence.event("lost_response", downstream="TimeoutError", upstream_status=201, + instance=received["instance"], message=outcome["id"]) + return outcome + finally: + release.set() + downstream.close() + served.result(timeout=10) + + +def wait(fn, seconds, label): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + value = fn() + except Failure: + value = None + if value and time.monotonic() <= deadline: + return value + time.sleep(0.1) + raise Failure(label) + + +def summarize(evidence, records, elapsed, window="full"): + times = sorted(r["elapsed_s"] for r in records if r["ok"]) + percentiles = {f"p{p}_ms": round(times[math.ceil(len(times)*p/100)-1]*1000, 3) if times else None + for p in (50, 95, 99)} + evidence.event("load_summary", window=window, elapsed_s=elapsed, attempts=len(records), successes=len(times), + successful_rps=len(times) / elapsed, failures=len(records)-len(times), + error_ratio=(len(records)-len(times))/len(records) if records else None, + statuses=dict(Counter(str(r["status"]) for r in records)), **percentiles) diff --git a/student-tests/lab5/compose.fault.test.yaml b/student-tests/lab5/compose.fault.test.yaml new file mode 100644 index 00000000..59d56921 --- /dev/null +++ b/student-tests/lab5/compose.fault.test.yaml @@ -0,0 +1,40 @@ +# Optional only on the isolated Lab 5 acceptance project. +# The worker can lose S3 while retaining its database and broker connections. +x-fault-env: &fault-env + LAB5_FAULT_DIR: /lab5-test-faults +x-fault-mount: &fault-mount + type: bind + source: ${LAB5_TEST_FAULT_HOST_DIR:?create a fresh writable fault directory} + target: /lab5-test-faults + +services: + api: + environment: *fault-env + volumes: [*fault-mount] + networks: + default: {} + lab5_edge: {} + lab5_object: {} + relay: + environment: *fault-env + volumes: [*fault-mount] + worker: + environment: *fault-env + volumes: [*fault-mount] + networks: + default: {} + lab5_object: {} + attachment-cleanup: + networks: + default: {} + lab5_object: {} + s3: + networks: + lab5_object: {} + s3-init: + networks: + default: {} + lab5_object: {} + +networks: + lab5_object: {} diff --git a/student-tests/lab5/compose.readiness.test.yaml b/student-tests/lab5/compose.readiness.test.yaml new file mode 100644 index 00000000..9619e31b --- /dev/null +++ b/student-tests/lab5/compose.readiness.test.yaml @@ -0,0 +1,13 @@ +# Optional on a dedicated test project. The LB can still reach API when a +# single API is disconnected from the private dependency network. +services: + api: + networks: + default: {} + lab5_edge: {} + lb: + networks: + lab5_edge: {} + +networks: + lab5_edge: {} diff --git a/student-tests/lab5/compose_action.py b/student-tests/lab5/compose_action.py new file mode 100755 index 00000000..1e3bc0dc --- /dev/null +++ b/student-tests/lab5/compose_action.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Student hook for teacher tests: explicit disposable Compose project only.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +PROJECT = os.environ.get("COMPOSE_PROJECT_NAME") +API = os.environ.get("LAB5_TEST_API", "api") +LB = os.environ.get("LAB5_TEST_LB", "lb") +DB = os.environ.get("LAB5_TEST_DB", "postgres") +STORES = os.environ.get("LAB5_TEST_STORES", "redis s3").split() + + +def run(argv, timeout=30): + result = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + if result.returncode: + raise RuntimeError("Docker command failed") + return result.stdout.decode().strip() + + +def main(): + if not PROJECT or PROJECT == "highload-lab5": + raise RuntimeError("disposable COMPOSE_PROJECT_NAME is required") + compose = ["docker", "compose", "--project-name", PROJECT] + for file in json.loads(os.environ.get("LAB5_TEST_COMPOSE_FILES", "[]")): + compose.extend(["--file", file]) + + def ids(service): + return run(compose + ["ps", "-q", service]).split() + + mode = os.environ.get("LAB5_HOOK_MODE") + if mode in ("unready", "ready"): + cid = os.environ.get("TARGET_CONTAINER_ID") + if cid not in ids(API): + # Compose ps abbreviates IDs whereas inspect returns the full ID. + if not cid or not any(cid.startswith(i) for i in ids(API)): + raise RuntimeError("selected API is not in the disposable project") + net = PROJECT + "_default" + if mode == "unready": + run(["docker", "network", "disconnect", net, cid]) + else: + current = json.loads(run(["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", cid])) + if net not in current: + run(["docker", "network", "connect", "--alias", API, net, cid]) + return + if mode == "stop-one": + alias = os.environ.get("TARGET_INSTANCE_ID") + matches = [i for i in ids(API) if run(["docker", "inspect", "-f", "{{.Config.Hostname}}", i]) == alias] + if len(matches) != 1: + raise RuntimeError("could not resolve exactly one running API alias") + run(["docker", "stop", "--signal", "SIGTERM", "--time", "5", matches[0]], 10) + return + if os.environ.get("LAB5_HOOK_MODE") != "restart": + raise RuntimeError("LAB5_HOOK_MODE must be restart or stop-one") + + def restart(selected, budget): + if not selected: + raise RuntimeError("selected service is not running") + run(["docker", "restart", "--time", "8", *selected], budget) + + def healthy(selected): + deadline = time.monotonic() + 25 + for cid in selected: + while time.monotonic() < deadline: + state = run(["docker", "inspect", "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", cid]) + if state in ("healthy", "running"): + break + time.sleep(0.25) + else: + raise RuntimeError("container did not become healthy") + + stores = [ids(s) for s in (DB, *STORES)] + if any(not g for g in stores): + raise RuntimeError("missing configured persistence store") + store_ids = [cid for group in stores for cid in group] + apis = ids(API) + restart(store_ids, 30) + healthy(store_ids) + restart(apis, 24) + healthy(apis) + restart(ids(LB), 12) + + +if __name__ == "__main__": + try: + main() + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as exc: + print("Compose hook failed:", exc if isinstance(exc, RuntimeError) else type(exc).__name__, file=sys.stderr) + sys.exit(1) diff --git a/student-tests/lab5/imageprobe/main.go b/student-tests/lab5/imageprobe/main.go new file mode 100644 index 00000000..4c041eab --- /dev/null +++ b/student-tests/lab5/imageprobe/main.go @@ -0,0 +1,115 @@ +// imageprobe exercises the ImageMagick runtime in its Docker image. +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" +) + +func magick(args ...string) string { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "magick", args...).CombinedOutput() + if err != nil { + log.Fatalf("magick %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} + +func checkImage(path, expected string, signature []byte, offset int) { + raw, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + if len(raw) < offset+len(signature) || !bytes.Equal(raw[offset:offset+len(signature)], signature) { + log.Fatalf("%s has an unexpected header", path) + } + if got := magick("identify", "+ping", "-regard-warnings", "-format", "%m %wx%h", path); got != expected { + log.Fatalf("unexpected output format or dimensions: %s", got) + } + fmt.Printf("PASS: decoded %s\n", expected) +} + +func checkDeadline(dir string) error { + fifo := filepath.Join(dir, "incomplete.png") + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "magick", "identify", "png:"+fifo) + var stderr bytes.Buffer + cmd.Stderr = &stderr + start := time.Now() + if err := cmd.Start(); err != nil { + return err + } + var writer *os.File + for ctx.Err() == nil { + file, err := os.OpenFile(fifo, os.O_WRONLY|syscall.O_NONBLOCK, 0) + if err == nil { + writer = file + break + } + if !errors.Is(err, syscall.ENXIO) { + cancel() + _ = cmd.Wait() + return err + } + time.Sleep(10 * time.Millisecond) + } + if writer == nil { + return fmt.Errorf("magick never opened the input FIFO: %v: %s", cmd.Wait(), stderr.String()) + } + defer writer.Close() + if _, err := writer.Write([]byte("\x89PNG\r\n\x1a\n")); err != nil { + cancel() + _ = cmd.Wait() + return err + } + err := cmd.Wait() + elapsed := time.Since(start) + var exit *exec.ExitError + if !errors.Is(ctx.Err(), context.DeadlineExceeded) || !errors.As(err, &exit) || + exit.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL || elapsed > 3*time.Second { + return fmt.Errorf("expected deadline and SIGKILL, got %v, %v after %s: %s", ctx.Err(), err, elapsed, stderr.String()) + } + fmt.Printf("PASS: CommandContext killed and reaped blocked magick after %s (deadline 1s)\n", elapsed.Round(time.Millisecond)) + return nil +} + +func main() { + log.SetFlags(0) + if len(os.Args) != 2 { + log.Fatal("usage: imageprobe /path/to/sample.png") + } + dir, err := os.MkdirTemp("", "imageprobe-") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + fmt.Println(strings.SplitN(magick("-version"), "\n", 2)[0]) + input := os.Args[1] + if got := magick("identify", "-format", "%m %wx%h", input); got != "PNG 320x200" { + log.Fatalf("unexpected fixture: %s", got) + } + jpeg := filepath.Join(dir, "sample.jpg") + webp := filepath.Join(dir, "sample.webp") + magick(input, "-auto-orient", "-resize", "256x256>", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "-quality", "85", jpeg) + magick(input, "-auto-orient", "-resize", "1280x1280>", "-strip", "-quality", "80", webp) + checkImage(jpeg, "JPEG 256x160", []byte{0xff, 0xd8, 0xff}, 0) + checkImage(webp, "WEBP 320x200", []byte("WEBP"), 8) + if err := checkDeadline(dir); err != nil { + log.Fatal(err) + } +} diff --git a/student-tests/lab5/pipeline.py b/student-tests/lab5/pipeline.py new file mode 100644 index 00000000..0eb776a0 --- /dev/null +++ b/student-tests/lab5/pipeline.py @@ -0,0 +1,386 @@ +"""Lab 5 grade 3+4: HTTP outputs and deterministic operator-injected faults.""" +import hashlib +import json +from pathlib import Path +import os +import struct +import tempfile +import time +import uuid +import zlib + +from checks import ROOT, Failure, Pending, check, post, key, assert_history, wait +from image_probe import jpeg_size + +STAGING = ("queued", "processing") +VARIANTS = { + "thumbnail": ("image/jpeg", 256, 160), + "crop": ("image/jpeg", 128, 128), + "optimized": ("image/webp", 320, 200), +} +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def attachment_path(att): + return f"/api/v1/attachments/{att['id']}" + + +def metadata(c, actor, att): + return c.call("GET", attachment_path(att), actor=actor, schema="Attachment")["json"] + + +def until_status(c, actor, att, desired, timeout): + started = time.monotonic() + while time.monotonic()-started < timeout: + current = metadata(c, actor, att) + state = current["status"] + if state == desired: + check(time.monotonic()-started <= timeout, f"{desired} observed after deadline") + c.evidence.event("attachment_state", attachment=att["id"], status=state, + seconds=time.monotonic()-started) + return current + check(state in STAGING, f"wanted {desired}, reached terminal state {state}") + time.sleep(0.25) + raise Failure(f"attachment never reached {desired} within {timeout} seconds") + + +def png_chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def oversized_pixels(): + # 40,010,000 pixels, but a small valid PNG; never allocate the decoded image. + width, height = 10000, 4001 + enc = zlib.compressobj() + row = bytes(width+1) + compressed = b"".join(enc.compress(row) for _ in range(height)) + enc.flush() + return (SAMPLE[:8] + png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0)) + + png_chunk(b"IDAT", compressed) + png_chunk(b"IEND", b"")) + + +def upload_rejects(c): + alice, _, _, chat = c.bootstrap() + path = f"/api/v1/chats/{chat['id']}/attachments?filename=limits.png" + n = 10 * 1024 * 1024 + padded = SAMPLE[:-12] + png_chunk(b"npAD", bytes(n + 1 - len(SAMPLE) - 12)) + SAMPLE[-12:] + check(len(padded) == n + 1, "invalid bytes-boundary fixture") + for data, status, code in ( + (padded, 413, "payload_too_large"), + (oversized_pixels(), 413, "payload_too_large"), + (b"not a PNG", 415, "unsupported_media_type"), + (SAMPLE[:30], 415, "unsupported_media_type"), + ): + r = c.call("POST", path, status, actor=alice, raw=data, headers={"Content-Type": "image/png"}) + check(r["json"]["error"]["code"] == code, f"upload must return {code}") + + +def decoded_webp_size(data): + # Read actual bitstream dimensions. Check the complete RIFF framing, not + # metadata or filename alone; full pixel decode can also run via ImageMagick. + check(len(data) >= 20 and data[:4] == b"RIFF" and data[8:12] == b"WEBP" + and int.from_bytes(data[4:8], "little") + 8 == len(data), "invalid WebP RIFF") + i, canvas, image = 12, None, None + while i < len(data): + check(i + 8 <= len(data), "truncated WebP chunk") + size = int.from_bytes(data[i+4:i+8], "little") + name, start = data[i:i+4], i+8 + i = start+size+(size % 2) + check(i <= len(data), "truncated WebP chunk body") + if name == b"VP8X": + check(size == 10, "invalid WebP canvas") + canvas = tuple(1 + int.from_bytes(data[start+x:start+x+3], "little") for x in (4, 7)) + elif name == b"VP8 ": + check(size >= 10 and data[start+3:start+6] == b"\x9d\x01\x2a", "invalid VP8 bitstream") + image = tuple(int.from_bytes(data[start+x:start+x+2], "little") & 0x3fff for x in (6, 8)) + elif name == b"VP8L": + check(size >= 5 and data[start] == 0x2f, "invalid VP8L bitstream") + bits = int.from_bytes(data[start+1:start+5], "little") + image = (1+(bits & 0x3fff), 1+((bits >> 14) & 0x3fff)) + check(image and (canvas is None or canvas == image), "WebP missing image or inconsistent canvas") + return image + + +def verify_ready(c, reader, queued, ready, *, decoder=None): + stable = ("id", "chat_id", "owner_id", "filename", "content_type", "sha256", "size_bytes", "created_at") + check(all(ready[k] == queued[k] for k in stable), "original metadata changed from queued to ready") + check(ready["status"] == "ready" and ready["error_code"] is None, "invalid ready state") + entries = {v["name"]: v for v in ready["variants"]} + check(len(ready["variants"]) == len(entries) and set(entries) >= set(VARIANTS), + "ready requires exactly one thumbnail, crop and optimized variant each") + path = attachment_path(ready) + "/content" + original = c.call("GET", path, actor=reader) + check(original["bytes"] == SAMPLE and original["headers"].get_content_type() == "image/png", + "published original changed") + for name, (mime, w, h) in VARIANTS.items(): + v = entries[name] + r = c.call("GET", f"{path}?variant={name}", actor=reader) + data = r["bytes"] + check((v["content_type"], v["width"], v["height"]) == (mime, w, h), f"{name}: metadata differs") + check(r["headers"].get_content_type() == mime and len(data) == v["size_bytes"] + and hashlib.sha256(data).hexdigest() == v["sha256"], f"{name}: MIME/size/hash mismatch") + size = jpeg_size(data) if mime == "image/jpeg" else decoded_webp_size(data) + check(size == (w, h), f"{name}: wrong dimensions in image bytes") + if decoder: + decoder(data, mime, w, h) + c.evidence.event("variant", name=name, width=w, height=h, bytes=len(data), sha256=v["sha256"]) + return ready + + +def variants(c, timeout, decoder): + alice, bob, eve, chat = c.bootstrap() + que, _ = c.attachment(alice, chat) + check(que["variants"] == [] and que["error_code"] is None, "queued upload disclosed a result or error") + ready = until_status(c, bob, que, "ready", timeout) + verify_ready(c, bob, que, ready, decoder=decoder) + c.call("GET", attachment_path(que), 404, actor=eve) + c.call("GET", attachment_path(que) + "/content?variant=crop", 404, actor=eve) + + +class Pipeline: + def __init__(self, stand, decoder): + self.stand, self.c, self.a = stand, stand.c, stand.a + self.decoder = decoder + + def action(self, command, *args): + if not self.a.pipeline_hook: + raise Pending("awaiting worker, broker, relay and app-specific fault adapter") + path = Path(self.a.pipeline_hook).resolve() + check(path.is_file() and os.access(path, os.X_OK), "pipeline hook is not executable") + env = dict(os.environ, BASE_URL=self.c.base) + if self.a.project: + env["COMPOSE_PROJECT_NAME"] = self.a.project + result = self.stand.command("pipeline " + command + (" " + args[0] if command in + ("arm", "await", "clear", "crash") else ""), + [str(path), command, *args], env=env, timeout=self.a.pipeline_timeout, + allow_failure=True) + if result.returncode == 78: + raise Pending(f"pipeline adapter prerequisite unavailable for {command}") + if result.returncode and path == Path(__file__).with_name("pipeline_action.py"): + # The bundled adapter emits bounded, whitelisted action labels. + self.c.evidence.event("adapter_diagnostic", action=command, note=result.stderr.decode()[:300].strip()) + check(result.returncode == 0, f"pipeline adapter rejected {command} (exit {result.returncode})") + if command in ("inspect", "await", "crash", "broker-restart", "restart-apis", + "replay", "invalid-job", "worker-limits", "drain"): + data = json.loads(result.stdout) + check(isinstance(data, dict), "pipeline hook did not return an object") + if command in ("broker-restart", "restart-apis", "invalid-job", "replay", "worker-limits", "drain"): + self.c.evidence.event(command, evidence=data) + return data + return None + + def snapshot(self, att): + o = self.action("inspect", att["id"]) + fields = ("job_id", "jobs", "outbox", "staging_exists", "variant_objects", "failed_registered", + "attempts", "max_attempts", "lease_seconds", "prefetch", "concurrency", + "job_timeout_seconds", "worker_memory_bytes", "retry_base_seconds", + "published", "generation", "result_count", "original_exists", "audit_replays", "envelope_valid", + "scheduled_delays_ms") + check(set(fields) <= o.keys(), "incomplete durable pipeline observation") + check(isinstance(o["job_id"], str), "observation has no job ID") + uuid.UUID(o["job_id"]) + attempts = [{k: row[k] for k in ("attempt", "kind", "delay_seconds", "error_code")} for row in o["attempts"] + if isinstance(row, dict) and {"attempt", "kind", "delay_seconds", "error_code"} <= row.keys()] + check(len(attempts) == len(o["attempts"]), "malformed attempt history") + rows = {k: o[k] for k in fields} + rows["attempts"] = attempts + self.c.evidence.event("pipeline_state", attachment=att["id"], **rows) + check(o["jobs"] == 1 and o["outbox"] >= 1 and o["envelope_valid"] is True, + "lost, duplicated or invalid durable processing intent") + check(all(isinstance(o[k], (int, float)) and o[k] > 0 for k in + ("max_attempts", "lease_seconds", "prefetch", "concurrency", + "job_timeout_seconds", "worker_memory_bytes", "retry_base_seconds")), + "unbounded worker/process configuration") + check(o["job_timeout_seconds"] < o["lease_seconds"] and o["prefetch"] >= o["concurrency"], + "worker lease/prefetch do not cover work admitted by worker") + return o + + def accepted(self, uploader, reader, chat): + att, _ = self.c.attachment(uploader, chat) + check(metadata(self.c, reader, att)["status"] in STAGING, "processed while worker must be held") + fail = self.c.call("GET", attachment_path(att) + "/content", 409, actor=reader) + check(fail["json"]["error"]["code"] == "conflict", "non-ready content must be a conflict") + before = self.snapshot(att) + check(before["staging_exists"] is True, "202 returned without durable staging") + return att, before + + def finished(self, reader, att, previous): + ready = until_status(self.c, reader, att, "ready", self.a.pipeline_timeout) + verify_ready(self.c, reader, att, ready, decoder=self.decoder) + + def cleaned(): + row = self.snapshot(att) + check(row["job_id"] == previous["job_id"], "job identity changed during recovery") + if row["staging_exists"] or row["result_count"] != len(VARIANTS) + 1: + return None + check(row["original_exists"] is True and sorted(row["variant_objects"]) == sorted(VARIANTS), + "S3 must contain the original and one published object per required variant") + return row + + current = wait(cleaned, self.a.pipeline_timeout, "staging or orphan results were not cleaned") + return ready, current + + def durable(self): + self.action("reset") + alice, bob, _, chat = self.c.bootstrap() + try: + self.action("worker-stop") + att, previous = self.accepted(alice, bob, chat) + msg = post(self.c, alice, chat, key(), "", body={"text": "", "attachment_ids": [att["id"]]})["json"] + wait(lambda: self.snapshot(att)["published"] > 0, 30, "original job was not published") + self.action("restart-apis") + self.action("broker-restart") + wait(lambda: self.c.call("GET", "/health/ready", schema="Health"), 60, "API not ready after restart") + check(metadata(self.c, bob, att)["status"] == "queued", "job did not stay queued without worker") + assert_history(self.c, bob, chat, [msg]) + self.action("worker-start") + ready, prior = self.finished(bob, att, previous) + self.action("relay-start") + self.action("redeliver", att["id"]) + after, row = self.finished(bob, att, previous) + check(after == ready and row["attempts"] == prior["attempts"], + "duplicate broker event changed published effect or executed again") + finally: + self.action("reset") + + def crash(self, point): + self.action("reset") + alice, bob, _, chat = self.c.bootstrap() + try: + if point == "publish-before-mark": + self.action("worker-stop") + self.action("arm", point) + att, _ = self.c.attachment(alice, chat) + mark = self.action("crash", point, att["id"]) + check(mark.get("point") == point and mark.get("attachment_id") == att["id"] + and mark.get("signal") == "SIGKILL", "fault did not identify and kill the expected process") + self.c.evidence.event("fault_reached", evidence=mark) + if point == "publish-before-mark": + self.action("worker-start") + _, current = self.finished(bob, att, mark) + self.action("drain") + if point == "object-before-ack": + check([row["kind"] for row in current["attempts"]] == ["lease_expired", "success"] + and current["generation"] > mark["generation"], "lost processing lease was not recovered") + if point == "ready-before-ack": + check(current["generation"] == mark["generation"] and current["attempts"] == mark["attempts"], + "redelivery started a new attempt after durable completion") + finally: + self.action("reset") + + def bounded_processing(self): + self.action("reset") + alice, bob, _, chat = self.c.bootstrap() + settings = wait(lambda: settings_with_consumer(self), 20, "no active worker consumer observed") + limits = settings["image_limits"] + check(all(value and value not in ("0", "unlimited", "0 seconds") for value in limits.values()) + and settings["worker_memory_bytes"] > 0, "worker or ImageMagick has unbounded resources") + consumers = settings["consumers"] + check(len(consumers) == settings["concurrency"] and all( + row["active"] and row["ack_required"] and row["prefetch"] == settings["prefetch"] > 0 + for row in consumers), "broker consumer count, ack or prefetch differs from worker configuration") + try: + self.action("arm", "object-before-ack") + att, _ = self.c.attachment(alice, chat) + marker = self.action("await", "object-before-ack", att["id"]) + # Leave the worker paused. Its own work deadline must end the + # attempt, record a temporary failure and permit automatic retry. + _, recovered = self.finished(bob, att, {"job_id": marker["job_id"]}) + observed = [(x["kind"], x["error_code"]) for x in recovered["attempts"]] + check(observed == [("temporary", "processing_timeout"), ("success", None)], + "worker did not time out and finish the same job on retry") + finally: + self.action("reset") + + def malformed_job(self): + self.action("reset") + alice, bob, _, chat = self.c.bootstrap() + try: + att, _ = self.c.attachment(alice, chat) + ready = until_status(self.c, bob, att, "ready", self.a.pipeline_timeout) + original, prior = self.finished(bob, att, self.snapshot(att)) + self.action("invalid-job", att["id"]) + later, after = self.finished(bob, att, prior) + check(ready == original == later and prior["attempts"] == after["attempts"], + "invalid broker message changed a ready result or created a new delivery effect") + finally: + self.action("reset") + + def retry_replay(self): + self.action("reset") + alice, bob, _, chat = self.c.bootstrap() + schedules = [] + try: + for point in ("retry-transient", "retry-transient", "fail-permanent"): + self.action("arm", point) + att, _ = self.c.attachment(alice, chat) + linked_key = key() + linked_body = {"text": "", "attachment_ids": [att["id"]]} + linked = None + if point == "fail-permanent": + linked = post(self.c, alice, chat, linked_key, "", body=linked_body)["json"] + self.action("activate", point, att["id"]) + failed = until_status(self.c, bob, att, "failed", self.a.pipeline_timeout) + code = "invalid_image" if point == "fail-permanent" else "retry_exhausted" + check(failed["error_code"] == code and not failed["variants"], "wrong terminal error code/metadata") + row = self.snapshot(att) + self.action("failed", att["id"]) + check(row["failed_registered"] and row["staging_exists"], "failed job not registered or replay input lost") + unavailable = self.c.call("GET", attachment_path(att) + "/content", 409, actor=bob) + check(unavailable["json"]["error"]["code"] == "conflict", "failed attachment content is not a conflict") + post(self.c, alice, chat, key(), "", body=linked_body, status=409, code="conflict") + if linked: + check(post(self.c, alice, chat, linked_key, "", body=linked_body)["json"] == linked, + "prior accepted Message must remain replayable after its attachment fails") + assert_history(self.c, bob, chat, [linked]) + attempts = row["attempts"] + if point == "fail-permanent": + check(len(attempts) == 1 and attempts[0]["kind"] == "permanent" + and attempts[0]["delay_seconds"] is None, "permanent fault was retried") + else: + check(len(attempts) == row["max_attempts"] and len(attempts) >= 2 + and all(x["kind"] == "temporary" for x in attempts), + "transient failure did not stop at configured attempt bound") + delays = row["scheduled_delays_ms"] + check(len(delays) == len(attempts)-1, "retry offsets were not recorded for each retry") + for i, (ms, attempt) in enumerate(zip(delays, attempts[:-1])): + upper = min(30, row["retry_base_seconds"] * (2 ** min(i, 4))) + observed = attempt["delay_seconds"] + check(1000 <= ms <= 1000 * upper and isinstance(observed, (int, float)) + and 0 < observed <= upper, "retry outside documented backoff and jitter bounds") + schedules.append(delays) + self.action("clear", point, att["id"]) + self.action("replay", att["id"]) + # A checksum fault changed metadata only within this test; replay + # must serve the original fixture's data and checksum. + ready, current = self.finished(bob, att, row) + check(current["audit_replays"] == row["audit_replays"] + 1, + "operator replay did not leave exactly one audit record") + self.action("replay", att["id"]) + snapshot = self.snapshot(att) + check(metadata(self.c, bob, att) == ready and snapshot["attempts"] == current["attempts"] + and snapshot["outbox"] == current["outbox"], "replay of ready job was not a no-op") + check(len(schedules) == 2 and schedules[0] != schedules[1], "identical retry schedules: jitter not observed") + finally: + self.action("reset") + + +def settings_with_consumer(pipeline): + settings = pipeline.action("worker-limits") + return settings if settings["consumers"] else None + + +def make_decoder(stand, executable): + if not executable: + return None + + def decode(data, mime, w, h): + # Bytes are synthetic and the command sees only a temporary file. + suffix = ".jpg" if mime == "image/jpeg" else ".webp" + with tempfile.NamedTemporaryFile(suffix=suffix) as file: + file.write(data) + file.flush() + result = stand.command("decode downloaded " + mime, [ + executable, "identify", "+ping", "-regard-warnings", "-format", "%wx%h", file.name, + ], timeout=10) + check(result.stdout.decode().strip() == f"{w}x{h}", "full image decode has wrong dimensions") + return decode diff --git a/student-tests/lab5/pipeline_action.py b/student-tests/lab5/pipeline_action.py new file mode 100755 index 00000000..a510ccb7 --- /dev/null +++ b/student-tests/lab5/pipeline_action.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""Fault adapter for the Lab 5 Compose implementation, only on an isolated project.""" +import json +import os +import re +from pathlib import Path +import subprocess +import sys +import time +import uuid + +sys.dont_write_bytecode = True +from compose_action import API, DB, PROJECT, ROOT + +HERE = Path(__file__).resolve().parent +BROKER = os.environ.get("LAB5_TEST_BROKER", "rabbitmq") +WORKER = os.environ.get("LAB5_TEST_WORKER", "worker") +RELAY = os.environ.get("LAB5_TEST_RELAY", "relay") +CLEANER = os.environ.get("LAB5_TEST_CLEANUP", "attachment-cleanup") +POINTS = {"commit-before-publish": ("before_publish", RELAY), + "publish-before-mark": ("after_publish", RELAY), + "object-before-ack": ("after_objects", WORKER), + "ready-before-ack": ("before_ack", WORKER)} +ORIGINAL = (ROOT / "tests/fixtures/sample.png").read_bytes() +S3_SETUP = """set -eu +rc=/usr/bin/rc +"$rc" alias set app http://s3:9000 "$(cat /run/secrets/s3_access_key_id)" \ + "$(cat /run/secrets/s3_secret_access_key)" --bucket-lookup path \ + --region "$S3_REGION" >/dev/null +""" +S3_RC = S3_SETUP + """for prefix do + "$rc" bucket list --recursive --json "app/$S3_BUCKET/$prefix" +done +""" + + +class Unsupported(Exception): + pass + + +def require(ok, message): + if not ok: + raise RuntimeError(message) + + +def run(argv, label, timeout=30, allow_fail=False): + try: + r = subprocess.run(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + except subprocess.TimeoutExpired: + raise RuntimeError(f"{label}: exceeded {timeout}s") from None + require(allow_fail or r.returncode == 0, f"{label}: exit {r.returncode}") + return r + + +def until(fn, label, seconds=24): + end = time.monotonic() + seconds + while time.monotonic() < end: + try: + value = fn() + except (RuntimeError, ValueError): + value = None + if value: + return value + time.sleep(0.2) + raise RuntimeError(label) + + +def compose(*args, label=None, timeout=30, allow_fail=False): + files = json.loads(os.environ.get("LAB5_TEST_COMPOSE_FILES", "[]")) + cmd = ["docker", "compose", "--project-name", PROJECT] + for file in files: + cmd += ["-f", file] + return run(cmd + list(args), label or "compose " + args[0], timeout, allow_fail) + + +def sql(query): + return compose("exec", "-T", DB, "psql", "-X", "-v", "ON_ERROR_STOP=1", "-At", + "-U", os.environ["LAB5_TEST_DB_USER"], "-d", os.environ["LAB5_TEST_DB_NAME"], + "-c", query, label="fixture SQL").stdout.decode().strip() + + +def one_container(service): + rows = compose("ps", "-aq", service).stdout.decode().split() + require(len(rows) == 1, f"expected one {service} in isolated project") + return rows[0] + + +def start(*services): + # Start the existing containers only. Compose start can also rerun a finished + # one-shot dependency, which would invalidate the migration experiment. + ids = [one_container(service) for service in services] + run(["docker", "start", *ids], "start selected containers") + + +def set_storage_net(attached): + cid = one_container(WORKER) + net = PROJECT + "_lab5_object" + current = json.loads(run(["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", cid], + "worker networks").stdout) + if attached and net not in current: + run(["docker", "network", "connect", "--alias", WORKER, net, cid], "restore worker S3 network") + if not attached and net in current: + run(["docker", "network", "disconnect", net, cid], "isolate worker S3 network") + + +def marker_dir(): + path = Path(os.environ.get("LAB5_TEST_FAULT_HOST_DIR", "/nonexistent")).resolve() + if not path.is_dir() or not (path / ".lab5-test-project").is_file(): + raise Unsupported("set a dedicated fault directory with .lab5-test-project marker") + require((path / ".lab5-test-project").read_text().strip() == PROJECT, "fault directory belongs to another project") + return path + + +def clean_point(point): + for suffix in ("arm", "used", "hit", "release"): + p = marker_dir() / f"{point}.{suffix}" + if p.exists(): + p.unlink() + + +def broker_ctl(command, *columns): + return compose("exec", "-T", "-e", "RABBITMQ_CTL_ERL_ARGS=+S 1:1 +A 1", BROKER, + "rabbitmqctl", command, "-p", "messenger", "--quiet", "--formatter", "json", + *columns, label="RabbitMQ " + command, timeout=20) + + +def queue(name="image.jobs"): + r = broker_ctl("list_queues", "name", "durable", "messages_ready", "messages_unacknowledged", "consumers").stdout + rows = json.loads(r) + image = [row for row in rows if row["name"] == name] + require(len(image) == 1 and image[0]["durable"] is True, "expected durable queue not found") + return image[0] + + +def snapshot(att): + row = sql(f""" +SELECT json_build_object( + 'job_id', j.id, 'jobs', (SELECT count(*) FROM attachment_jobs WHERE attachment_id=a.id), + 'outbox', (SELECT count(*) FROM outbox WHERE job_id=j.id), + 'published', (SELECT count(*) FROM outbox WHERE job_id=j.id AND sent_at IS NOT NULL), + 'status', a.status, 'max_attempts', j.max_attempts, + 'failed_registered', a.status='failed' AND a.error_code IS NOT NULL, + 'generation', j.lease_generation, 'original_key', a.object_key, + 'has_staging_ref', a.staging_key IS NOT NULL, + 'processing_age_seconds', (SELECT extract(epoch FROM clock_timestamp()-r.started_at) + FROM attachment_attempts r WHERE r.job_id=j.id AND r.lease_generation=j.lease_generation), + 'variant_keys', COALESCE((SELECT json_object_agg(name, object_key) FROM attachment_variants + WHERE attachment_id = a.id AND pipeline_version = j.pipeline_version), '{{}}'), + 'audit_replays', (SELECT count(*) FROM attachment_replays WHERE job_id=j.id), + 'envelope_valid', EXISTS (SELECT 1 FROM outbox o WHERE o.job_id=j.id AND + o.payload->>'schema_version'='1' AND o.payload->>'type'='attachment.process.v1' + AND o.payload->>'job_id'=j.id::text AND o.payload->>'attachment_id'=a.id::text + AND o.payload->>'attempt'='1' AND o.payload->>'pipeline_version'=j.pipeline_version::text + AND (o.payload - ARRAY['schema_version','type','job_id','attachment_id','attempt','pipeline_version', + 'created_at','traceparent','tracestate']) = '{{}}'::jsonb), + 'attempts', COALESCE((SELECT json_agg(json_build_object( + 'attempt', r.attempt, + 'kind', CASE r.outcome WHEN 'ready' THEN 'success' WHEN 'retry' THEN 'temporary' + WHEN 'failed' THEN CASE WHEN r.error_code='retry_exhausted' THEN 'temporary' ELSE 'permanent' END + WHEN 'lease_expired' THEN 'lease_expired' ELSE 'processing' END, + 'delay_seconds', extract(epoch FROM o.available_at-r.finished_at), 'error_code', r.error_code + ) ORDER BY r.attempt) FROM attachment_attempts r LEFT JOIN outbox o + ON o.dedup_key = 'retry/' || j.id || '/' || (r.attempt + 1) WHERE r.job_id=j.id), '[]') +) +FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version +WHERE a.id='{att}'; +""") + require(row and "\n" not in row, "expected exactly one pipeline observation") + return json.loads(row) + + +def list_key_sets(*prefixes): + r = compose("run", "--no-deps", "--rm", "-T", "--entrypoint", "/bin/sh", "s3-init", + "-c", S3_RC, "lab5-test", *prefixes, label="list synthetic S3 prefixes", timeout=20) + source = r.stdout.decode().lstrip() + decoder = json.JSONDecoder() + sets = [] + for prefix in prefixes: + page, offset = decoder.raw_decode(source) + source = source[offset:].lstrip() + require(page.get("truncated") is False and isinstance(page.get("items"), list), + "S3 prefix listing is incomplete") + keys = {entry["key"] for entry in page["items"] if not entry["is_dir"]} + require(all(key.startswith(prefix) for key in keys), "S3 response escaped the fixture prefix") + sets.append(keys) + require(not source, "unexpected S3 output after the requested listings") + return sets + + +def runtime(): + cid = one_container(WORKER) + fields = run(["docker", "inspect", "-f", "{{json .Config.Env}}\t{{.HostConfig.Memory}}", cid], + "worker limits").stdout.decode().strip().split("\t") + env, mem = json.loads(fields[0]), int(fields[1]) + settings = dict(x.split("=", 1) for x in env if "=" in x) + + def value(names, default): + return next((int(settings[n]) for n in names if n in settings), default) + + return {"worker_memory_bytes": mem, + "concurrency": value(("WORKER_CONCURRENCY",), 1), + "prefetch": value(("WORKER_PREFETCH", "JOB_PREFETCH"), 1), + "lease_seconds": value(("JOB_LEASE_SECONDS", "WORKER_LEASE_SECONDS"), 90), + "job_timeout_seconds": value(("JOB_TIMEOUT_SECONDS", "WORKER_TIMEOUT_SECONDS"), 60), + "retry_base_seconds": value(("JOB_RETRY_BASE_SECONDS", "WORKER_RETRY_BASE_SECONDS"), 2)} + + +def worker_limits(): + settings = runtime() + r = compose("exec", "-T", WORKER, "magick", "-list", "resource", label="ImageMagick policy") + limits = dict((name.strip(), val.strip()) for line in r.stdout.decode().splitlines() + if ":" in line for name, val in [line.split(":", 1)]) + settings["image_limits"] = {key: limits.get(key, "") for key in ("Memory", "Map", "Disk", "Thread", "Time")} + r = broker_ctl("list_consumers") + rows = json.loads(r.stdout) + settings["consumers"] = [ + {"prefetch": row["prefetch_count"], "ack_required": row["ack_required"], "active": row["active"]} + for row in rows if row["queue_name"] == "image.jobs" + ] + return settings + + +def retry_delays_ms(job_id): + cid = one_container(WORKER) + # Preserve only the observed scheduling values, never container logs. + output = run(["docker", "logs", cid], "worker retry delay evidence") + delays = [] + for line in (output.stderr + output.stdout).decode().splitlines(): + if "job retry scheduled" not in line: + continue + if line.startswith("{"): + item = json.loads(line) + if item.get("job_id") == job_id and item.get("msg") == "job retry scheduled": + delays.append(int(item["delay_ms"])) + elif f"job_id={job_id}" in line: + match = re.search(r"\bdelay_ms=(\d+)", line) + require(match is not None, "worker retry log did not record jitter") + delays.append(int(match.group(1))) + return delays + + +def inspect(att): + row = snapshot(att) + staging, results = list_key_sets(f"staging/{att}/", f"results/{att}/") + keys = row.pop("variant_keys") + original = row.pop("original_key") + row["staging_exists"] = f"staging/{att}/original" in staging + row["variant_objects"] = sorted(name for name in keys if keys[name] in results) + row["original_exists"] = original in results + row["result_count"] = len(results) + row["scheduled_delays_ms"] = retry_delays_ms(row["job_id"]) if any( + attempt["delay_seconds"] is not None for attempt in row["attempts"]) else [] + row.update(runtime()) + return row + + +def restart_broker_with_pending(): + # Stop the relay so it cannot republish a lost message during recovery. + compose("stop", "-t", "4", RELAY) + before = until(lambda: (q if (q := queue())["messages_ready"] > 0 and q["consumers"] == 0 else None), + "expected a durable broker delivery before restart") + cid = one_container(BROKER) + run(["docker", "restart", "--time", "8", cid], "physical RabbitMQ restart", timeout=70) + after = until(queue, "durable queue did not reappear", seconds=45) + require(after["messages_ready"] == before["messages_ready"] and after["consumers"] == 0, + "accepted broker deliveries did not survive restart") + return {"broker_before": before, "broker_after": after} + + +def drained(): + def empty(): + value = queue() + return value if value["consumers"] > 0 and value["messages_ready"] == 0 and value["messages_unacknowledged"] == 0 else None + return until(empty, "broker delivery was not acknowledged/drained", seconds=45) + + + +def await_publication(outbox_id): + until(lambda: sql(f"SELECT sent_at IS NOT NULL FROM outbox WHERE id='{outbox_id}'") == "t", + "fixture delivery not confirmed") + return drained() + + +def publish_fixture(att, field=None, *, version=None): + dedup = "test_delivery/" + str(uuid.uuid4()) + payload = "o.payload" + if field in ("attachment_id", "job_id"): + payload = f"jsonb_set(o.payload, '{{{field}}}', to_jsonb('{uuid.uuid4()}'::text))" + elif field == "pipeline_version": + payload = "jsonb_set(o.payload, '{pipeline_version}', to_jsonb(j.pipeline_version+1))" + version_filter = "" if version is None else f" AND j.pipeline_version={int(version)}" + row = sql(f"""INSERT INTO outbox(job_id,payload,available_at,dedup_key) + SELECT o.job_id,{payload}, clock_timestamp(), '{dedup}' FROM outbox o + JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='{att}'{version_filter} + ORDER BY o.created_at, o.id LIMIT 1 RETURNING id""").splitlines() + require(len(row) == 2 and row[1] == "INSERT 0 1", "unable to insert fixture delivery") + outbox_id = str(uuid.UUID(row[0])) + await_publication(outbox_id) + return outbox_id + + +def invalid_job(att): + before = queue("image.invalid")["messages_ready"] + for field in ("attachment_id", "job_id", "pipeline_version"): + publish_fixture(att, field) + after = until(lambda: (n if (n := queue("image.invalid")["messages_ready"]) == before+3 else None), + "invalid delivery was not rejected to DLQ") + return {"dead_letters_before": before, "dead_letters_after": after} + + +def failed(att): + row = snapshot(att) + result = compose("exec", "-T", CLEANER, "/app/jobs", "failed", label="list failed via operator CLI") + values = json.loads(result.stdout) + require(isinstance(values, list) and any(r["job_id"] == row["job_id"] and r["attachment_id"] == att + for r in values), + "failed job absent from operator registry") + + +def replay(att): + default = [CLEANER, "/app/jobs", "replay", "-job-id", "{job_id}", "-request-id", "{request_id}", + "-operator", "test", "-reason", "acceptance", "-extra-attempts", "1"] + # JSON argv with explicitly recognized substitutions; shell is never used. + cmd = json.loads(os.environ["LAB5_TEST_REPLAY_CMD"]) if os.environ.get("LAB5_TEST_REPLAY_CMD") else default + row = snapshot(att) + mapping = {"{job_id}": row["job_id"], "{attachment_id}": att, "{request_id}": str(uuid.uuid4())} + argv = [mapping.get(x, x) for x in cmd] + result = compose("exec", "-T", *argv, label="operator replay") + data = json.loads(result.stdout) + require(data["job_id"] == row["job_id"] and data["replayed"] is (row["status"] == "failed"), + "operator replay did not report the expected status") + return {"accepted": data["replayed"], "request_id": mapping["{request_id}"], "job_id": row["job_id"]} + + +def wait_marker(label, att): + point, _ = POINTS[label] + job = snapshot(att)["job_id"] + hit = marker_dir() / (point + ".hit") + matched = until(lambda: hit.read_text().strip() if hit.is_file() else None, + "instrumented process did not reach intended fault") + require(matched == job, "fault marker matched a different job") + return hit, {"point": label, "attachment_id": att, "job_id": matched} + + +def crash_observed(label, att): + point, service = POINTS[label] + cid = one_container(service) + hit, evidence = wait_marker(label, att) + row = snapshot(att) + expected = {"commit-before-publish": "queued", "publish-before-mark": "queued", + "object-before-ack": "processing", "ready-before-ack": "ready"}[label] + require(row["status"] == expected, "attachment state does not match held crash boundary") + if expected == "queued": + require(row["published"] == 0 and row["outbox"] == 1, "outbox is not committed and unsent") + if expected != "ready": + require(row["has_staging_ref"], "staging reference removed before safe completion") + if expected == "processing": + age_limit = runtime()["job_timeout_seconds"] - row["processing_age_seconds"] + else: + age_limit = None + marker_mtime = hit.stat().st_mtime + started = time.monotonic() + # Read state and signal consecutively, within the actual fault window. + # The relay timeout starts slightly before the marker (transaction setup). + limit = 6 if expected == "queued" else 25 + if expected == "processing": + require(age_limit > 0.3, "worker deadline expired before the crash") + else: + require(time.time() - marker_mtime < limit, "fault marker expired before the crash") + require(not (marker_dir() / (point + ".release")).exists(), "fault point was released") + run(["docker", "kill", "--signal", "KILL", cid], "kill at held fault marker", timeout=5) + elapsed = time.monotonic() - started + if expected == "processing": + require(elapsed < age_limit, "worker deadline elapsed before SIGKILL") + else: + require(time.time() - marker_mtime < limit, "fault window elapsed before SIGKILL") + stored = None + try: + if expected == "processing": + stored = len(list_key_sets(f"results/{att}/")[0]) + require(stored >= 4, "S3 did not contain the written output set at worker crash") + finally: + clean_point(point) + start(service) + evidence.update(container=cid, signal="SIGKILL", status=expected, generation=row["generation"], + attempts=row["attempts"], outbox=row["outbox"], published=row["published"], + result_count=stored, kill_elapsed_s=elapsed) + return evidence + + +def main(): + require(bool(PROJECT) and PROJECT != "highload-lab5", "explicit disposable project required") + marker_dir() + command, *args = sys.argv[1:] + if command == "reset": + services = set(compose("config", "--services").stdout.decode().split()) + if not {WORKER, RELAY, BROKER} <= services: + raise Unsupported("worker, relay and broker are not configured") + for point, _ in POINTS.values(): + hit = marker_dir() / (point + ".hit") + if hit.exists(): + (marker_dir() / (point + ".release")).touch() + (marker_dir() / (point + ".arm")).unlink(missing_ok=True) + start(BROKER, RELAY, WORKER) + set_storage_net(True) + return None + if command == "worker-limits": + return worker_limits() + if command == "drain": + return drained() + if command == "worker-stop": + compose("stop", "-t", "5", WORKER) + elif command == "worker-start": + start(WORKER) + elif command == "relay-start": + start(RELAY) + elif command == "restart-apis": + old = compose("ps", "-q", API).stdout.decode().split() + require(len(old) >= 2, "need two real API containers to restart") + def starts(): + return {i: run(["docker", "inspect", "-f", "{{.State.StartedAt}}", i], + "API StartedAt").stdout.decode().strip() for i in old} + before = starts() + run(["docker", "restart", "--time", "8", *old], "restart APIs") + after = starts() + require(all(after[i] != before[i] for i in old), "not all APIs restarted") + return {"started_before": before, "started_after": after} + elif command == "broker-restart": + return restart_broker_with_pending() + elif command == "arm" and args[0] in POINTS: + point, _ = POINTS[args[0]] + clean_point(point) + (marker_dir() / (point + ".arm")).touch(exist_ok=False) + elif command == "await": + return wait_marker(args[0], str(uuid.UUID(args[1])))[1] + elif command == "crash": + return crash_observed(args[0], str(uuid.UUID(args[1]))) + elif command == "arm" and args[0] == "retry-transient": + set_storage_net(False) + elif command == "clear" and args[0] == "retry-transient": + set_storage_net(True) + elif command == "arm" and args[0] == "fail-permanent": + compose("stop", "-t", "5", WORKER) + elif command in ("activate", "clear") and args[0] == "fail-permanent": + att = str(uuid.UUID(args[1])) + if command == "activate": + value = "0" * 64 + else: + import hashlib + value = hashlib.sha256(ORIGINAL).hexdigest() + require(sql(f"UPDATE attachments SET sha256='{value}' WHERE id='{att}' RETURNING id").splitlines() == + [att, "UPDATE 1"], + "could not inject/restore staging checksum fault") + if command == "activate": + start(WORKER) + elif command == "activate" and args[0] == "retry-transient": + pass + elif command in ("inspect", "redeliver", "replay", "failed", "invalid-job"): + att = str(uuid.UUID(args[0])) + return {"inspect": inspect, "redeliver": publish_fixture, "replay": replay, + "failed": failed, "invalid-job": invalid_job}[command](att) + else: + raise Unsupported("pipeline action unavailable") + return None + + +if __name__ == "__main__": + try: + result = main() + if result is not None: + print(json.dumps(result)) + except Unsupported as exc: + print(str(exc), file=sys.stderr) + sys.exit(78) + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as exc: + print("pipeline adapter:", exc if isinstance(exc, RuntimeError) else type(exc).__name__, file=sys.stderr) + sys.exit(1) diff --git a/student-tests/lab5/stand.py b/student-tests/lab5/stand.py new file mode 100644 index 00000000..84faa6b8 --- /dev/null +++ b/student-tests/lab5/stand.py @@ -0,0 +1,457 @@ +"""Observe and exercise a Compose stand. Business traffic always goes via the LB.""" +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import http.client +import io +import json +from pathlib import Path +import os +import subprocess +import threading +import time +from urllib.parse import urlsplit + +from checks import ROOT, Failure, Pending, assert_history, check, history, message_path, key, post, summarize, wait +from client import shape + + +class Stand: + # Whitelist inspect fields: no environment, mounts, or container log dumps. + FORMAT = "\t".join(("{{.Id}}", "{{.Config.Hostname}}", "{{.Image}}", "{{.State.Status}}", + "{{.State.StartedAt}}", "{{.State.FinishedAt}}", "{{.State.ExitCode}}", + "{{.State.OOMKilled}}", "{{.HostConfig.NanoCpus}}", "{{.HostConfig.CpuQuota}}", + "{{.HostConfig.CpuPeriod}}", "{{.HostConfig.Memory}}", + "{{json .NetworkSettings.Ports}}", "{{json .HostConfig.PortBindings}}", + "{{.HostConfig.NetworkMode}}", "{{.HostConfig.PublishAllPorts}}")) + + def __init__(self, args, c): + self.a = args + self.c = c + self.e = c.evidence + self.compose = ["docker", "compose"] + if args.project: + self.compose += ["--project-name", args.project] + for file in args.compose_file: + self.compose += ["--file", str(Path(file).resolve())] + + def command(self, name, argv, *, timeout=30, env=None, allow_failure=False): + start = time.monotonic() + try: + result = subprocess.run(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout) + except (OSError, subprocess.TimeoutExpired) as exc: + self.e.event("command", action=name, elapsed_s=time.monotonic()-start, error=type(exc).__name__) + raise Failure(f"{name}: {type(exc).__name__}") from None + self.e.event("command", action=name, elapsed_s=time.monotonic()-start, exit_code=result.returncode) + check(allow_failure or result.returncode == 0, f"{name}: exit {result.returncode}") + return result + + def ids(self, service, all_states=True): + return self.command("compose ps " + service, self.compose + ["ps"] + + (["-a"] if all_states else []) + ["-q", service]).stdout.decode().split() + + def containers(self, service): + results = [] + for cid in self.ids(service): + result = self.command("inspect " + service, ["docker", "inspect", "--format", self.FORMAT, cid]) + parts = result.stdout.decode().rstrip("\n").split("\t") + fields = ("id", "alias", "image", "state", "started_at", "finished_at", "exit_code", "oom", + "nano_cpus", "cpu_quota", "cpu_period", "memory", "ports", "bindings", "network", "publish_all") + check(len(parts) == len(fields), "unexpected docker inspect format") + for index in range(6, len(fields)): + parts[index] = json.loads(parts[index]) if fields[index] != "network" else parts[index] + results.append(dict(zip(fields, parts))) + return results + + def public_topology(self): + try: + if not hasattr(self, "engine"): + fmt = "{{.ServerVersion}}\t{{.OSType}}\t{{.Architecture}}\t{{.NCPU}}\t{{.MemTotal}}" + result = self.command("Docker availability", ["docker", "info", "--format", fmt]) + self.engine = result.stdout.decode().strip().split("\t") + check(len(self.engine) == 5, "unexpected Docker engine metadata") + self.e.event("docker_environment", version=self.engine[0], os=self.engine[1], + architecture=self.engine[2], logical_cpus=int(self.engine[3]), + memory_bytes=int(self.engine[4])) + except Failure: + raise Pending("Docker daemon not available") from None + services = self.command("Compose service names", self.compose + ["config", "--services"]).stdout.decode().split() + if self.a.api not in services or self.a.lb not in services: + raise Pending("Compose does not define API and LB services") + api = [r for r in self.containers(self.a.api) if r["state"] == "running"] + lb = [r for r in self.containers(self.a.lb) if r["state"] == "running"] + self.e.event("topology", api=api, lb=lb) + if len(api) < 2 or not lb: + raise Pending("stand needs two running API replicas and a public LB") + self.a_ids = {r["alias"] for r in api} + check(len(self.a_ids) == len(api), "API aliases are not unique") + self.lb = lb[0] + port = urlsplit(self.c.base).port or (443 if self.c.base.startswith("https:") else 80) + check(any(int(binding["HostPort"]) == port for r in lb + for group in (r["ports"] or {}).values() for binding in group or []), + "BASE_URL port is not published by the selected LB") + return api + + def resources(self): + api = self.public_topology() + for r in api: + check(r["nano_cpus"] > 0 or (r["cpu_quota"] > 0 and r["cpu_period"] > 0), + "API CPU has no hard limit") + check(r["memory"] > 0, "API RAM has no hard limit") + check(not any((r["ports"] or {}).values()) and not any((r["bindings"] or {}).values()) + and not r["publish_all"] and r["network"] != "host", "API bypasses LB via host ports/network") + self.stats() + + def stats(self): + ids = self.ids(self.a.api, False) + [self.lb["id"]] + result = self.command("docker stats", ["docker", "stats", "--no-stream", "--format", + "{{.ID}}\t{{.CPUPerc}}\t{{.MemUsage}}", *ids], timeout=10) + self.e.event("resource_sample", samples=result.stdout.decode().splitlines()) + + def observe(self, actor, wanted=None, *, duration=3, forbidden=None): + wanted = self.a_ids if wanted is None else wanted + deadline, seen, attempts = time.monotonic() + duration, set(), 0 + while time.monotonic() < deadline: + result = self.c.call("GET", "/api/v1/users/me", actor=actor, schema="User") + check(result["json"] == actor["user"], "same Bearer token resolved to another user") + alias = result["headers"]["X-Instance-Id"] + check(alias != forbidden, "LB routed authenticated request to the excluded replica") + check(alias in self.a_ids, "LB routed to an API outside this Compose project") + seen.add(alias) + attempts += 1 + if forbidden is not None: + time.sleep(0.05) + if forbidden is None and wanted <= seen: + break + self.e.event("routing", wanted=sorted(wanted), observed=sorted(seen), excluded=forbidden, attempts=attempts) + check(wanted <= seen and attempts >= (10 if forbidden else 2), "LB did not exercise the required replicas") + return seen + + def network_target(self, target): + fmt = "{{json .NetworkSettings.Networks}}" + def networks(cid): + r = self.command("container networks", ["docker", "inspect", "--format", fmt, cid]) + return json.loads(r.stdout) + net = networks(target["id"]) + shared = net.keys() & networks(self.lb["id"]).keys() + chosen = self.a.network or (sorted(shared)[0] if shared else None) + check(chosen in shared and net[chosen]["IPAddress"], "no shared API/LB network with IPv4") + return net[chosen]["IPAddress"] + + def direct_health(self, target, endpoint, status): + # From the LB network namespace, bypass routing only for health diagnostics. + ip = self.network_target(target) + result = self.command("direct health/" + endpoint, [ + "docker", "run", "--rm", "--pull", "never", "--network", "container:" + self.lb["id"], + "--entrypoint", "curl", self.a.probe_image, "--http1.1", "--noproxy", "*", + "--include", "--silent", "--show-error", "--max-time", "4", + f"http://{ip}:{self.a.api_port}/health/{endpoint}", + ], timeout=12) + response = http.client.HTTPResponse(type("Socket", (), {"makefile": lambda *_: io.BytesIO(result.stdout)})()) + with response: + response.begin() + self.e.event("direct_health", endpoint=endpoint, status=response.status, + instance=response.getheader("X-Instance-Id")) + check(response.status == status and response.getheader("X-Instance-Id") == target["alias"], + f"target /health/{endpoint}: expected {status}, got {response.status}") + value = json.loads(response.read()) + rid = response.getheader("X-Request-Id") + check(bool(rid) and response.headers.get_content_type() == "application/json", + "direct health response is missing required headers") + if status == 200: + check(shape(value, "Health") == {"status": "ok"}, "direct health payload differs") + else: + check(shape(value, "Error")["error"]["code"] == "unavailable" + and value["error"]["request_id"] == rid, "readiness 503 error envelope differs") + return True + + def hook(self, executable, target=None, timeout=90, **extra): + if not executable: + raise Pending("operator hook is not configured") + path = Path(executable).resolve() + check(path.is_file() and os.access(path, os.X_OK), "operator hook must be an executable file") + env = dict(os.environ, BASE_URL=self.c.base, **extra) + if self.a.project: + env["COMPOSE_PROJECT_NAME"] = self.a.project + if target: + env.update(TARGET_INSTANCE_ID=target["alias"], TARGET_CONTAINER_ID=target["id"]) + self.command("hook " + path.name, [str(path)], env=env, timeout=timeout) + + def unready(self): + api = self.public_topology() + if not (self.a.unready_hook and self.a.ready_hook): + raise Pending("configure hooks to fault and restore one API dependency without disconnecting LB") + alice = self.c.user("readiness") + self.observe(alice) + target = api[0] + self.direct_health(target, "live", 200) + self.direct_health(target, "ready", 200) + try: + self.hook(self.a.unready_hook, target, LAB5_HOOK_MODE="unready") + wait(lambda: self.direct_health(target, "ready", 503), 20, "target never returned readiness 503") + self.direct_health(target, "live", 200) + # Allow LB convergence, then require sustained exclusion while the + # target remains reachable from the LB network and answers 503. + others = self.a_ids - {target["alias"]} + def routed_elsewhere(): + response = self.c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + return response["headers"]["X-Instance-Id"] in others + wait(routed_elsewhere, 10, "no authenticated request routed to a ready replica") + wait(lambda: self.observe(alice, others, duration=3, forbidden=target["alias"]), 15, + "LB did not consistently exclude a running 503 replica") + self.direct_health(target, "live", 200) + self.direct_health(target, "ready", 503) + finally: + self.hook(self.a.ready_hook, target, LAB5_HOOK_MODE="ready") + wait(lambda: self.direct_health(target, "ready", 200), 20, "target stayed unready after restore") + wait(lambda: self.observe(alice), 20, "restored API did not rejoin LB") + + def ready_aliases(self, actor): + api = self.public_topology() + self.observe(actor) + return api + + def migration_snapshot(self): + service = self.a.migrate + rows = self.containers(service) + check(len(rows) == 1, "expected one migrator container") + row = rows[0] + check(row["state"] == "exited" and row["exit_code"] == 0, "migrator did not exit cleanly") + log = self.command("migrator logs", ["docker", "logs", row["id"]]) + completions = [] + for line in (log.stdout + log.stderr).splitlines(): + try: + item = json.loads(line) + except (ValueError, UnicodeError): + continue + if isinstance(item, dict) and item.get("msg") == "migrations completed": + completions.append(item.get("applied")) + check(len(completions) == 1, "migrator must log exactly one successful invocation") + sql = ("SELECT version_id, is_applied, count(*) FROM goose_db_version " + "GROUP BY 1,2 ORDER BY 1,2;") + output = self.command("migration version counts", self.compose + [ + "exec", "-T", self.a.db, "psql", "-X", "-v", "ON_ERROR_STOP=1", "-At", + "-U", self.a.db_user, "-d", self.a.db_name, "-c", sql, + ]).stdout.decode().splitlines() + versions = sorted(int(p.name.split("_", 1)[0]) for p in (ROOT / "db/migrations").glob("*.sql")) + check(output == [f"{v}|t|1" for v in [0, *versions]], "migration history differs from one apply per version") + check(all(r["started_at"] >= row["finished_at"] for r in self.containers(self.a.api) + if r["state"] == "running"), "API started before migrations completed") + self.e.event("migrations", container=row, applied=completions, versions=output) + return row, output + + def persistence(self): + api = self.public_topology() + if not self.a.restart_hook: + raise Pending("configure a hook that restarts API and stores without removing volumes") + stores = [self.a.db, *self.a.store] + before = {s: self.containers(s) for s in stores} + check(all(rows and all(r["state"] == "running" for r in rows) for rows in before.values()), + "configured persistence stores must be running") + alice, bob, _, chat = self.c.bootstrap() + self.observe(alice) + k, text = key(), "committed before restarting API and stores" + original = post(self.c, alice, chat, k, text)["json"] + self.hook(self.a.restart_hook) + wait(lambda: self.c.call("GET", "/health/ready", schema="Health"), 90, "LB did not recover after restart") + after = self.public_topology() + for s, previous, current in [(self.a.api, api, after)] + [ + (s, before[s], self.containers(s)) for s in stores]: + check(len(previous) == len(current) and all(r["state"] == "running" for r in current) + and not {r["started_at"] for r in previous} & {r["started_at"] for r in current}, + f"restart did not change StartedAt for every {s}") + self.e.event("restart_observed", service=s, before=previous, after=current) + self.observe(alice) + check(post(self.c, alice, chat, k, text)["json"] == original, "idempotency lost after restart") + assert_history(self.c, bob, chat, [original]) + + def verify_stopped(self, target): + now = {r["id"]: r for r in self.containers(self.a.api)} + row = now.get(target["id"]) + check(row is not None and row["state"] == "exited", "SIGTERM did not stop the selected process") + check(row["exit_code"] == 0 and not row["oom"], "API did not exit gracefully on SIGTERM") + self.e.event("stopped", container=row) + + def failover(self): + api = self.public_topology() + alice, bob, _, chat = self.c.bootstrap() + self.observe(alice) + with traffic(self.c, alice, bob, self.a.workers) as load: + self.sample_under_load(load) + for target in api: + self.e.event("target", action="SIGTERM", alias=target["alias"], container=target["id"]) + k, text, sent = wait(lambda: write_from(self.c, alice, chat, target["alias"]), 10, + "could not confirm original POST on selected replica") + others = self.a_ids - {target["alias"]} + endpoint = self.network_target(target) + started = time.monotonic() + first_record = len(load.records) + # Race real client traffic and recovery probes with Docker stop. + with ThreadPoolExecutor(max_workers=1) as pool: + stop = pool.submit(self.command, "SIGTERM " + target["alias"], [ + "docker", "stop", "--signal", "SIGTERM", "--time", "20", target["id"], + ]) + try: + recovered = self.recover(alice, bob, chat, target, stop, started) + self.e.event("recovery", target=target["alias"], seconds=recovered) + if recovered is not None: + print(f" {target['alias']}: read/write restored in {recovered:.3f} s", flush=True) + check(recovered is not None and recovered <= 10, "read and write recovery exceeded 10 seconds") + stop.result() + self.verify_stopped(target) + probe = self.command("fresh TCP to stopped API", ["docker", "exec", self.lb["id"], + "nc", "-w", "2", "-z", endpoint, str(self.a.api_port)], + allow_failure=True, timeout=5) + check(probe.returncode == 1, "fresh connection to stopped replica was accepted or probe failed") + replay = post(self.c, alice, chat, k, text) + check(replay["json"] == sent["json"] and replay["headers"]["X-Instance-Id"] in others, + "different replica did not replay the original Message") + self.observe(alice, others, duration=self.a.window, forbidden=target["alias"]) + read_back = history(self.c, bob, chat) + check([m for m in read_back if m["text"] == text] == [sent["json"]], + "confirmed pre-stop POST missing, changed or duplicated") + check(len({m["text"] for m in read_back}) == len(read_back), + "an uncertain retry created duplicate effects") + finally: + summarize(self.e, load.records[first_record:], time.monotonic() - started, + window="target " + target["alias"]) + # Settle the stop before starting the same container. + try: + stop.result() + finally: + self.command("restore " + target["alias"], ["docker", "start", target["id"]]) + wait(lambda: self.observe(alice), 25, "restored replica did not rejoin LB") + + def recover(self, alice, bob, chat, target, stop, started): + k, text = key(), "recovery probe " + key() + deadline = started + 10 + while time.monotonic() < deadline: + if stop.done(): + stop.result() + try: + result = post(self.c, alice, chat, k, text) + if result["headers"]["X-Instance-Id"] != target["alias"]: + page = self.c.call("GET", message_path(chat) + "?limit=100", actor=bob, schema="MessagePage") + if (page["headers"]["X-Instance-Id"] != target["alias"] + and result["json"] in page["json"]["items"]): + return time.monotonic() - started + except Failure: + pass # Keep the exact key/body after an uncertain write. + time.sleep(0.1) + return None + + def sample_under_load(self, load): + time.sleep(self.a.warmup) + check(load.successes() >= 1, "no successful business request during load warm-up") + self.stats() + + def rolling(self): + api = self.public_topology() + if not self.a.roll_hook: + raise Pending("awaiting rolling update hook/candidate image") + before = self.migration_snapshot() + alice, bob, _, _ = self.c.bootstrap() + self.observe(alice) + original_ids = {r["id"] for r in api} + previous = {r["id"]: r for r in api} + with traffic(self.c, alice, bob, self.a.workers) as load: + self.sample_under_load(load) + start_index = len(load.records) + for target in api: + step_at = time.monotonic() + self.e.event("target", action="rolling_update", alias=target["alias"], container=target["id"]) + self.hook(self.a.roll_hook, target) + current = {r["id"]: r for r in self.public_topology()} + stable = set(previous) - {target["id"]} + new = set(current) - set(previous) + check(set(current) == stable | new and len(new) == 1, "rolling step did not replace exactly one API") + check(all(current[r] == previous[r] for r in stable), "rolling step also disturbed another API") + replacement = current[new.pop()] + check(replacement["image"] != target["image"], "rolling step did not apply a new immutable image") + self.observe(alice) + time.sleep(self.a.window) + self.e.event("rolling_step", previous=target, replacement=replacement, + seconds=time.monotonic() - step_at) + previous = current + check(not original_ids & set(previous), "some original replicas were not updated") + stats = load.records[start_index:] + check(stats and all(r["ok"] for r in stats), "business request failed during rolling update") + check(self.migration_snapshot() == before, "rolling update restarted migrations or changed version history") + + +def write_from(c, actor, chat, alias): + k, text = key(), "before SIGTERM " + key() + r = post(c, actor, chat, k, text) + return (k, text, r) if r["headers"]["X-Instance-Id"] == alias else None + + +class Traffic: + def __init__(self, c, actor, reader, chat): + self.c, self.actor, self.reader, self.chat = c, actor, reader, chat + self.stop = threading.Event() + self.records = [] + self.confirmed = {} + self.offered = set() + self.lock = threading.Lock() + + def lane(self): + while not self.stop.is_set(): + k = key() + with self.lock: + self.offered.add(k) + for op in ("write", "read"): + started = time.monotonic() + ok = False + status = None + try: + if op == "write": + result = post(self.c, self.actor, self.chat, k, k) + with self.lock: + self.confirmed[k] = result["json"] + else: + result = self.c.call("GET", "/api/v1/users/me", actor=self.actor, schema="User") + check(result["json"] == self.actor["user"], "load token resolved to another user") + ok = True + except Failure: + pass + finally: + status = self.c.local.status + row = self.c.evidence.event("load_request", op=op, status=status, ok=ok, + instance=self.c.local.instance, elapsed_s=time.monotonic()-started) + with self.lock: + self.records.append(row) + if self.stop.is_set(): + break + + def successes(self): + with self.lock: + return sum(r["ok"] for r in self.records) + + def reconcile(self): + found = history(self.c, self.reader, self.chat) + by_text = {m["text"]: m for m in found} + check(len(by_text) == len(found) and set(by_text) <= self.offered, + "load history contains duplicate writes or unknown data") + check(all(by_text.get(k) == v for k, v in self.confirmed.items()), + "confirmed load write missing or changed in history") + self.c.evidence.event("load_history", observed=len(found), confirmed=len(self.confirmed), + offered=len(self.offered)) + + +@contextmanager +def traffic(c, actor, reader, workers): + chat = c.call("POST", "/api/v1/chats", 201, actor=actor, schema="Chat", + body={"title": "Lab 5 load", "member_ids": [actor["id"], reader["id"]]})["json"] + load = Traffic(c, actor, reader, chat) + started = time.monotonic() + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(load.lane) for _ in range(workers)] + try: + yield load + finally: + load.stop.set() + for future in futures: + future.result() + elapsed = time.monotonic() - started + summarize(c.evidence, load.records, elapsed) + load.reconcile() diff --git a/student-tests/lab5/versioned.py b/student-tests/lab5/versioned.py new file mode 100644 index 00000000..d9c4a394 --- /dev/null +++ b/student-tests/lab5/versioned.py @@ -0,0 +1,430 @@ +"""Track 2 experiment. Each trial uses a separate attachment and real deliveries.""" +from collections import defaultdict +from datetime import datetime, timedelta +import hashlib +import json +from statistics import median +import time +import uuid + +from checks import Failure, Pending, check, wait +from pipeline import SAMPLE, VARIANTS, metadata, verify_ready + +ORIGINAL = hashlib.sha256(SAMPLE).hexdigest() + + +def utc(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def jobs(row): + return {j["version"]: j for j in row["jobs"]} + + +def deliveries(row, version): + return [e for e in row["outbox"] if e["version"] == version] + + +def fingerprint(row): + # Dynamic clock and transport bookkeeping are intentionally outside the + # effect. Attempts, pointers, generations and timestamps remain comparable. + return {k: row[k] for k in ("version", "status", "original", "updated_at", "jobs")} + + +class Versioned: + def __init__(self, pipeline, ops): + self.pipeline = pipeline + self.c = pipeline.c + self.e = self.c.evidence + self.ops = ops + self.timeout = pipeline.a.pipeline_timeout + + def state(self, att): + attachment = str(uuid.UUID(att["id"])) + value = self.ops.sql(f""" +SELECT json_build_object( + 'version', a.pipeline_version, 'status', a.status, 'original', a.object_key, + 'sha256', a.sha256, 'size', a.size_bytes, 'updated_at', a.updated_at, 'db_now', clock_timestamp(), + 'jobs', COALESCE((SELECT json_agg(json_build_object( + 'id', j.id, 'version', j.pipeline_version, 'generation', j.lease_generation, + 'attempt_count', j.attempts_started, 'lease_until', j.lease_until, + 'published_original', j.published_original_key, + 'variants', COALESCE((SELECT json_object_agg(v.name, json_build_object( + 'key',v.object_key,'sha256',v.sha256,'size',v.size_bytes,'width',v.width,'height',v.height)) + FROM attachment_variants v WHERE v.attachment_id=a.id AND v.pipeline_version=j.pipeline_version), '{{}}'), + 'attempts', COALESCE((SELECT json_agg(json_build_object( + 'attempt', x.attempt, 'generation', x.lease_generation,'started_at', x.started_at, + 'finished_at', x.finished_at, 'outcome', x.outcome, 'error_code', x.error_code) ORDER BY x.attempt) + FROM attachment_attempts x WHERE x.job_id=j.id), '[]') + ) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id), '[]'), + 'outbox', COALESCE((SELECT json_agg(json_build_object( + 'id', o.id, 'job_id', j.id, 'version', j.pipeline_version, 'available_at', o.available_at, + 'sent_at', o.sent_at, 'envelope', o.payload) ORDER BY o.created_at,o.id) + FROM outbox o JOIN attachment_jobs j ON o.job_id=j.id WHERE j.attachment_id=a.id), '[]') +) FROM attachments a WHERE a.id='{attachment}'; +""") + check(bool(value), "accepted attachment disappeared") + state = json.loads(value) + check(state["sha256"] == ORIGINAL and state["size"] == len(SAMPLE), "original metadata changed") + return state + + def observe(self, name, att, row=None): + row = self.state(att) if row is None else row + self.e.event("versioned_state", stage=name, attachment=att["id"], state=row) + return row + + def ready(self, att, version, job_id, started=None): + started = time.monotonic() if started is None else started + deadline = started + self.timeout + while time.monotonic() < deadline: + row = self.state(att) + check(row["version"] == version and jobs(row).get(version, {}).get("id") == job_id, + "active version or job changed while waiting for ready") + check(row["status"] in ("queued", "processing", "ready"), f"version {version} reached {row['status']}") + if row["status"] == "ready" and time.monotonic() <= deadline: + elapsed = time.monotonic() - started + self.e.event("versioned_ready", version=version, job=job_id, seconds=elapsed) + return elapsed + time.sleep(0.25) + raise Failure(f"version {version} did not reach ready within {self.timeout}s") + + def verify_s3(self, expected): + # Read every named object, including historical published objects that + # are no longer exposed by GET. Store digests and sizes, not the bodies. + shell = self.ops.S3_SETUP + """for key do + "$rc" object show "app/$S3_BUCKET/$key" >/tmp/lab5-object + digest=$(sha256sum /tmp/lab5-object) + size=$(wc -c utc(v2["sent_at"]), + "broker publication order or original envelope does not prove reorder") + check(fingerprint(after) == fingerprint(before) and jobs(after)[2]["attempt_count"] == 1, + "v1 sent after v2 changed jobs or published data") + self.no_dlq(dead) + return after, elapsed + finally: + if oid: + self.release_outbox(oid) + self.ops.start(self.ops.RELAY) + + def release_outbox(self, oid): + self.ops.sql(f"UPDATE outbox SET available_at=clock_timestamp() WHERE id='{oid}' AND sent_at IS NULL") + + def arm(self, point): + self.ops.clean_point(point) + (self.ops.marker_dir() / (point + ".arm")).touch(exist_ok=False) + + def marker(self, point, jid): + path = self.ops.marker_dir() / (point + ".hit") + value = self.ops.until(lambda: path.read_text().strip() if path.is_file() else None, + f"no marker for {point}", seconds=30) + check(value == jid, f"{point} marker was from a different job") + self.e.event("versioned_marker", point=point, job=jid) + + def release(self, point): + (self.ops.marker_dir() / (point + ".release")).touch() + + def crash(self, point): + reader, att, baseline = self.ready_v1() + cid = self.ops.one_container(self.ops.WORKER) + self.arm(point) + try: + jid = self.upgrade(att) + self.marker(point, jid) + row = self.observe("before crash", att) + job = jobs(row)[2] + check(row["status"] == "processing" and job["attempt_count"] == 1 and job["lease_until"], + "marker did not hold an active v2 attempt") + expected = 0 if point == "before_objects" else 4 + prefix = f"results/{att['id']}/v2/{job['generation']}/" + objects = self.ops.list_key_sets(prefix)[0] + check(len(objects) == expected, "S3 side effect does not match the intended crash window") + work_end = utc(job["attempts"][0]["started_at"]) + timedelta( + seconds=self.ops.runtime()["job_timeout_seconds"]) + deadline = min(utc(job["lease_until"]), work_end) + read_started = time.monotonic() + remaining = (deadline - utc(self.state(att)["db_now"])).total_seconds() + check(remaining > 0, "work deadline or lease elapsed before crash") + started = time.monotonic() + self.ops.run(["docker", "kill", "--signal", "KILL", cid], "kill held worker", timeout=8) + check(time.monotonic() - read_started < remaining, "SIGKILL did not land while attempt was held") + self.e.event("versioned_crash", point=point, container=cid, job=jid, + work_budget_remaining_s=remaining, object_keys=sorted(objects)) + finally: + self.release(point) + self.ops.start(self.ops.WORKER) + elapsed = self.ready(att, 2, jid, started) + self.ops.drained() + after = self.verify(reader, att, 2, {1, 2}, "after recovery") + check(jobs(after)[1] == jobs(baseline)[1], "v2 crash changed historical v1") + job = jobs(after)[2] + check(job["attempt_count"] == 2 and + [x["outcome"] for x in job["attempts"]] == ["lease_expired", "ready"] and + job["generation"] > jobs(row)[2]["generation"], "crash did not recover in a new fenced attempt") + return after, elapsed + + def workers(self, count): + self.ops.compose("up", "-d", "--no-deps", "--no-build", "--scale", f"{self.ops.WORKER}={count}", + self.ops.WORKER, label=f"set {count} worker processes") + + def started_at(self, cid): + return self.ops.run(["docker", "inspect", "-f", "{{.State.StartedAt}}", cid], + "old worker StartedAt").stdout.decode().strip() + + def rejection(self, cid, jid, since): + result = self.ops.run(["docker", "logs", "--timestamps", "--since", since, cid], + "old worker finalization log") + for line in (result.stdout + result.stderr).decode().splitlines(): + if "job finalization rejected" in line and jid in line: + return line.split()[0] + return None + + def late_worker(self): + old_cid = self.ops.one_container(self.ops.WORKER) + original_process = self.started_at(old_cid) + budget = self.ops.runtime()["job_timeout_seconds"] + point = "after_objects" + self.arm(point) + scaled = False + try: + reader, att = self.started_v1() + jid = jobs(self.state(att))[1]["id"] + self.marker(point, jid) + hold = self.observe("v1 stopped after its S3 writes", att) + old = jobs(hold)[1] + check(hold["status"] == "processing" and old["attempt_count"] == 1 and old["lease_until"], + "v1 was not held with an active lease") + prefix = f"results/{att['id']}/v1/{old['generation']}/" + keys = self.ops.list_key_sets(prefix)[0] + self.e.event("versioned_old_effect", job=jid, keys=sorted(keys)) + check(len(keys) == 4, "old worker did not write its output set") + scaled = True + self.workers(2) + self.ops.until(lambda: self.ops.queue()["consumers"] == 2, "two workers did not join") + check(self.started_at(old_cid) == original_process, "old worker was replaced") + next_id = self.upgrade(att) + elapsed = self.ready(att, 2, next_id) + before = self.observe("v2 ready while v1 remains held", att) + deadline = min(utc(old["lease_until"]), utc(old["attempts"][0]["started_at"]) + timedelta(seconds=budget)) + dead = self.ops.queue("image.invalid")["messages_ready"] + released = utc(self.ops.sql("SELECT clock_timestamp()")) + check(released < deadline, "old work deadline/lease elapsed before release") + self.release(point) + reject = self.ops.until(lambda: self.rejection(old_cid, jid, old["attempts"][0]["started_at"]), + "resumed old worker did not reject finalization") + self.e.event("versioned_rejected_commit", job=jid, container=old_cid, released_at=released.isoformat(), + rejected_at=reject, deadline=deadline.isoformat()) + check(released <= utc(reject) < deadline and self.started_at(old_cid) == original_process, + "old worker did not reach the version check within its original lease") + self.ops.drained() + after = self.verify(reader, att, 2, {2}, "after old worker rejection") + check(jobs(after)[1]["attempt_count"] == 1 and jobs(after)[2]["attempt_count"] == 1 + and jobs(after)[2] == jobs(before)[2] and after["original"] == before["original"] + and after["updated_at"] == before["updated_at"], "old worker overrode the v2 effect") + self.no_dlq(dead) + return after, elapsed + finally: + self.release(point) + if scaled: + self.workers(1) + + +def series(pipeline, repetitions): + # Imported after acceptance has set the explicit Compose scope. + import pipeline_action as ops + + if not ops.PROJECT or ops.PROJECT == "highload-lab5": + raise Failure("explicit isolated versioned project required") + try: + ops.marker_dir() + except ops.Unsupported as exc: + raise Pending(str(exc)) from None + pipeline.stand.public_topology() + supported = ops.sql("""SELECT EXISTS (SELECT 1 FROM information_schema.columns +WHERE table_schema=current_schema() AND table_name='attachment_jobs' +AND column_name='published_original_key')""") == "t" + if not supported: + raise Pending("versioned pipeline migration is not installed on this stand") + + experiment = Versioned(pipeline, ops) + cases = ( + ("duplicate-v2-and-stale-v1", experiment.duplicate_and_stale), + ("reorder-v2-before-v1", experiment.reorder), + ("crash-before-S3", lambda: experiment.crash("before_objects")), + ("crash-after-S3", lambda: experiment.crash("after_objects")), + ("late-v1-worker-after-v2", experiment.late_worker), + ) + samples = defaultdict(list) + digests = set() + for repetition in range(1, repetitions + 1): + for name, case in cases: + start = time.monotonic() + pipeline.c.evidence.event("versioned_trial_start", scenario=name, repetition=repetition) + try: + pipeline.action("reset") + ops.one_container(ops.WORKER) + ops.drained() + row, ready_s = case() + digest = {n: v["sha256"] for n, v in jobs(row)[2]["variants"].items()} + digests.add(tuple(sorted(digest.items()))) + sample = {"status": "PASS", "ready_s": ready_s, "v2_sha256": digest, + "attempts": {v: j["attempt_count"] for v, j in jobs(row).items()}} + except (Failure, RuntimeError, OSError, ValueError) as exc: + note = str(exc) if isinstance(exc, (Failure, RuntimeError)) else type(exc).__name__ + sample = {"status": "FAIL", "reason": note} + sample["elapsed_s"] = time.monotonic() - start + samples[name].append(sample) + pipeline.c.evidence.event("versioned_trial", scenario=name, repetition=repetition, **sample) + print(f"{sample['status']}: {name}, repeat {repetition}/{repetitions}" + + (f" — {sample['reason']}" if "reason" in sample else ""), flush=True) + + aggregates = {} + for name, trials in samples.items(): + ready = [r["ready_s"] for r in trials if r["status"] == "PASS"] + aggregates[name] = { + "attempted": len(trials), "passed": len(ready), "failed": len(trials) - len(ready), + "ready_seconds": [min(ready), median(ready), max(ready)] if ready else None, + } + stable = len(digests) == 1 + pipeline.c.evidence.event("versioned_summary", repeats=repetitions, results=aggregates, + identical_v2=stable, v2_sha256=dict(next(iter(digests))) if stable else None) + check(all(result["passed"] == repetitions for result in aggregates.values()) and stable, + "versioned experiment failed a trial or produced different v2 checksums")