diff --git a/Makefile b/Makefile index 5d86d17..43e0a2a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,9 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh diff --git a/REPORT.md b/REPORT.md index e98f75e..a084953 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,182 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Пластинин Денис, БВТ2303. +- Целевая оценка: 5. +- Выбранный трек: 2, доказанная идемпотентность versioned pipeline. +- Ветка решения: `solution/bvt2303-plastinin`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Локальные пароли из `lab5.env.example` предназначены только для изолированного учебного стенда. -## Архитектура и решения +```sh +cp lab5.env.example .env +docker compose config --quiet +docker compose up -d --build --scale api=2 --wait --wait-timeout 180 +make test +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +После запуска доступны: + +- API через Nginx: `http://localhost:8080`; +- Swagger UI: `http://localhost:8081`; +- Prometheus: `http://localhost:9090`; +- Grafana: `http://localhost:3000`, пользователь `admin`, пароль берётся из `GRAFANA_ADMIN_PASSWORD`. + +Интеграционные проверки PostgreSQL и Valkey запускаются отдельно: + +```sh +docker compose --profile tests run --rm --build tests +``` + +Эксперимент трека 5 временно останавливает обычные dispatcher и worker, запускает управляемые сбои, затем возвращает процессы в работу: + +```sh +make test-lab5 +``` + +Обязательный сценарий с остановленным worker и разрывом commit → publish: + +```sh +make test-lab5-acceptance +``` + +Обычная остановка сохраняет volumes: + +```sh +docker compose down +``` + +Намеренное удаление всех учебных данных: + +```sh +docker compose down -v +``` + +## Архитектура + +```mermaid +flowchart LR + Client --> Nginx + Nginx --> API1[API replica 1] + Nginx --> API2[API replica 2] + API1 --> PostgreSQL + API2 --> PostgreSQL + API1 --> Valkey[Valkey sessions] + API2 --> Valkey + API1 --> S3[(MinIO / S3)] + API2 --> S3 + PostgreSQL -->|outbox lease| Dispatcher + Dispatcher -->|XADD| Redis[(Redis Streams + AOF)] + Redis -->|XREADGROUP| Worker + Worker --> S3 + Worker -->|lease + result transaction| PostgreSQL + Worker -->|XACK after commit| Redis +``` + +PostgreSQL является источником истины для metadata, состояния job, попытки, lease, версий pipeline, вариантов и outbox. MinIO хранит исходник в staging и готовые объекты. Valkey хранит только сессии. Отдельный Redis используется как durable broker: включены AOF, `appendfsync always`, volume и политика `noeviction`. + +Temporal не используется. В работе один короткий тип workflow без ожидания внешних событий и компенсаций между множеством сервисов. Transactional outbox, job lease и Redis consumer group закрывают требуемые окна отказа с меньшим числом компонентов. Добавление Temporal потребовало бы Temporal Service и отдельного persistent store, при этом PostgreSQL job/outbox всё равно понадобился бы для атомарной связи с attachment metadata. + +## Граница ответа 202 + +API полностью читает и проверяет PNG/JPEG: MIME, фактический формат, размер и число пикселей. Затем создаётся `pending` metadata, а байты записываются в приватный ключ `staging//` в S3. После успешной записи одна транзакция PostgreSQL переводит attachment в `queued`, создаёт job и outbox event. Только после commit клиент получает `202 Accepted` со snapshot состояния `queued`. + +Если запись staging не удалась, `202` не возвращается. Если соединение потеряно после commit, сохранённый outbox остаётся источником восстановления. Бинарные данные, S3 ключи и credentials в Redis не передаются: envelope содержит только schema version, job ID, attachment ID, pipeline version, attempt, время и trace context. + +## Обработка изображений + +Worker повторно проверяет доверенный staging объект и создаёт четыре content-addressed объекта: + +| Вариант | Формат | Правило | +| --- | --- | --- | +| `original` | исходный PNG/JPEG | байты не изменяются | +| `thumbnail` | JPEG quality 85 | fit в `256×256`, без увеличения | +| `crop` | JPEG quality 85 | центральный квадрат `128×128`, увеличение разрешено | +| `optimized` | WebP quality 82, method 4 | fit в `1280×1280`, без увеличения | + +EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments//pipelines///.`, поэтому повтор той же операции записывает тот же объект. + +Concurrency и prefetch ограничены значениями `WORKER_CONCURRENCY` и `WORKER_PREFETCH`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `WORKER_MEMORY_BYTES` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд. + +## Доставка, retry и восстановление + +Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль. + +Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями. + +Временная ошибка переводит job обратно в `queued`, увеличивает `attempt` и в той же транзакции создаёт новый outbox event. Задержка растёт экспоненциально и получает детерминированный jitter до 25%. Постоянные ошибки (`invalid_image`, `image_too_large`, `source_missing`) и исчерпание пяти попыток переводят attachment в `failed`. Явный replay выполняется командой: + +```sh +docker compose run --rm api /app/messenger replay-attachment +``` + +Replay допустим только для `failed`, увеличивает attempt и сохраняет audit record. Повтор `ready` job становится no-op. `XAUTOCLAIM` возвращает зависшие Redis deliveries, а PostgreSQL recovery возвращает job с истёкшим lease в очередь. Старый consumer не может подтвердить delivery после передачи новому consumer: Lua script проверяет владельца PEL перед `XACK` или переносом в failed stream. + +## Эксперимент трека 5 + +### Гипотеза + +At-least-once доставка не создаст второй логический результат, если effect адресуется по `(attachment_id, pipeline_version, variant)`, а публикация metadata защищена `attempt` и lease token. Старая версия pipeline не сможет перезаписать текущую. + +### Методика + +Go-программа `cmd/lab5experiment` работает с настоящими PostgreSQL, Redis Streams и MinIO из Compose. Используется PNG `320×200`. Обычные worker и dispatcher на время прогона остановлены, чтобы порядок событий задавался экспериментом. + +Автоматически выполняются: + +1. crash после claim, но до side effect; +2. crash после записи всех S3 объектов, но до commit результата; +3. crash после commit результата, но до consumer ACK; +4. две дополнительные доставки уже завершённой job; +5. постоянная ошибка повреждённого staging и явный replay; +6. завершение pipeline v2, после которого доставляется устаревшая job v1. + +Для первых двух сценариев ожидается истечение lease, увеличение attempt и завершение следующей попыткой. В третьем и четвёртом сценариях повтор должен подтвердиться как no-op. Повреждённый staging дважды завершается одинаковым безопасным `invalid_image`, причём replay увеличивает attempt и проходит только через audit/outbox. После готовой версии v2 поздняя v1 не меняет metadata. + +### Результат + +Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidence/lab5-resilience.json`. + +| Воздействие | До восстановления | После восстановления | +| --- | --- | --- | +| crash до side effect | 0 конечных объектов | attempt 2, `ready`, 4 объекта | +| crash после side effect | 4 объекта, DB ещё `processing` | attempt 2, `ready`, те же 4 объекта | +| crash после commit до ACK | DB уже `ready`, 4 объекта | attempt 1, повтор no-op, 4 объекта | +| две duplicate deliveries | 4 объекта | 4 объекта | +| permanent failure и replay | `failed`, `invalid_image` | replay attempt 2, снова `failed`, `invalid_image` | +| stale v1 после готовой v2 | `ready` v2 | остаётся `ready` v2, 8 объектов для двух версий | + +Все пять подтверждённых logical jobs дошли до терминального состояния: четыре `ready`, одна ожидаемо `failed`. Инварианты `accepted_jobs_reach_terminal_state`, `duplicate_effect_is_unique` и `old_pipeline_cannot_replace_new` получили `true`. Прогон занял 6981 ms. + +Это не exactly-once доставка: Redis может доставить одно сообщение несколько раз, а dispatcher может повторить `XADD`. Exactly-once относится только к наблюдаемому эффекту, который достигается идемпотентными ключами и compare-and-set в PostgreSQL. + +## Мониторинг и CI + +Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard содержит RPS, status classes, p95 API latency, Go heap, PostgreSQL, Valkey, MinIO, throughput pipeline по результатам, p95 обработки, in-flight jobs, retry/reclaim, outbox publish failures и Redis broker memory/commands. + +CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда | Результат | +| --- | --- | --- | +| Публичный API lab5 | `make test` | PASS: queued → ready, original и thumbnail проверены | +| PostgreSQL и Valkey | `docker compose --profile tests run --rm --build tests` | оба пакета PASS | +| Transform | `go test ./internal/messenger/attachment/transform` | thumbnail/crop/WebP, EXIF, alpha и лимиты PASS | +| Redis и outbox | `go test ./internal/broker/redisstream ./internal/outbox` | envelope, poison, owner fencing и lost lease PASS | +| Worker остановлен, outbox gap, restart broker/API | `make test-lab5-acceptance` | queued link и 409 до worker; после restart `ready`, original сохранён | +| Трек 5 | `make test-lab5` | 5/5 jobs terminal, 4 ready, 1 expected failed, все три инварианта true | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Ограничения -## Эксперименты и отказы +Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Lease должен быть больше максимального времени обработки; для больших production изображений нужен heartbeat продления lease. Deadline проверяется между стадиями, но библиотечный вызов WebP не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Источники и библиотеки -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Redis Streams и consumer groups: . +- Redis persistence: . +- PostgreSQL `SKIP LOCKED`: . +- MinIO Go SDK: . +- OpenTelemetry trace context: . +- Go-пакеты: `pgx/v5`, `go-redis/v9`, `minio-go/v7`, `deepteams/webp`, Prometheus client. diff --git a/cmd/lab4experiment/main.go b/cmd/lab4experiment/main.go deleted file mode 100644 index 9ce5d5a..0000000 --- a/cmd/lab4experiment/main.go +++ /dev/null @@ -1,676 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "flag" - "fmt" - "image" - "image/jpeg" - "io" - "net/http" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" - - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" - "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile" -) - -const experimentPassword = "Lab4-experiment-password" - -type apiClient struct { - baseURL string - client *http.Client - token string -} - -type attachment struct { - ID string `json:"id"` -} - -type setup struct { - client *apiClient - chatID string -} - -func main() { - if err := run(os.Args[1:], os.Stdout); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func run(arguments []string, output io.Writer) error { - if len(arguments) == 0 { - return errors.New("usage: lab4experiment memory|consistency [flags]") - } - switch arguments[0] { - case "memory": - return runMemory(arguments[1:], output) - case "consistency": - return runConsistency(arguments[1:], output) - default: - return errors.New("usage: lab4experiment memory|consistency [flags]") - } -} - -type memoryResult struct { - Replicas int `json:"replicas"` - Concurrency int `json:"concurrency"` - Rounds int `json:"rounds"` - SettleTimeMillis int64 `json:"settle_time_ms"` - ImageBytes int `json:"image_bytes"` - UploadRateBytes int64 `json:"upload_rate_bytes_per_second"` - SuccessfulUploads int64 `json:"successful_uploads"` - FailedUploads int64 `json:"failed_uploads"` - DurationMillis int64 `json:"duration_ms"` - HeapBaselineBytes int64 `json:"heap_baseline_bytes"` - HeapPeakBytes int64 `json:"heap_peak_bytes"` - HeapDeltaBytes int64 `json:"heap_delta_bytes"` - RSSBaselineBytes int64 `json:"rss_baseline_bytes"` - RSSPeakBytes int64 `json:"rss_peak_bytes"` - RSSDeltaBytes int64 `json:"rss_delta_bytes"` -} - -func runMemory(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("memory", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL") - replicas := flags.Int("replicas", 2, "expected API replicas") - concurrency := flags.Int("concurrency", 12, "number of concurrent uploads") - rounds := flags.Int("rounds", 4, "number of upload waves") - width := flags.Int("width", 2048, "image width") - height := flags.Int("height", 2048, "image height") - uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload") - settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid memory experiment flags") - } - if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 { - return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive") - } - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) - defer cancel() - fixture, err := jpegFixture(*width, *height) - if err != nil { - return err - } - if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil { - return err - } - settle := time.NewTimer(*settleTime) - defer settle.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-settle.C: - } - heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - return err - } - rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - return err - } - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - - var successful atomic.Int64 - var failed atomic.Int64 - var firstError struct { - sync.Mutex - err error - } - heapPeak := heapBaseline - rssPeak := rssBaseline - loadDone := make(chan struct{}) - sampleDone := make(chan error, 1) - go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone) - started := time.Now() - for round := range *rounds { - var workers sync.WaitGroup - workers.Add(*concurrency) - for index := range *concurrency { - go func(round, index int) { - defer workers.Done() - reader := newRateReader(fixture, *uploadRate) - filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil { - failed.Add(1) - firstError.Lock() - if firstError.err == nil { - firstError.err = err - } - firstError.Unlock() - return - } - successful.Add(1) - }(round, index) - } - workers.Wait() - } - close(loadDone) - if err := <-sampleDone; err != nil { - return err - } - result := memoryResult{ - Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(), - ImageBytes: len(fixture), UploadRateBytes: *uploadRate, - SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(), - HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline, - RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline, - } - if result.FailedUploads > 0 { - return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err) - } - return writeJSON(output, result) -} - -func sampleMemory( - ctx context.Context, - prometheusURL string, - expectedReplicas int, - done <-chan struct{}, - heapPeak *int64, - rssPeak *int64, - result chan<- error, -) { - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - result <- ctx.Err() - return - case <-done: - result <- nil - return - case <-ticker.C: - replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err != nil { - result <- err - return - } - if replicas != int64(expectedReplicas) { - result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas) - return - } - heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes")) - if err != nil { - result <- err - return - } - rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes")) - if err != nil { - result <- err - return - } - *heapPeak = max(*heapPeak, heap) - *rssPeak = max(*rssPeak, rss) - } - } -} - -type consistencyResult struct { - PageSize int `json:"page_size"` - SeededConsistent int `json:"seeded_consistent"` - FreshApply reconciliationRun `json:"fresh_apply"` - DryRun reconciliationRun `json:"dry_run"` - Apply reconciliationRun `json:"apply"` - RepeatedApply reconciliationRun `json:"repeated_apply"` - FreshPreserved bool `json:"fresh_preserved"` - OrphanDeleted bool `json:"orphan_deleted"` - UnusedDeleted bool `json:"unused_metadata_deleted"` - InUsePreserved bool `json:"in_use_metadata_preserved"` - AbandonedDeleted bool `json:"abandoned_attachment_deleted"` - TotalDurationMillis int64 `json:"total_duration_ms"` -} - -type reconciliationRun struct { - DurationMillis int64 `json:"duration_ms"` - Report reconcile.Report `json:"report"` -} - -func runConsistency(arguments []string, output io.Writer) error { - flags := flag.NewFlagSet("consistency", flag.ContinueOnError) - flags.SetOutput(io.Discard) - baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") - pageSize := flags.Int("page-size", 10, "reconciliation page size") - if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { - return errors.New("invalid consistency experiment flags") - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - settings, err := config.LoadReconciliation() - if err != nil { - return err - } - pool, err := database.Open(ctx, database.Config{ - URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, - MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, - HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, - }) - if err != nil { - return err - } - defer pool.Close() - objects, err := objectstore.New(objectstore.Config{ - Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, - AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, - UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, - }) - if err != nil { - return err - } - metadata := postgres.New(pool) - clean, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - initial, err := clean.Run(ctx) - if err != nil { - return err - } - if hasAnomalies(initial) { - return errors.New("consistency experiment requires a stack without existing attachment anomalies") - } - - started := time.Now() - environment, err := prepare(ctx, *baseURL) - if err != nil { - return err - } - fixture, err := jpegFixture(320, 200) - if err != nil { - return err - } - seededConsistent := *pageSize + 1 - for index := range seededConsistent { - filename := fmt.Sprintf("consistent-%03d.jpg", index) - if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil { - return err - } - } - unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil { - return err - } - abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture))) - if err != nil { - return err - } - if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("prepare abandoned pending attachment: %w", err) - } - keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID}) - if err != nil { - return err - } - if err := objects.Delete(ctx, keys[unused.ID]); err != nil { - return err - } - if err := objects.Delete(ctx, keys[inUse.ID]); err != nil { - return err - } - orphanKey := "attachments/experiment-" + uuid.NewString() + "/original" - if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil { - return err - } - - fresh, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour, - UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - freshRun, err := measuredReconciliation(ctx, fresh) - if err != nil { - return err - } - freshPreserved := objectExists(ctx, objects, orphanKey) && - attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) && - attachmentExists(ctx, pool, abandoned.ID) - if _, err := pool.Exec(ctx, ` - UPDATE attachments - SET created_at = clock_timestamp() - INTERVAL '2 hours' - WHERE id = $1`, abandoned.ID); err != nil { - return fmt.Errorf("age abandoned attachment: %w", err) - } - - dryRun, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, - }) - if err != nil { - return err - } - dryRunResult, err := measuredReconciliation(ctx, dryRun) - if err != nil { - return err - } - apply, err := reconcile.New(metadata, objects, reconcile.Config{ - Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true, - }) - if err != nil { - return err - } - applyResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - repeatedResult, err := measuredReconciliation(ctx, apply) - if err != nil { - return err - } - - result := consistencyResult{ - PageSize: *pageSize, SeededConsistent: seededConsistent, - FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult, - FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey), - UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID), - AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]), - TotalDurationMillis: time.Since(started).Milliseconds(), - } - if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted || - repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 { - return fmt.Errorf("unexpected reconciliation result: %+v", result) - } - return writeJSON(output, result) -} - -func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) { - started := time.Now() - report, err := reconciler.Run(ctx) - return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err -} - -func hasAnomalies(report reconcile.Report) bool { - return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 || - report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 || - report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 || - report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0 -} - -func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) { - rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids) - if err != nil { - return nil, fmt.Errorf("query attachment keys: %w", err) - } - defer rows.Close() - keys := make(map[string]string, len(ids)) - for rows.Next() { - var id string - var key string - if err := rows.Scan(&id, &key); err != nil { - return nil, fmt.Errorf("scan attachment key: %w", err) - } - keys[id] = key - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate attachment keys: %w", err) - } - if len(keys) != len(ids) { - return nil, errors.New("not all attachment keys were found") - } - return keys, nil -} - -func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool { - var exists bool - return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists -} - -func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool { - _, err := objects.Stat(ctx, key) - return err == nil -} - -func prepare(ctx context.Context, baseURL string) (setup, error) { - client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}} - username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "") - var user struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{ - "username": username, "display_name": "Lab 4 experiment", "password": experimentPassword, - }, &user, ""); err != nil { - return setup{}, err - } - credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword)) - var session struct { - Token string `json:"token"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil { - return setup{}, err - } - client.token = session.Token - var chat struct { - ID string `json:"id"` - } - if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{ - "title": "Lab 4 experiment", "member_ids": []string{user.ID}, - }, &chat, "Bearer "+client.token); err != nil { - return setup{}, err - } - return setup{client: client, chatID: chat.ID}, nil -} - -func (c *apiClient) upload( - ctx context.Context, - chatID string, - filename string, - body io.Reader, - contentLength int64, -) (attachment, error) { - path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename - request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body) - if err != nil { - return attachment{}, err - } - request.ContentLength = contentLength - request.Header.Set("Authorization", "Bearer "+c.token) - request.Header.Set("Content-Type", "image/jpeg") - response, err := c.client.Do(request) - if err != nil { - return attachment{}, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusCreated { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message) - } - var result attachment - if err := json.NewDecoder(response.Body).Decode(&result); err != nil { - return attachment{}, err - } - return result, nil -} - -func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error { - return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{ - "text": "", "attachment_ids": []string{attachmentID}, - }, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString()) -} - -func (c *apiClient) jsonRequest( - ctx context.Context, - method string, - path string, - payload any, - result any, - authorization string, - extraHeader ...string, -) error { - var body io.Reader - if payload != nil { - encoded, err := json.Marshal(payload) - if err != nil { - return err - } - body = bytes.NewReader(encoded) - } - request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) - if err != nil { - return err - } - if payload != nil { - request.Header.Set("Content-Type", "application/json") - } - if authorization != "" { - request.Header.Set("Authorization", authorization) - } - for index := 0; index+1 < len(extraHeader); index += 2 { - request.Header.Set(extraHeader[index], extraHeader[index+1]) - } - response, err := c.client.Do(request) - if err != nil { - return err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode < 200 || response.StatusCode >= 300 { - message, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message) - } - if result == nil { - return nil - } - return json.NewDecoder(response.Body).Decode(result) -} - -func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery()) - if err == nil && count == int64(expected) { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err()) - case <-ticker.C: - } - } -} - -func activeReplicasQuery() string { - return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))` -} - -func freshMetricQuery(metric string) string { - return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric) -} - -func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil) - if err != nil { - return 0, err - } - values := request.URL.Query() - values.Set("query", query) - request.URL.RawQuery = values.Encode() - response, err := http.DefaultClient.Do(request) - if err != nil { - return 0, err - } - defer func() { _ = response.Body.Close() }() - if response.StatusCode != http.StatusOK { - return 0, fmt.Errorf("prometheus status %d", response.StatusCode) - } - var payload struct { - Status string `json:"status"` - Data struct { - Result []struct { - Value []json.RawMessage `json:"value"` - } `json:"result"` - } `json:"data"` - } - if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { - return 0, err - } - if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 { - return 0, errors.New("prometheus query returned no scalar value") - } - var raw string - if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil { - return 0, err - } - value, err := strconv.ParseFloat(raw, 64) - return int64(value), err -} - -func jpegFixture(width, height int) ([]byte, error) { - pixels := image.NewRGBA(image.Rect(0, 0, width, height)) - state := uint64(0x9e3779b97f4a7c15) - for index := range pixels.Pix { - state ^= state << 13 - state ^= state >> 7 - state ^= state << 17 - pixels.Pix[index] = byte(state >> 24) - } - var encoded bytes.Buffer - if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil { - return nil, err - } - return encoded.Bytes(), nil -} - -type rateReader struct { - reader *bytes.Reader - rate int64 - started time.Time - read int64 -} - -func newRateReader(data []byte, rate int64) *rateReader { - return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()} -} - -func (r *rateReader) Read(buffer []byte) (int, error) { - for { - allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read - if allowed > 0 { - if int64(len(buffer)) > allowed { - buffer = buffer[:allowed] - } - count, err := r.reader.Read(buffer) - r.read += int64(count) - return count, err - } - time.Sleep(10 * time.Millisecond) - } -} - -func writeJSON(output io.Writer, value any) error { - encoder := json.NewEncoder(output) - encoder.SetIndent("", " ") - return encoder.Encode(value) -} diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go new file mode 100644 index 0000000..5dc3072 --- /dev/null +++ b/cmd/lab5experiment/api_scenario.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" +) + +type apiState struct { + BaseURL string `json:"base_url"` + Token string `json:"token"` + AttachmentID string `json:"attachment_id"` +} + +type apiUser struct { + ID string `json:"id"` +} + +type apiSession struct { + Token string `json:"token"` +} + +type apiChat struct { + ID string `json:"id"` +} + +func runAPIProduce(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-produce [-base-url URL]") + } + client := &http.Client{Timeout: 10 * time.Second} + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + password := "Lab5-" + uuid.NewString() + owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password) + if err != nil { + return err + } + token, err := createAPISession(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + var attachment model.Attachment + response, err := apiRequest( + client, http.MethodPost, + *baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png", + token, "image/png", fixture, &attachment, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued { + return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status) + } + messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}}) + if err != nil { + return err + } + response, err = apiRequestWithHeaders( + client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages", + token, "application/json", messageBody, nil, + map[string]string{"Idempotency-Key": uuid.NewString()}, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("link queued attachment returned %d", response.StatusCode) + } + response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil) + if err != nil { + return err + } + if response.StatusCode != http.StatusConflict { + return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) + } + return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) +} + +func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { + flags := flag.NewFlagSet("api-verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]") + } + var state apiState + if err := json.NewDecoder(input).Decode(&state); err != nil { + return fmt.Errorf("decode API scenario state: %w", err) + } + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) + defer cancel() + var attachment model.Attachment + for { + response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("attachment metadata returned %d", response.StatusCode) + } + if attachment.Status == model.AttachmentReady { + break + } + if attachment.Status == model.AttachmentFailed { + return fmt.Errorf("attachment failed with %v", attachment.ErrorCode) + } + if err := wait(ctx, 250*time.Millisecond); err != nil { + return err + } + } + if len(attachment.Variants) != 3 { + return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants)) + } + response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) { + return errors.New("original changed after worker and broker restart") + } + response, thumbnail, err := apiBytes( + client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" { + return errors.New("thumbnail response is invalid") + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail)) + if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 { + return errors.New("thumbnail dimensions are invalid") + } + return json.NewEncoder(output).Encode(map[string]any{ + "accepted_while_worker_stopped": true, + "accepted_before_outbox_publish": true, + "linked_while_queued": true, + "content_conflict_while_queued": true, + "ready_after_api_broker_restart": true, + "original_preserved": true, + "thumbnail_dimensions": "256x160", + }) +} + +func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": username, "password": password, + }) + if err != nil { + return apiUser{}, err + } + var user apiUser + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user) + if err != nil { + return apiUser{}, err + } + if response.StatusCode != http.StatusCreated { + return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode) + } + return user, nil +} + +func createAPISession(client *http.Client, baseURL, username, password string) (string, error) { + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Basic "+credentials) + response, err := client.Do(request) + if err != nil { + return "", err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create session returned %d", response.StatusCode) + } + var session apiSession + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return "", err + } + return session.Token, nil +} + +func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) { + body, err := json.Marshal(map[string]any{ + "title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID}, + }) + if err != nil { + return apiChat{}, err + } + var chat apiChat + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat) + if err != nil { + return apiChat{}, err + } + if response.StatusCode != http.StatusCreated { + return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode) + } + return chat, nil +} + +func apiRequest( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, +) (*http.Response, error) { + return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil) +} + +func apiRequestWithHeaders( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, + headers map[string]string, +) (*http.Response, error) { + request, err := http.NewRequest(method, requestURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + for name, value := range headers { + request.Header.Set(name, value) + } + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if result != nil && response.StatusCode < http.StatusBadRequest { + if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil { + return nil, err + } + } + return response, nil +} + +func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) { + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + return nil, nil, err + } + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + return response, body, err +} diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go new file mode 100644 index 0000000..b6532d6 --- /dev/null +++ b/cmd/lab5experiment/main.go @@ -0,0 +1,605 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres" +) + +const ( + experimentLease = 2 * time.Second + maxAttempts = 5 +) + +type result struct { + StartedAt string `json:"started_at"` + DurationMillis int64 `json:"duration_ms"` + PipelineVersion int `json:"pipeline_version"` + FaultScenarios []scenario `json:"fault_scenarios"` + DuplicateDeliveries int `json:"duplicate_deliveries"` + V1ObjectCount int `json:"v1_object_count"` + V2ObjectCount int `json:"v2_object_count"` + AcceptedJobs int `json:"accepted_jobs"` + TerminalJobs int `json:"terminal_jobs"` + ReadyJobs int `json:"ready_jobs"` + FailedJobs int `json:"failed_jobs"` + PermanentErrorCode string `json:"permanent_error_code"` + ReplayAttempt int `json:"replay_attempt"` + InvariantResults map[string]bool `json:"invariant_results"` +} + +type scenario struct { + Name string `json:"name"` + AttemptAfterRecovery int `json:"attempt_after_recovery"` + ObjectsBeforeRecovery int `json:"objects_before_recovery"` + ObjectsAfterRecovery int `json:"objects_after_recovery"` + FinalStatus string `json:"final_status"` +} + +type experiment struct { + settings config.WorkerConfig + repository *postgres.Store + objects *objectstore.Store + broker *redisstream.Broker + dispatcher *outbox.Dispatcher + worker *worker.Worker + ownerID string + chatID string + fixture []byte +} + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(arguments []string, output io.Writer) error { + if len(arguments) > 0 { + switch arguments[0] { + case "api-produce": + return runAPIProduce(arguments[1:], output) + case "api-verify": + return runAPIVerify(arguments[1:], os.Stdin, output) + } + } + flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) + flags.SetOutput(io.Discard) + timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { + return errors.New("usage: lab5experiment [-timeout 2m]") + } + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + experiment, closeExperiment, err := newExperiment(ctx) + if err != nil { + return err + } + defer closeExperiment() + + report, err := experiment.run(ctx) + if err != nil { + return err + } + report.StartedAt = started.UTC().Format(time.RFC3339) + report.DurationMillis = time.Since(started).Milliseconds() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func newExperiment(ctx context.Context) (*experiment, func(), error) { + settings, err := config.LoadWorker() + if err != nil { + return nil, nil, err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return nil, nil, err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions, err := redis.ParseURL(settings.BrokerURL) + if err != nil { + pool.Close() + return nil, nil, err + } + client := redis.NewClient(redisOptions) + if err := client.Ping(ctx).Err(); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + consumer := "lab5experiment-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, + ClaimMinIdle: experimentLease, ReadCount: 32, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + if err := broker.Ensure(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + repository := postgres.New(pool) + dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ + BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, + RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processorWorker, err := worker.New( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, + worker.Config{ + WorkerID: consumer, Concurrency: 1, + LeaseDuration: experimentLease, ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + fixture, err := pngFixture(320, 200) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + value := &experiment{ + settings: settings, repository: repository, objects: objects, broker: broker, + dispatcher: dispatcher, worker: processorWorker, fixture: fixture, + } + if err := value.seed(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + closeExperiment := func() { + pool.Close() + _ = client.Close() + } + return value, closeExperiment, nil +} + +func (e *experiment) seed(ctx context.Context) error { + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") + if err != nil { + return err + } + member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") + if err != nil { + return err + } + chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) + if err != nil { + return err + } + e.ownerID = owner.ID + e.chatID = chat.ID + return nil +} + +func (e *experiment) run(ctx context.Context) (result, error) { + report := result{ + PipelineVersion: 1, + InvariantResults: map[string]bool{ + "accepted_jobs_reach_terminal_state": false, + "duplicate_effect_is_unique": false, + "old_pipeline_cannot_replace_new": false, + }, + } + for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { + observed, err := e.runLeaseRecovery(ctx, fault) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, observed) + report.AcceptedJobs++ + if observed.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + } + permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) + if err != nil { + return result{}, err + } + report.AcceptedJobs++ + report.TerminalJobs++ + report.FailedJobs++ + report.PermanentErrorCode = permanentCode + report.ReplayAttempt = replayAttempt + + commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, commitScenario) + report.AcceptedJobs++ + if commitScenario.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + + payload, err := redisstream.EncodeEnvelope(envelope) + if err != nil { + return result{}, err + } + for range 2 { + if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { + return result{}, err + } + } + duplicateDeliveries, err := e.readDeliveries(ctx, 2) + if err != nil { + return result{}, err + } + for _, delivery := range duplicateDeliveries { + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return result{}, err + } + } + report.DuplicateDeliveries = len(duplicateDeliveries) + report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V1ObjectCount != 4 { + return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) + } + report.InvariantResults["duplicate_effect_is_unique"] = true + + oldPayload := payload + v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} + if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { + return result{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return result{}, err + } + v2Delivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { + return result{}, err + } + if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { + return result{}, err + } + staleDelivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { + return result{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return result{}, err + } + if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { + return result{}, errors.New("new pipeline version was not preserved after stale delivery") + } + report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V2ObjectCount != 8 { + return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) + } + report.InvariantResults["old_pipeline_cannot_replace_new"] = true + report.AcceptedJobs++ + report.ReadyJobs++ + report.TerminalJobs++ + report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs + return report, nil +} + +func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { + attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") + if err != nil { + return "", 0, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { + return "", 0, errors.New("corrupted staging object did not reach failed") + } + firstCode := *metadata.ErrorCode + replayed, err := e.repository.ReplayFailedAttachment( + ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", + ) + if err != nil { + return "", 0, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", 0, err + } + replayDelivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { + return "", 0, errors.New("replayed permanent failure changed terminal classification") + } + return firstCode, replayed.Attempt, nil +} + +func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + if fault == worker.FaultBeforeSideEffect && before != 0 { + return scenario{}, errors.New("objects were written before the injected side effect") + } + if fault == worker.FaultAfterSideEffect && before != 4 { + return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, err + } + recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) + if err != nil { + return scenario{}, err + } + if recovered.Requeued != 1 { + return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return scenario{}, err + } + retry, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { + return scenario{}, err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) + if err != nil { + return scenario{}, err + } + return scenario{ + Name: string(fault), AttemptAfterRecovery: job.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, nil +} + +func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + return scenario{ + Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, delivery.Envelope, attachmentID, nil +} + +func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { + attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") + return attachmentID, job, err +} + +func (e *experiment) queueAttachmentData( + ctx context.Context, + data []byte, + contentType string, +) (string, string, model.AttachmentJob, error) { + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/experiment" + digest := sha256.Sum256(data) + _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, + }) + if err != nil { + return "", "", model.AttachmentJob{}, err + } + if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { + return "", "", model.AttachmentJob{}, err + } + job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} + if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { + return "", "", model.AttachmentJob{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", "", model.AttachmentJob{}, err + } + return attachmentID, stagingKey, job, nil +} + +func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { + deliveries, err := e.readDeliveries(ctx, 1) + if err != nil { + return redisstream.Delivery{}, err + } + return deliveries[0], nil +} + +func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { + var result []redisstream.Delivery + for len(result) < expected { + deliveries, err := e.broker.Read(ctx) + if err != nil { + return nil, err + } + result = append(result, deliveries...) + } + if len(result) != expected { + return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) + } + return result, nil +} + +func (e *experiment) ackReclaimed(ctx context.Context) error { + deliveries, _, err := e.broker.Reclaim(ctx, "0-0") + if err != nil { + return err + } + if len(deliveries) != 1 { + return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) + } + return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) +} + +func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { + objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) + if err != nil { + return 0, err + } + if next != "" { + return 0, errors.New("unexpected object pagination") + } + return len(objects), nil +} + +func pngFixture(width, height int) ([]byte, error) { + value := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, value); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func wait(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/evidence/lab5-idempotency.json b/evidence/lab5-idempotency.json new file mode 100644 index 0000000..c6e9050 --- /dev/null +++ b/evidence/lab5-idempotency.json @@ -0,0 +1,42 @@ +{ + "started_at": "2026-10-22T14:19:25Z", + "duration_ms": 6981, + "pipeline_version": 1, + "fault_scenarios": [ + { + "name": "before_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 0, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_side_effect", + "attempt_after_recovery": 2, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + }, + { + "name": "after_commit", + "attempt_after_recovery": 1, + "objects_before_recovery": 4, + "objects_after_recovery": 4, + "final_status": "ready" + } + ], + "duplicate_deliveries": 2, + "v1_object_count": 4, + "v2_object_count": 8, + "accepted_jobs": 5, + "terminal_jobs": 5, + "ready_jobs": 4, + "failed_jobs": 1, + "permanent_error_code": "invalid_image", + "replay_attempt": 2, + "invariant_results": { + "accepted_jobs_reach_terminal_state": true, + "duplicate_effect_is_unique": true, + "old_pipeline_cannot_replace_new": true + } +} diff --git a/evidence/lab5-resilience.json b/evidence/lab5-resilience.json new file mode 100644 index 0000000..c82d186 --- /dev/null +++ b/evidence/lab5-resilience.json @@ -0,0 +1 @@ +{"accepted_before_outbox_publish":true,"accepted_while_worker_stopped":true,"content_conflict_while_queued":true,"linked_while_queued":true,"original_preserved":true,"ready_after_api_broker_restart":true,"thumbnail_dimensions":"256x160"} diff --git a/scripts/test-lab5-acceptance.sh b/scripts/test-lab5-acceptance.sh new file mode 100755 index 0000000..f62397c --- /dev/null +++ b/scripts/test-lab5-acceptance.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -eu + +state_file=$(mktemp) + +restore_pipeline() { + rm -f "$state_file" + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM + +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file" +docker compose restart api redis-broker >/dev/null +docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null +docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file" + diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 0000000..9bf4511 --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -eu + +restore_pipeline() { + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment +