test: verify image pipeline recovery

This commit is contained in:
2026-10-22 17:22:32 +03:00
parent 42336ce7c2
commit bc090c0f8b
9 changed files with 1152 additions and 700 deletions
+7 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,9 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-lab5:
./scripts/test-lab5.sh
test-lab5-acceptance:
./scripts/test-lab5-acceptance.sh
+168 -23
View File
@@ -1,37 +1,182 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: Пластинин Денис, БВТ2303.
- Целевая оценка: 5.
- Выбранный трек: 2, доказанная идемпотентность versioned pipeline.
- Ветка решения: `solution/bvt2303-plastinin`.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker с Compose и Python 3.9+. Все серверные зависимости поднимаются из репозитория. Локальные пароли из `lab5.env.example` предназначены только для изолированного учебного стенда.
## Архитектура и решения
```sh
cp lab5.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
make test
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
После запуска доступны:
- API через Nginx: `http://localhost:8080`;
- Swagger UI: `http://localhost:8081`;
- Prometheus: `http://localhost:9090`;
- Grafana: `http://localhost:3000`, пользователь `admin`, пароль берётся из `GRAFANA_ADMIN_PASSWORD`.
Интеграционные проверки PostgreSQL и Valkey запускаются отдельно:
```sh
docker compose --profile tests run --rm --build tests
```
Эксперимент трека 5 временно останавливает обычные dispatcher и worker, запускает управляемые сбои, затем возвращает процессы в работу:
```sh
make test-lab5
```
Обязательный сценарий с остановленным worker и разрывом commit → publish:
```sh
make test-lab5-acceptance
```
Обычная остановка сохраняет volumes:
```sh
docker compose down
```
Намеренное удаление всех учебных данных:
```sh
docker compose down -v
```
## Архитектура
```mermaid
flowchart LR
Client --> Nginx
Nginx --> API1[API replica 1]
Nginx --> API2[API replica 2]
API1 --> PostgreSQL
API2 --> PostgreSQL
API1 --> Valkey[Valkey sessions]
API2 --> Valkey
API1 --> S3[(MinIO / S3)]
API2 --> S3
PostgreSQL -->|outbox lease| Dispatcher
Dispatcher -->|XADD| Redis[(Redis Streams + AOF)]
Redis -->|XREADGROUP| Worker
Worker --> S3
Worker -->|lease + result transaction| PostgreSQL
Worker -->|XACK after commit| Redis
```
PostgreSQL является источником истины для metadata, состояния job, попытки, lease, версий pipeline, вариантов и outbox. MinIO хранит исходник в staging и готовые объекты. Valkey хранит только сессии. Отдельный Redis используется как durable broker: включены AOF, `appendfsync always`, volume и политика `noeviction`.
Temporal не используется. В работе один короткий тип workflow без ожидания внешних событий и компенсаций между множеством сервисов. Transactional outbox, job lease и Redis consumer group закрывают требуемые окна отказа с меньшим числом компонентов. Добавление Temporal потребовало бы Temporal Service и отдельного persistent store, при этом PostgreSQL job/outbox всё равно понадобился бы для атомарной связи с attachment metadata.
## Граница ответа 202
API полностью читает и проверяет PNG/JPEG: MIME, фактический формат, размер и число пикселей. Затем создаётся `pending` metadata, а байты записываются в приватный ключ `staging/<attachment_id>/<uuid>` в S3. После успешной записи одна транзакция PostgreSQL переводит attachment в `queued`, создаёт job и outbox event. Только после commit клиент получает `202 Accepted` со snapshot состояния `queued`.
Если запись staging не удалась, `202` не возвращается. Если соединение потеряно после commit, сохранённый outbox остаётся источником восстановления. Бинарные данные, S3 ключи и credentials в Redis не передаются: envelope содержит только schema version, job ID, attachment ID, pipeline version, attempt, время и trace context.
## Обработка изображений
Worker повторно проверяет доверенный staging объект и создаёт четыре content-addressed объекта:
| Вариант | Формат | Правило |
| --- | --- | --- |
| `original` | исходный PNG/JPEG | байты не изменяются |
| `thumbnail` | JPEG quality 85 | fit в `256×256`, без увеличения |
| `crop` | JPEG quality 85 | центральный квадрат `128×128`, увеличение разрешено |
| `optimized` | WebP quality 82, method 4 | fit в `1280×1280`, без увеличения |
EXIF orientation применяется к производным вариантам. Прозрачность сводится на белый фон. Повторное кодирование удаляет metadata. Ключ эффекта имеет вид `attachments/<id>/pipelines/<version>/<variant>/<sha256>.<ext>`, поэтому повтор той же операции записывает тот же объект.
Concurrency и prefetch ограничены значениями `WORKER_CONCURRENCY` и `WORKER_PREFETCH`. Перед decode worker оценивает память по размеру входа, числу исходных пикселей и производным буферам, затем занимает часть `WORKER_MEMORY_BYTES` через weighted semaphore. Для разрешённых 40 млн пикселей зарезервировано 704 MiB внутри контейнера с лимитом 1 GiB; несогласованная конфигурация останавливает worker при старте. Одна попытка ограничена 20 секундами при lease 30 секунд.
## Доставка, retry и восстановление
Dispatcher забирает due outbox events через `FOR UPDATE SKIP LOCKED` и lease token. После подтверждённого `XADD` event помечается опубликованным. Сбой после `XADD`, но до этой отметки создаёт допустимый дубль.
Worker читает consumer group через `XREADGROUP`. При старте обработки PostgreSQL атомарно проверяет `job_id`, `pipeline_version`, `attempt`, выдаёт lease token и переводит attachment в `processing`. Результат применяется только при совпадении всех fencing полей и живом lease. `XACK` выполняется после commit результата. После ACK обработанная запись удаляется из основного stream; pending записи не вытесняются. Failed stream ограничен приблизительно 10 000 записями.
Временная ошибка переводит job обратно в `queued`, увеличивает `attempt` и в той же транзакции создаёт новый outbox event. Задержка растёт экспоненциально и получает детерминированный jitter до 25%. Постоянные ошибки (`invalid_image`, `image_too_large`, `source_missing`) и исчерпание пяти попыток переводят attachment в `failed`. Явный replay выполняется командой:
```sh
docker compose run --rm api /app/messenger replay-attachment <attachment-id> <actor> <reason>
```
Replay допустим только для `failed`, увеличивает attempt и сохраняет audit record. Повтор `ready` job становится no-op. `XAUTOCLAIM` возвращает зависшие Redis deliveries, а PostgreSQL recovery возвращает job с истёкшим lease в очередь. Старый consumer не может подтвердить delivery после передачи новому consumer: Lua script проверяет владельца PEL перед `XACK` или переносом в failed stream.
## Эксперимент трека 5
### Гипотеза
At-least-once доставка не создаст второй логический результат, если effect адресуется по `(attachment_id, pipeline_version, variant)`, а публикация metadata защищена `attempt` и lease token. Старая версия pipeline не сможет перезаписать текущую.
### Методика
Go-программа `cmd/lab5experiment` работает с настоящими PostgreSQL, Redis Streams и MinIO из Compose. Используется PNG `320×200`. Обычные worker и dispatcher на время прогона остановлены, чтобы порядок событий задавался экспериментом.
Автоматически выполняются:
1. crash после claim, но до side effect;
2. crash после записи всех S3 объектов, но до commit результата;
3. crash после commit результата, но до consumer ACK;
4. две дополнительные доставки уже завершённой job;
5. постоянная ошибка повреждённого staging и явный replay;
6. завершение pipeline v2, после которого доставляется устаревшая job v1.
Для первых двух сценариев ожидается истечение lease, увеличение attempt и завершение следующей попыткой. В третьем и четвёртом сценариях повтор должен подтвердиться как no-op. Повреждённый staging дважды завершается одинаковым безопасным `invalid_image`, причём replay увеличивает attempt и проходит только через audit/outbox. После готовой версии v2 поздняя v1 не меняет metadata.
### Результат
Raw summaries сохранены в `evidence/lab5-idempotency.json` и `evidence/lab5-resilience.json`.
| Воздействие | До восстановления | После восстановления |
| --- | --- | --- |
| crash до side effect | 0 конечных объектов | attempt 2, `ready`, 4 объекта |
| crash после side effect | 4 объекта, DB ещё `processing` | attempt 2, `ready`, те же 4 объекта |
| crash после commit до ACK | DB уже `ready`, 4 объекта | attempt 1, повтор no-op, 4 объекта |
| две duplicate deliveries | 4 объекта | 4 объекта |
| permanent failure и replay | `failed`, `invalid_image` | replay attempt 2, снова `failed`, `invalid_image` |
| stale v1 после готовой v2 | `ready` v2 | остаётся `ready` v2, 8 объектов для двух версий |
Все пять подтверждённых logical jobs дошли до терминального состояния: четыре `ready`, одна ожидаемо `failed`. Инварианты `accepted_jobs_reach_terminal_state`, `duplicate_effect_is_unique` и `old_pipeline_cannot_replace_new` получили `true`. Прогон занял 6981 ms.
Это не exactly-once доставка: Redis может доставить одно сообщение несколько раз, а dispatcher может повторить `XADD`. Exactly-once относится только к наблюдаемому эффекту, который достигается идемпотентными ключами и compare-and-set в PostgreSQL.
## Мониторинг и CI
Prometheus опрашивает API replicas, worker, dispatcher, PostgreSQL exporter, отдельные exporters Valkey и Redis broker, Nginx и MinIO. Grafana dashboard содержит RPS, status classes, p95 API latency, Go heap, PostgreSQL, Valkey, MinIO, throughput pipeline по результатам, p95 обработки, in-flight jobs, retry/reclaim, outbox publish failures и Redis broker memory/commands.
CI для pull request и `main` выполняет contract check, OpenAPI validation, formatting, lint, race tests, build, публичный smoke, интеграционные тесты, persistence/failover lab4 и fault injection lab5. Production workflow запускается вручную и требует approval environment перед rolling deployment.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Команда | Результат |
| --- | --- | --- |
| Публичный API lab5 | `make test` | PASS: queued → ready, original и thumbnail проверены |
| PostgreSQL и Valkey | `docker compose --profile tests run --rm --build tests` | оба пакета PASS |
| Transform | `go test ./internal/messenger/attachment/transform` | thumbnail/crop/WebP, EXIF, alpha и лимиты PASS |
| Redis и outbox | `go test ./internal/broker/redisstream ./internal/outbox` | envelope, poison, owner fencing и lost lease PASS |
| Worker остановлен, outbox gap, restart broker/API | `make test-lab5-acceptance` | queued link и 409 до worker; после restart `ready`, original сохранён |
| Трек 5 | `make test-lab5` | 5/5 jobs terminal, 4 ready, 1 expected failed, все три инварианта true |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Ограничения
## Эксперименты и отказы
Один MinIO, PostgreSQL и Redis container не дают высокой доступности самих хранилищ. `appendfsync always` уменьшает окно потери Redis, но не заменяет репликацию и резервное копирование. Lease должен быть больше максимального времени обработки; для больших production изображений нужен heartbeat продления lease. Deadline проверяется между стадиями, но библиотечный вызов WebP не прерывается посередине кодирования. Failed stream ограничен, поэтому перед достижением retention limit записи нужно выгружать во внешнее долговременное хранилище. Staging у `failed` сохраняется для явного replay; его окончательное удаление остаётся отдельной операторской retention policy.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Источники и библиотеки
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Redis Streams и consumer groups: <https://redis.io/docs/latest/develop/data-types/streams/>.
- Redis persistence: <https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/>.
- PostgreSQL `SKIP LOCKED`: <https://www.postgresql.org/docs/current/sql-select.html>.
- MinIO Go SDK: <https://github.com/minio/minio-go>.
- OpenTelemetry trace context: <https://www.w3.org/TR/trace-context/>.
- Go-пакеты: `pgx/v5`, `go-redis/v9`, `minio-go/v7`, `deepteams/webp`, Prometheus client.
-676
View File
@@ -1,676 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"image"
"image/jpeg"
"io"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
)
const experimentPassword = "Lab4-experiment-password"
type apiClient struct {
baseURL string
client *http.Client
token string
}
type attachment struct {
ID string `json:"id"`
}
type setup struct {
client *apiClient
chatID string
}
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(arguments []string, output io.Writer) error {
if len(arguments) == 0 {
return errors.New("usage: lab4experiment memory|consistency [flags]")
}
switch arguments[0] {
case "memory":
return runMemory(arguments[1:], output)
case "consistency":
return runConsistency(arguments[1:], output)
default:
return errors.New("usage: lab4experiment memory|consistency [flags]")
}
}
type memoryResult struct {
Replicas int `json:"replicas"`
Concurrency int `json:"concurrency"`
Rounds int `json:"rounds"`
SettleTimeMillis int64 `json:"settle_time_ms"`
ImageBytes int `json:"image_bytes"`
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
SuccessfulUploads int64 `json:"successful_uploads"`
FailedUploads int64 `json:"failed_uploads"`
DurationMillis int64 `json:"duration_ms"`
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
HeapPeakBytes int64 `json:"heap_peak_bytes"`
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
RSSPeakBytes int64 `json:"rss_peak_bytes"`
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
}
func runMemory(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
flags.SetOutput(io.Discard)
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
replicas := flags.Int("replicas", 2, "expected API replicas")
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
rounds := flags.Int("rounds", 4, "number of upload waves")
width := flags.Int("width", 2048, "image width")
height := flags.Int("height", 2048, "image height")
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("invalid memory experiment flags")
}
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
fixture, err := jpegFixture(*width, *height)
if err != nil {
return err
}
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
return err
}
settle := time.NewTimer(*settleTime)
defer settle.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-settle.C:
}
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
if err != nil {
return err
}
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
if err != nil {
return err
}
environment, err := prepare(ctx, *baseURL)
if err != nil {
return err
}
var successful atomic.Int64
var failed atomic.Int64
var firstError struct {
sync.Mutex
err error
}
heapPeak := heapBaseline
rssPeak := rssBaseline
loadDone := make(chan struct{})
sampleDone := make(chan error, 1)
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
started := time.Now()
for round := range *rounds {
var workers sync.WaitGroup
workers.Add(*concurrency)
for index := range *concurrency {
go func(round, index int) {
defer workers.Done()
reader := newRateReader(fixture, *uploadRate)
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
failed.Add(1)
firstError.Lock()
if firstError.err == nil {
firstError.err = err
}
firstError.Unlock()
return
}
successful.Add(1)
}(round, index)
}
workers.Wait()
}
close(loadDone)
if err := <-sampleDone; err != nil {
return err
}
result := memoryResult{
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
}
if result.FailedUploads > 0 {
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
}
return writeJSON(output, result)
}
func sampleMemory(
ctx context.Context,
prometheusURL string,
expectedReplicas int,
done <-chan struct{},
heapPeak *int64,
rssPeak *int64,
result chan<- error,
) {
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
result <- ctx.Err()
return
case <-done:
result <- nil
return
case <-ticker.C:
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
if err != nil {
result <- err
return
}
if replicas != int64(expectedReplicas) {
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
return
}
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
if err != nil {
result <- err
return
}
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
if err != nil {
result <- err
return
}
*heapPeak = max(*heapPeak, heap)
*rssPeak = max(*rssPeak, rss)
}
}
}
type consistencyResult struct {
PageSize int `json:"page_size"`
SeededConsistent int `json:"seeded_consistent"`
FreshApply reconciliationRun `json:"fresh_apply"`
DryRun reconciliationRun `json:"dry_run"`
Apply reconciliationRun `json:"apply"`
RepeatedApply reconciliationRun `json:"repeated_apply"`
FreshPreserved bool `json:"fresh_preserved"`
OrphanDeleted bool `json:"orphan_deleted"`
UnusedDeleted bool `json:"unused_metadata_deleted"`
InUsePreserved bool `json:"in_use_metadata_preserved"`
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
TotalDurationMillis int64 `json:"total_duration_ms"`
}
type reconciliationRun struct {
DurationMillis int64 `json:"duration_ms"`
Report reconcile.Report `json:"report"`
}
func runConsistency(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
flags.SetOutput(io.Discard)
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
pageSize := flags.Int("page-size", 10, "reconciliation page size")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("invalid consistency experiment flags")
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
settings, err := config.LoadReconciliation()
if err != nil {
return err
}
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return err
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
return err
}
metadata := postgres.New(pool)
clean, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
initial, err := clean.Run(ctx)
if err != nil {
return err
}
if hasAnomalies(initial) {
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
}
started := time.Now()
environment, err := prepare(ctx, *baseURL)
if err != nil {
return err
}
fixture, err := jpegFixture(320, 200)
if err != nil {
return err
}
seededConsistent := *pageSize + 1
for index := range seededConsistent {
filename := fmt.Sprintf("consistent-%03d.jpg", index)
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
return err
}
}
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
return err
}
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
if err != nil {
return err
}
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
}
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
if err != nil {
return err
}
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
return err
}
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
return err
}
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
return err
}
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
}
freshRun, err := measuredReconciliation(ctx, fresh)
if err != nil {
return err
}
freshPreserved := objectExists(ctx, objects, orphanKey) &&
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
attachmentExists(ctx, pool, abandoned.ID)
if _, err := pool.Exec(ctx, `
UPDATE attachments
SET created_at = clock_timestamp() - INTERVAL '2 hours'
WHERE id = $1`, abandoned.ID); err != nil {
return fmt.Errorf("age abandoned attachment: %w", err)
}
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
})
if err != nil {
return err
}
dryRunResult, err := measuredReconciliation(ctx, dryRun)
if err != nil {
return err
}
apply, err := reconcile.New(metadata, objects, reconcile.Config{
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
})
if err != nil {
return err
}
applyResult, err := measuredReconciliation(ctx, apply)
if err != nil {
return err
}
repeatedResult, err := measuredReconciliation(ctx, apply)
if err != nil {
return err
}
result := consistencyResult{
PageSize: *pageSize, SeededConsistent: seededConsistent,
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
TotalDurationMillis: time.Since(started).Milliseconds(),
}
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
return fmt.Errorf("unexpected reconciliation result: %+v", result)
}
return writeJSON(output, result)
}
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
started := time.Now()
report, err := reconciler.Run(ctx)
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
}
func hasAnomalies(report reconcile.Report) bool {
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
}
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
if err != nil {
return nil, fmt.Errorf("query attachment keys: %w", err)
}
defer rows.Close()
keys := make(map[string]string, len(ids))
for rows.Next() {
var id string
var key string
if err := rows.Scan(&id, &key); err != nil {
return nil, fmt.Errorf("scan attachment key: %w", err)
}
keys[id] = key
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate attachment keys: %w", err)
}
if len(keys) != len(ids) {
return nil, errors.New("not all attachment keys were found")
}
return keys, nil
}
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
var exists bool
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
}
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
_, err := objects.Stat(ctx, key)
return err == nil
}
func prepare(ctx context.Context, baseURL string) (setup, error) {
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
var user struct {
ID string `json:"id"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
}, &user, ""); err != nil {
return setup{}, err
}
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
var session struct {
Token string `json:"token"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
return setup{}, err
}
client.token = session.Token
var chat struct {
ID string `json:"id"`
}
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
}, &chat, "Bearer "+client.token); err != nil {
return setup{}, err
}
return setup{client: client, chatID: chat.ID}, nil
}
func (c *apiClient) upload(
ctx context.Context,
chatID string,
filename string,
body io.Reader,
contentLength int64,
) (attachment, error) {
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
if err != nil {
return attachment{}, err
}
request.ContentLength = contentLength
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "image/jpeg")
response, err := c.client.Do(request)
if err != nil {
return attachment{}, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
}
var result attachment
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
return attachment{}, err
}
return result, nil
}
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
"text": "", "attachment_ids": []string{attachmentID},
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
}
func (c *apiClient) jsonRequest(
ctx context.Context,
method string,
path string,
payload any,
result any,
authorization string,
extraHeader ...string,
) error {
var body io.Reader
if payload != nil {
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
body = bytes.NewReader(encoded)
}
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
if err != nil {
return err
}
if payload != nil {
request.Header.Set("Content-Type", "application/json")
}
if authorization != "" {
request.Header.Set("Authorization", authorization)
}
for index := 0; index+1 < len(extraHeader); index += 2 {
request.Header.Set(extraHeader[index], extraHeader[index+1])
}
response, err := c.client.Do(request)
if err != nil {
return err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode < 200 || response.StatusCode >= 300 {
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
}
if result == nil {
return nil
}
return json.NewDecoder(response.Body).Decode(result)
}
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
if err == nil && count == int64(expected) {
return nil
}
select {
case <-ctx.Done():
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
case <-ticker.C:
}
}
}
func activeReplicasQuery() string {
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
}
func freshMetricQuery(metric string) string {
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
}
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
if err != nil {
return 0, err
}
values := request.URL.Query()
values.Set("query", query)
request.URL.RawQuery = values.Encode()
response, err := http.DefaultClient.Do(request)
if err != nil {
return 0, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
}
var payload struct {
Status string `json:"status"`
Data struct {
Result []struct {
Value []json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
return 0, err
}
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
return 0, errors.New("prometheus query returned no scalar value")
}
var raw string
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
return 0, err
}
value, err := strconv.ParseFloat(raw, 64)
return int64(value), err
}
func jpegFixture(width, height int) ([]byte, error) {
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
state := uint64(0x9e3779b97f4a7c15)
for index := range pixels.Pix {
state ^= state << 13
state ^= state >> 7
state ^= state << 17
pixels.Pix[index] = byte(state >> 24)
}
var encoded bytes.Buffer
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
return nil, err
}
return encoded.Bytes(), nil
}
type rateReader struct {
reader *bytes.Reader
rate int64
started time.Time
read int64
}
func newRateReader(data []byte, rate int64) *rateReader {
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
}
func (r *rateReader) Read(buffer []byte) (int, error) {
for {
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
if allowed > 0 {
if int64(len(buffer)) > allowed {
buffer = buffer[:allowed]
}
count, err := r.reader.Read(buffer)
r.read += int64(count)
return count, err
}
time.Sleep(10 * time.Millisecond)
}
}
func writeJSON(output io.Writer, value any) error {
encoder := json.NewEncoder(output)
encoder.SetIndent("", " ")
return encoder.Encode(value)
}
+298
View File
@@ -0,0 +1,298 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"image"
_ "image/jpeg"
"io"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
)
type apiState struct {
BaseURL string `json:"base_url"`
Token string `json:"token"`
AttachmentID string `json:"attachment_id"`
}
type apiUser struct {
ID string `json:"id"`
}
type apiSession struct {
Token string `json:"token"`
}
type apiChat struct {
ID string `json:"id"`
}
func runAPIProduce(arguments []string, output io.Writer) error {
flags := flag.NewFlagSet("api-produce", flag.ContinueOnError)
flags.SetOutput(io.Discard)
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("usage: lab5experiment api-produce [-base-url URL]")
}
client := &http.Client{Timeout: 10 * time.Second}
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
password := "Lab5-" + uuid.NewString()
owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password)
if err != nil {
return err
}
member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password)
if err != nil {
return err
}
token, err := createAPISession(client, *baseURL, "qa_"+suffix, password)
if err != nil {
return err
}
chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID)
if err != nil {
return err
}
fixture, err := pngFixture(320, 200)
if err != nil {
return err
}
var attachment model.Attachment
response, err := apiRequest(
client, http.MethodPost,
*baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png",
token, "image/png", fixture, &attachment,
)
if err != nil {
return err
}
if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued {
return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status)
}
messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}})
if err != nil {
return err
}
response, err = apiRequestWithHeaders(
client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages",
token, "application/json", messageBody, nil,
map[string]string{"Idempotency-Key": uuid.NewString()},
)
if err != nil {
return err
}
if response.StatusCode != http.StatusCreated {
return fmt.Errorf("link queued attachment returned %d", response.StatusCode)
}
response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil)
if err != nil {
return err
}
if response.StatusCode != http.StatusConflict {
return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode)
}
return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID})
}
func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error {
flags := flag.NewFlagSet("api-verify", flag.ContinueOnError)
flags.SetOutput(io.Discard)
waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]")
}
var state apiState
if err := json.NewDecoder(input).Decode(&state); err != nil {
return fmt.Errorf("decode API scenario state: %w", err)
}
client := &http.Client{Timeout: 10 * time.Second}
ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout)
defer cancel()
var attachment model.Attachment
for {
response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK {
return fmt.Errorf("attachment metadata returned %d", response.StatusCode)
}
if attachment.Status == model.AttachmentReady {
break
}
if attachment.Status == model.AttachmentFailed {
return fmt.Errorf("attachment failed with %v", attachment.ErrorCode)
}
if err := wait(ctx, 250*time.Millisecond); err != nil {
return err
}
}
if len(attachment.Variants) != 3 {
return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants))
}
response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token)
if err != nil {
return err
}
fixture, err := pngFixture(320, 200)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) {
return errors.New("original changed after worker and broker restart")
}
response, thumbnail, err := apiBytes(
client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token,
)
if err != nil {
return err
}
if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" {
return errors.New("thumbnail response is invalid")
}
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail))
if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 {
return errors.New("thumbnail dimensions are invalid")
}
return json.NewEncoder(output).Encode(map[string]any{
"accepted_while_worker_stopped": true,
"accepted_before_outbox_publish": true,
"linked_while_queued": true,
"content_conflict_while_queued": true,
"ready_after_api_broker_restart": true,
"original_preserved": true,
"thumbnail_dimensions": "256x160",
})
}
func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) {
body, err := json.Marshal(map[string]string{
"username": username, "display_name": username, "password": password,
})
if err != nil {
return apiUser{}, err
}
var user apiUser
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user)
if err != nil {
return apiUser{}, err
}
if response.StatusCode != http.StatusCreated {
return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode)
}
return user, nil
}
func createAPISession(client *http.Client, baseURL, username, password string) (string, error) {
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil)
if err != nil {
return "", err
}
request.Header.Set("Authorization", "Basic "+credentials)
response, err := client.Do(request)
if err != nil {
return "", err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return "", fmt.Errorf("create session returned %d", response.StatusCode)
}
var session apiSession
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
return "", err
}
return session.Token, nil
}
func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) {
body, err := json.Marshal(map[string]any{
"title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID},
})
if err != nil {
return apiChat{}, err
}
var chat apiChat
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat)
if err != nil {
return apiChat{}, err
}
if response.StatusCode != http.StatusCreated {
return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode)
}
return chat, nil
}
func apiRequest(
client *http.Client,
method string,
requestURL string,
token string,
contentType string,
body []byte,
result any,
) (*http.Response, error) {
return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil)
}
func apiRequestWithHeaders(
client *http.Client,
method string,
requestURL string,
token string,
contentType string,
body []byte,
result any,
headers map[string]string,
) (*http.Response, error) {
request, err := http.NewRequest(method, requestURL, bytes.NewReader(body))
if err != nil {
return nil, err
}
if token != "" {
request.Header.Set("Authorization", "Bearer "+token)
}
if contentType != "" {
request.Header.Set("Content-Type", contentType)
}
for name, value := range headers {
request.Header.Set(name, value)
}
response, err := client.Do(request)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if result != nil && response.StatusCode < http.StatusBadRequest {
if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil {
return nil, err
}
}
return response, nil
}
func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) {
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
return nil, nil, err
}
request.Header.Set("Authorization", "Bearer "+token)
response, err := client.Do(request)
if err != nil {
return nil, nil, err
}
defer func() { _ = response.Body.Close() }()
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
return response, body, err
}
+605
View File
@@ -0,0 +1,605 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"os"
"strings"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
)
const (
experimentLease = 2 * time.Second
maxAttempts = 5
)
type result struct {
StartedAt string `json:"started_at"`
DurationMillis int64 `json:"duration_ms"`
PipelineVersion int `json:"pipeline_version"`
FaultScenarios []scenario `json:"fault_scenarios"`
DuplicateDeliveries int `json:"duplicate_deliveries"`
V1ObjectCount int `json:"v1_object_count"`
V2ObjectCount int `json:"v2_object_count"`
AcceptedJobs int `json:"accepted_jobs"`
TerminalJobs int `json:"terminal_jobs"`
ReadyJobs int `json:"ready_jobs"`
FailedJobs int `json:"failed_jobs"`
PermanentErrorCode string `json:"permanent_error_code"`
ReplayAttempt int `json:"replay_attempt"`
InvariantResults map[string]bool `json:"invariant_results"`
}
type scenario struct {
Name string `json:"name"`
AttemptAfterRecovery int `json:"attempt_after_recovery"`
ObjectsBeforeRecovery int `json:"objects_before_recovery"`
ObjectsAfterRecovery int `json:"objects_after_recovery"`
FinalStatus string `json:"final_status"`
}
type experiment struct {
settings config.WorkerConfig
repository *postgres.Store
objects *objectstore.Store
broker *redisstream.Broker
dispatcher *outbox.Dispatcher
worker *worker.Worker
ownerID string
chatID string
fixture []byte
}
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(arguments []string, output io.Writer) error {
if len(arguments) > 0 {
switch arguments[0] {
case "api-produce":
return runAPIProduce(arguments[1:], output)
case "api-verify":
return runAPIVerify(arguments[1:], os.Stdin, output)
}
}
flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError)
flags.SetOutput(io.Discard)
timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 {
return errors.New("usage: lab5experiment [-timeout 2m]")
}
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
experiment, closeExperiment, err := newExperiment(ctx)
if err != nil {
return err
}
defer closeExperiment()
report, err := experiment.run(ctx)
if err != nil {
return err
}
report.StartedAt = started.UTC().Format(time.RFC3339)
report.DurationMillis = time.Since(started).Milliseconds()
encoder := json.NewEncoder(output)
encoder.SetIndent("", " ")
return encoder.Encode(report)
}
func newExperiment(ctx context.Context) (*experiment, func(), error) {
settings, err := config.LoadWorker()
if err != nil {
return nil, nil, err
}
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return nil, nil, err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
pool.Close()
return nil, nil, err
}
redisOptions, err := redis.ParseURL(settings.BrokerURL)
if err != nil {
pool.Close()
return nil, nil, err
}
client := redis.NewClient(redisOptions)
if err := client.Ping(ctx).Err(); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
consumer := "lab5experiment-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second,
ClaimMinIdle: experimentLease, ReadCount: 32,
})
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
if err := broker.Ensure(ctx); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
repository := postgres.New(pool)
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
})
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
processorWorker, err := worker.New(
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
worker.Config{
WorkerID: consumer, Concurrency: 1,
LeaseDuration: experimentLease, ProcessingTimeout: time.Second,
ReclaimInterval: experimentLease,
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
},
)
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
fixture, err := pngFixture(320, 200)
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
value := &experiment{
settings: settings, repository: repository, objects: objects, broker: broker,
dispatcher: dispatcher, worker: processorWorker, fixture: fixture,
}
if err := value.seed(ctx); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
closeExperiment := func() {
pool.Close()
_ = client.Close()
}
return value, closeExperiment, nil
}
func (e *experiment) seed(ctx context.Context) error {
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment")
if err != nil {
return err
}
member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment")
if err != nil {
return err
}
chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID)
if err != nil {
return err
}
e.ownerID = owner.ID
e.chatID = chat.ID
return nil
}
func (e *experiment) run(ctx context.Context) (result, error) {
report := result{
PipelineVersion: 1,
InvariantResults: map[string]bool{
"accepted_jobs_reach_terminal_state": false,
"duplicate_effect_is_unique": false,
"old_pipeline_cannot_replace_new": false,
},
}
for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} {
observed, err := e.runLeaseRecovery(ctx, fault)
if err != nil {
return result{}, err
}
report.FaultScenarios = append(report.FaultScenarios, observed)
report.AcceptedJobs++
if observed.FinalStatus == model.AttachmentReady {
report.ReadyJobs++
report.TerminalJobs++
}
}
permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx)
if err != nil {
return result{}, err
}
report.AcceptedJobs++
report.TerminalJobs++
report.FailedJobs++
report.PermanentErrorCode = permanentCode
report.ReplayAttempt = replayAttempt
commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx)
if err != nil {
return result{}, err
}
report.FaultScenarios = append(report.FaultScenarios, commitScenario)
report.AcceptedJobs++
if commitScenario.FinalStatus == model.AttachmentReady {
report.ReadyJobs++
report.TerminalJobs++
}
payload, err := redisstream.EncodeEnvelope(envelope)
if err != nil {
return result{}, err
}
for range 2 {
if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil {
return result{}, err
}
}
duplicateDeliveries, err := e.readDeliveries(ctx, 2)
if err != nil {
return result{}, err
}
for _, delivery := range duplicateDeliveries {
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
return result{}, err
}
}
report.DuplicateDeliveries = len(duplicateDeliveries)
report.V1ObjectCount, err = e.objectCount(ctx, attachmentID)
if err != nil {
return result{}, err
}
if report.V1ObjectCount != 4 {
return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount)
}
report.InvariantResults["duplicate_effect_is_unique"] = true
oldPayload := payload
v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1}
if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil {
return result{}, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return result{}, err
}
v2Delivery, err := e.readDelivery(ctx)
if err != nil {
return result{}, err
}
if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil {
return result{}, err
}
if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil {
return result{}, err
}
staleDelivery, err := e.readDelivery(ctx)
if err != nil {
return result{}, err
}
if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil {
return result{}, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return result{}, err
}
if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 {
return result{}, errors.New("new pipeline version was not preserved after stale delivery")
}
report.V2ObjectCount, err = e.objectCount(ctx, attachmentID)
if err != nil {
return result{}, err
}
if report.V2ObjectCount != 8 {
return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount)
}
report.InvariantResults["old_pipeline_cannot_replace_new"] = true
report.AcceptedJobs++
report.ReadyJobs++
report.TerminalJobs++
report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs
return report, nil
}
func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) {
attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png")
if err != nil {
return "", 0, err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return "", 0, err
}
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
return "", 0, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return "", 0, err
}
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil {
return "", 0, errors.New("corrupted staging object did not reach failed")
}
firstCode := *metadata.ErrorCode
replayed, err := e.repository.ReplayFailedAttachment(
ctx, attachmentID, "lab5experiment", "verify explicit replay fencing",
)
if err != nil {
return "", 0, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return "", 0, err
}
replayDelivery, err := e.readDelivery(ctx)
if err != nil {
return "", 0, err
}
if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil {
return "", 0, err
}
metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return "", 0, err
}
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode {
return "", 0, errors.New("replayed permanent failure changed terminal classification")
}
return firstCode, replayed.Attempt, nil
}
func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) {
attachmentID, _, err := e.queueAttachment(ctx)
if err != nil {
return scenario{}, err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, err
}
if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) {
return scenario{}, fmt.Errorf("fault %s returned %v", fault, err)
}
before, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, err
}
if fault == worker.FaultBeforeSideEffect && before != 0 {
return scenario{}, errors.New("objects were written before the injected side effect")
}
if fault == worker.FaultAfterSideEffect && before != 4 {
return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before)
}
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
return scenario{}, err
}
recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now())
if err != nil {
return scenario{}, err
}
if recovered.Requeued != 1 {
return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued)
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return scenario{}, err
}
retry, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, err
}
if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil {
return scenario{}, err
}
if err := e.ackReclaimed(ctx); err != nil {
return scenario{}, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return scenario{}, err
}
after, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, err
}
job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String())
if err != nil {
return scenario{}, err
}
return scenario{
Name: string(fault), AttemptAfterRecovery: job.Attempt,
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
}, nil
}
func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) {
attachmentID, _, err := e.queueAttachment(ctx)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) {
return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err)
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
before, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := e.ackReclaimed(ctx); err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
after, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
return scenario{
Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt,
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
}, delivery.Envelope, attachmentID, nil
}
func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) {
attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png")
return attachmentID, job, err
}
func (e *experiment) queueAttachmentData(
ctx context.Context,
data []byte,
contentType string,
) (string, string, model.AttachmentJob, error) {
attachmentID := uuid.NewString()
stagingKey := "staging/" + attachmentID + "/experiment"
digest := sha256.Sum256(data)
_, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{
ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png",
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1,
})
if err != nil {
return "", "", model.AttachmentJob{}, err
}
if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil {
return "", "", model.AttachmentJob{}, err
}
job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1}
if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil {
return "", "", model.AttachmentJob{}, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return "", "", model.AttachmentJob{}, err
}
return attachmentID, stagingKey, job, nil
}
func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) {
deliveries, err := e.readDeliveries(ctx, 1)
if err != nil {
return redisstream.Delivery{}, err
}
return deliveries[0], nil
}
func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) {
var result []redisstream.Delivery
for len(result) < expected {
deliveries, err := e.broker.Read(ctx)
if err != nil {
return nil, err
}
result = append(result, deliveries...)
}
if len(result) != expected {
return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected)
}
return result, nil
}
func (e *experiment) ackReclaimed(ctx context.Context) error {
deliveries, _, err := e.broker.Reclaim(ctx, "0-0")
if err != nil {
return err
}
if len(deliveries) != 1 {
return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries))
}
return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone)
}
func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) {
objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100)
if err != nil {
return 0, err
}
if next != "" {
return 0, errors.New("unexpected object pagination")
}
return len(objects), nil
}
func pngFixture(width, height int) ([]byte, error) {
value := image.NewNRGBA(image.Rect(0, 0, width, height))
for y := range height {
for x := range width {
value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
}
}
var output bytes.Buffer
if err := png.Encode(&output, value); err != nil {
return nil, err
}
return output.Bytes(), nil
}
func wait(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
+42
View File
@@ -0,0 +1,42 @@
{
"started_at": "2026-10-22T14:19:25Z",
"duration_ms": 6981,
"pipeline_version": 1,
"fault_scenarios": [
{
"name": "before_side_effect",
"attempt_after_recovery": 2,
"objects_before_recovery": 0,
"objects_after_recovery": 4,
"final_status": "ready"
},
{
"name": "after_side_effect",
"attempt_after_recovery": 2,
"objects_before_recovery": 4,
"objects_after_recovery": 4,
"final_status": "ready"
},
{
"name": "after_commit",
"attempt_after_recovery": 1,
"objects_before_recovery": 4,
"objects_after_recovery": 4,
"final_status": "ready"
}
],
"duplicate_deliveries": 2,
"v1_object_count": 4,
"v2_object_count": 8,
"accepted_jobs": 5,
"terminal_jobs": 5,
"ready_jobs": 4,
"failed_jobs": 1,
"permanent_error_code": "invalid_image",
"replay_attempt": 2,
"invariant_results": {
"accepted_jobs_reach_terminal_state": true,
"duplicate_effect_is_unique": true,
"old_pipeline_cannot_replace_new": true
}
}
+1
View File
@@ -0,0 +1 @@
{"accepted_before_outbox_publish":true,"accepted_while_worker_stopped":true,"content_conflict_while_queued":true,"linked_while_queued":true,"original_preserved":true,"ready_after_api_broker_restart":true,"thumbnail_dimensions":"256x160"}
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
state_file=$(mktemp)
restore_pipeline() {
rm -f "$state_file"
docker compose up -d dispatcher worker >/dev/null
}
trap restore_pipeline EXIT INT TERM
docker compose stop worker dispatcher >/dev/null
docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file"
docker compose restart api redis-broker >/dev/null
docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null
docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file"
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
restore_pipeline() {
docker compose up -d dispatcher worker >/dev/null
}
trap restore_pipeline EXIT INT TERM
docker compose stop worker dispatcher >/dev/null
docker compose --profile experiments run --rm lab5-experiment