diff --git a/.env.example b/.env.example index cb25c265..700a36b4 100644 --- a/.env.example +++ b/.env.example @@ -6,9 +6,8 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# Sessions are stored in Valkey on this stand. +SESSION_STORE_URL_FILE=/run/secrets/session_store_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 S3_ENDPOINT=http://s3:8333 diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index 936f2ec4..4b763da0 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -30,8 +30,11 @@ jobs: with: python-version: '3.12' - run: make secrets + - run: chmod +x scripts/restart.sh scripts/stop-one.sh - run: docker compose up -d --build --wait - run: mkdir -p evidence-ci + - name: Image codecs in the worker runtime + run: make codec-test | tee evidence-ci/codec.txt - name: Public smoke run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt - name: Persistence across a restart of the API and both stores diff --git a/Dockerfile b/Dockerfile index a3e94d88..02a6a856 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,17 @@ FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ -RUN go mod download && go mod verify +RUN --mount=type=cache,target=/go/pkg/mod go mod download && go mod verify COPY cmd ./cmd COPY internal ./internal COPY migrations ./migrations -RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/worker ./cmd/migrate ./cmd/reconcile ./cmd/s3init # The load generator gets an image of its own; the API image does not carry it. FROM build AS loadgen-build COPY tools ./tools -RUN --mount=type=cache,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \ CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen @@ -22,6 +22,12 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen USER app ENTRYPOINT ["/usr/local/bin/loadgen"] +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS worker +RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp && adduser -D -H -u 10001 app +COPY --from=build /out/worker /usr/local/bin/worker +USER app +ENTRYPOINT ["/usr/local/bin/worker"] + # Last, so that `build: .` builds it; it also runs the migrate service. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app diff --git a/Makefile b/Makefile index 96ccf732..d54d0197 100644 --- a/Makefile +++ b/Makefile @@ -110,3 +110,14 @@ concurrency: explain: docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql + +# Set CODEC_STRESS=1 after generating bin/decoder-fixtures to include the 40 MP cases. +.PHONY: codec-test +codec-test: export GOOS = linux +codec-test: export GOARCH = $(shell docker version --format "{{.Server.Arch}}") +codec-test: export CGO_ENABLED = 0 +codec-test: export MSYS_NO_PATHCONV = 1 +codec-test: + go test -c -o bin/imaging.test ./internal/imaging + docker build --target worker -t mtusi-messenger-worker:lab5 . + docker run --rm --network none --cpus 2 --memory 2g --entrypoint /codec/imaging.test -e CODEC_REQUIRED=1 $(if $(filter 1,$(CODEC_STRESS)),-e CODEC_FIXTURES=/codec/decoder-fixtures,) --mount "type=bind,source=$(CURDIR)/bin,target=/codec,readonly" mtusi-messenger-worker:lab5 -test.v diff --git a/cmd/server/main.go b/cmd/server/main.go index 4a3c85b8..90904146 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -19,11 +19,13 @@ import ( "messenger/internal/app" "messenger/internal/attachment" "messenger/internal/chat" + "messenger/internal/pipeline" "messenger/internal/platform/config" "messenger/internal/platform/ident" "messenger/internal/platform/postgres" "messenger/internal/platform/s3store" "messenger/internal/platform/valkey" + "messenger/internal/processing" "messenger/internal/server" ) @@ -85,10 +87,25 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) IdempotencyTTL: cfg.IdempotencyTTL, Blobs: blobs, AttachmentsCfg: attachment.Config{ - MaxBytes: cfg.S3.MaxBytes, - MaxPixels: cfg.S3.MaxPixels, + BarrierDir: os.Getenv("LAB5_BARRIERS"), + MaxBytes: cfg.S3.MaxBytes, + MaxPixels: cfg.S3.MaxPixels, }, }) + var jobs config.Jobs + var barrier func(context.Context, pipeline.Job, string) error + if dir := os.Getenv("LAB5_BARRIERS"); dir != "" { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + barrier = processing.FileBarrier(dir) + } + if blobs != nil { + jobs, err = config.LoadJobs(os.LookupEnv) + if err != nil { + return err + } + } var debugLn net.Listener if cfg.DebugAddr != "" { if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil { @@ -99,6 +116,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string) ctx, cancel := context.WithCancel(ctx) var background sync.WaitGroup + if blobs != nil { + background.Go(func() { processing.Dispatch(ctx, log, pipeline.NewStore(pool), jobs, barrier) }) + } background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) }) if reconciler != nil { background.Go(func() { @@ -151,7 +171,7 @@ func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Looku if err != nil { return nil, nil, err } - reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour) + reconciler, err := attachment.NewReconcilerWithRetention(pool, reconBlobs, time.Hour, reconCfg.StagingRetention) if err != nil { return nil, nil, err } diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 00000000..a27cb0b2 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,155 @@ +// Command worker runs image consumers or an explicit operator action. +package main + +import ( + "context" + "encoding/json" + "errors" + + "log/slog" + "os" + "os/signal" + + "strconv" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/broker" + "messenger/internal/imaging" + "messenger/internal/pipeline" + "messenger/internal/platform/config" + "messenger/internal/platform/postgres" + "messenger/internal/platform/s3store" + "messenger/internal/processing" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + if err := run(ctx, os.Args[1:]); err != nil { + slog.Error("worker stopped", "error", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) error { + if len(args) == 1 && args[0] == "--prepare-large-png" { + return imaging.PrepareLargePNG(os.Stdin) + } + db, err := config.LoadDatabaseURL(os.LookupEnv) + if err != nil { + return err + } + cfg, err := config.LoadJobs(os.LookupEnv) + if err != nil { + return err + } + s3, err := config.LoadS3(os.LookupEnv) + if err != nil { + return err + } + blobs, err := s3store.New(s3) + if err != nil { + return err + } + pool, err := postgres.NewPool(ctx, db, 5) + if err != nil { + return err + } + defer pool.Close() + store := pipeline.NewStore(pool) + w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()} + if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") { + dir := strings.TrimPrefix(args[0], "--barriers=") + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + w.Checkpoint = processing.FileBarrier(dir) + args = nil + } + if len(args) == 0 { + w.Run(ctx) + return nil + } + op, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + if len(args) == 1 && args[0] == "failed" { + jobs, err := store.Failed(op) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + const usage = "use failed, replay , reprocess , or duplicate " + if len(args) < 3 { + return errors.New(usage) + } + id, err := uuid.Parse(args[1]) + if err != nil { + return errors.New("invalid attachment UUID") + } + exists := func(ctx context.Context, key string) (bool, error) { + _, err := blobs.Stat(ctx, key) + if errors.Is(err, attachment.ErrObjectMissing) { + return false, nil + } + return err == nil, err + } + switch args[0] { + case "replay": + if len(args) != 3 { + return errors.New(usage) + } + return store.Replay(op, id, args[2], exists) + case "reprocess": + if len(args) != 3 { + return errors.New(usage) + } + v, err := strconv.Atoi(args[2]) + if err != nil { + return err + } + return store.Reprocess(op, id, v, exists) + case "duplicate": + if len(args) != 4 { + return errors.New(usage) + } + version, err := strconv.Atoi(args[2]) + if err != nil { + return err + } + attempt, err := strconv.Atoi(args[3]) + if err != nil { + return err + } + return duplicate(op, pool, cfg.URL, id, version, attempt) + default: + return errors.New(usage) + } +} + +// duplicate sends a previously issued envelope through the actual broker. It +// cannot change SQL state or manufacture a future attempt. +func duplicate(ctx context.Context, pool *pgxpool.Pool, url string, id uuid.UUID, version, attempt int) error { + j := pipeline.Job{SchemaVersion: 1, Type: "attachment.process.v1", AttachmentID: id, PipelineVersion: version, Attempt: attempt} + err := pool.QueryRow(ctx, `SELECT job_id,created_at FROM attachment_jobs + WHERE attachment_id=$1 AND pipeline_version=$2 AND $3::integer BETWEEN 1 AND attempt`, id, version, attempt).Scan(&j.JobID, &j.CreatedAt) + if err != nil { + return err + } + p, err := broker.NewPublisher(ctx, url) + if err != nil { + return err + } + defer p.Close() + body, err := json.Marshal(j) + if err != nil { + return err + } + return p.Publish(ctx, body) +} diff --git a/compose.yaml b/compose.yaml index 1181f6a5..8c4dddbf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -39,6 +39,9 @@ services: environment: DATABASE_URL_FILE: /run/secrets/database_url SESSION_STORE_URL_FILE: /run/secrets/session_store_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} @@ -55,6 +58,7 @@ services: secrets: - database_url - session_store_url + - broker_url - s3_access_key_id - s3_secret_access_key - s3_reconcile_access_key_id @@ -80,6 +84,55 @@ services: cpus: "1.0" memory: 2G + broker: + image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 + hostname: broker + secrets: + - source: rabbitmq_config + target: /etc/rabbitmq/conf.d/90-lab.conf + volumes: + - rabbitdata:/var/lib/rabbitmq + networks: [data] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 12 + deploy: + resources: + limits: + cpus: "1" + memory: 512M + + worker: + build: + context: . + target: worker + image: mtusi-messenger-worker:lab5 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "2" + memory: 2G + migrate: image: mtusi-messenger:lab5 entrypoint: ["/usr/local/bin/migrate"] @@ -222,6 +275,7 @@ services: image: mtusi-messenger:lab5 entrypoint: ["/usr/local/bin/reconcile"] environment: + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} DATABASE_URL_FILE: /run/secrets/database_url S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} S3_REGION: ${S3_REGION:-us-east-1} @@ -252,6 +306,7 @@ services: memory: 128M volumes: + rabbitdata: pgdata: valkeydata: s3data: @@ -262,6 +317,10 @@ networks: internal: true secrets: + broker_url: + file: ./secrets/broker_url + rabbitmq_config: + file: ./secrets/rabbitmq.conf database_url: file: ./secrets/database_url session_store_url: diff --git a/go.mod b/go.mod index 2f297f40..46837d52 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/minio/minio-go/v7 v7.4.1 github.com/pressly/goose/v3 v3.27.0 + github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.23.0 github.com/testcontainers/testcontainers-go v0.39.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 diff --git a/go.sum b/go.sum index 9807f1b1..96bcfa2d 100644 --- a/go.sum +++ b/go.sum @@ -171,6 +171,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/internal/broker/rabbit.go b/internal/broker/rabbit.go new file mode 100644 index 00000000..e4cf9f38 --- /dev/null +++ b/internal/broker/rabbit.go @@ -0,0 +1,115 @@ +// Package broker owns the AMQP connection for one publisher or consumer. +package broker + +import ( + "context" + "errors" + "net" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const queue = "attachment.jobs" + +type link struct { + conn *amqp.Connection + channel *amqp.Channel +} + +func open(ctx context.Context, url string) (*link, error) { + conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 5 * time.Second, Dial: func(network, addr string) (net.Conn, error) { + d := net.Dialer{Timeout: 3 * time.Second} + c, err := d.DialContext(ctx, network, addr) + if err == nil { + err = c.SetDeadline(time.Now().Add(5 * time.Second)) + } + return c, err + }}) + // Parse errors can contain the complete connection URL. + if err != nil { + return nil, errors.New("RabbitMQ connection unavailable") + } + l := &link{conn: conn} + l.channel, err = conn.Channel() + if err == nil { + // PostgreSQL limits processing attempts. A broker redelivery can occur + // before PostgreSQL is reachable and must leave the job recoverable. + _, err = l.channel.QueueDeclare(queue, true, false, false, false, amqp.Table{ + "x-queue-type": "quorum", "x-delivery-limit": int32(-1), + }) + } + if err != nil { + l.Close() + return nil, err + } + return l, nil +} + +func (l *link) Close() { _ = l.conn.CloseDeadline(time.Now().Add(time.Second)) } + +type Publisher struct { + *link + returned <-chan amqp.Return +} + +func NewPublisher(ctx context.Context, url string) (*Publisher, error) { + l, err := open(ctx, url) + if err != nil { + return nil, err + } + if err = l.channel.Confirm(false); err != nil { + l.Close() + return nil, err + } + return &Publisher{l, l.channel.NotifyReturn(make(chan amqp.Return, 1))}, nil +} + +// Publish has one in-flight confirm on its channel. An ambiguous result means +// the caller discards this publisher and leaves the outbox row pending. +func (p *Publisher) Publish(ctx context.Context, body []byte) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + stop := context.AfterFunc(ctx, p.Close) + defer stop() + confirmation, err := p.channel.PublishWithDeferredConfirmWithContext(ctx, "", queue, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body}) + if err != nil { + return err + } + ok, err := confirmation.WaitContext(ctx) + if err != nil { + return err + } + if !ok { + return errors.New("broker did not confirm publication") + } + // RabbitMQ sends basic.return before the associated publisher confirm. + select { + case <-p.returned: + return errors.New("broker returned publication") + default: + return nil + } +} + +type Consumer struct { + *link + Deliveries <-chan amqp.Delivery +} + +func NewConsumer(ctx context.Context, url string) (*Consumer, error) { + l, err := open(ctx, url) + if err != nil { + return nil, err + } + if err = l.channel.Qos(1, 0, false); err != nil { + l.Close() + return nil, err + } + d, err := l.channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil) + if err != nil { + l.Close() + return nil, err + } + return &Consumer{l, d}, nil +} diff --git a/internal/broker/rabbit_test.go b/internal/broker/rabbit_test.go new file mode 100644 index 00000000..7143d9d8 --- /dev/null +++ b/internal/broker/rabbit_test.go @@ -0,0 +1,56 @@ +package broker + +import ( + "context" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func TestConfirmAndRedeliverUnacked(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + c, err := testcontainers.Run(ctx, "rabbitmq:4.1.4-alpine", testcontainers.WithEnv(map[string]string{"RABBITMQ_DEFAULT_USER": "lab", "RABBITMQ_DEFAULT_PASS": "test-only"}), testcontainers.WithExposedPorts("5672/tcp"), testcontainers.WithWaitStrategy(wait.ForLog("Server startup complete"))) + if err != nil { + t.Fatal(err) + } + testcontainers.CleanupContainer(t, c) + endpoint, err := c.PortEndpoint(ctx, "5672/tcp", "") + if err != nil { + t.Fatal(err) + } + url := "amqp://lab:test-only@" + endpoint + "/" + pub, err := NewPublisher(ctx, url) + if err != nil { + t.Fatal(err) + } + defer pub.Close() + if err = pub.Publish(ctx, []byte("work")); err != nil { + t.Fatal(err) + } + // Broker redeliveries while PostgreSQL is unreachable do not consume + // the application budget. Exercise more than the quorum queue default. + for attempt := 0; attempt <= 22; attempt++ { + consumer, err := NewConsumer(ctx, url) + if err != nil { + t.Fatal(err) + } + select { + case msg := <-consumer.Deliveries: + if string(msg.Body) != "work" || msg.Redelivered != (attempt > 0) { + t.Fatalf("unexpected delivery on attempt %d", attempt) + } + if attempt == 22 { + if err := msg.Ack(false); err != nil { + t.Fatal(err) + } + } + case <-time.After(5 * time.Second): + consumer.Close() + t.Fatalf("confirmed job missing on delivery %d", attempt+1) + } + consumer.Close() + } +} diff --git a/internal/platform/config/jobs.go b/internal/platform/config/jobs.go new file mode 100644 index 00000000..8b497f04 --- /dev/null +++ b/internal/platform/config/jobs.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "strconv" + "time" +) + +type Jobs struct { + URL string + Concurrency, MaxAttempts int + Timeout time.Duration +} + +func LoadJobs(lookup Lookup) (Jobs, error) { + url, err := secret(lookup, "BROKER_URL_FILE") + if err != nil { + return Jobs{}, err + } + c := Jobs{URL: url, Concurrency: 2, MaxAttempts: 5} + seconds := 60 + for _, p := range []struct { + key string + value *int + max int + }{ + {"WORKER_CONCURRENCY", &c.Concurrency, 2}, + {"JOB_MAX_ATTEMPTS", &c.MaxAttempts, 100}, + {"JOB_TIMEOUT_SECONDS", &seconds, 3600}, + } { + if s, ok := lookup(p.key); ok { + n, err := strconv.Atoi(s) + if err != nil || n < 1 || n > p.max { + return Jobs{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max) + } + *p.value = n + } + } + c.Timeout = time.Duration(seconds) * time.Second + return c, nil +} diff --git a/internal/platform/config/jobs_test.go b/internal/platform/config/jobs_test.go new file mode 100644 index 00000000..0f5d2a82 --- /dev/null +++ b/internal/platform/config/jobs_test.go @@ -0,0 +1,28 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestLoadJobs(t *testing.T) { + path := filepath.Join(t.TempDir(), "url") + if err := os.WriteFile(path, []byte("amqp://worker:secret@broker:5672/\n"), 0600); err != nil { + t.Fatal(err) + } + values := map[string]string{"BROKER_URL_FILE": path} + lookup := func(k string) (string, bool) { v, ok := values[k]; return v, ok } + cfg, err := LoadJobs(lookup) + if err != nil || cfg.Timeout != 60*time.Second || cfg.MaxAttempts != 5 || cfg.Concurrency < 1 || cfg.URL != "amqp://worker:secret@broker:5672/" { + t.Fatalf("unexpected defaults or secret read, err=%v", err) + } + for _, key := range []string{"JOB_TIMEOUT_SECONDS", "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS"} { + values[key] = "0" + if _, err := LoadJobs(lookup); err == nil { + t.Errorf("%s accepted zero", key) + } + delete(values, key) + } +} diff --git a/internal/processing/barrier.go b/internal/processing/barrier.go new file mode 100644 index 00000000..5b8155e6 --- /dev/null +++ b/internal/processing/barrier.go @@ -0,0 +1,18 @@ +package processing + +import ( + "context" + "fmt" + "path/filepath" + + "messenger/internal/pipeline" + "messenger/internal/platform/faultbarrier" +) + +// Only the explicit fault experiment enables these one-shot barriers. +func FileBarrier(dir string) func(context.Context, pipeline.Job, string) error { + return func(ctx context.Context, j pipeline.Job, phase string) error { + base := filepath.Join(dir, fmt.Sprintf("%s-v%d-%s", j.AttachmentID, j.PipelineVersion, phase)) + return faultbarrier.Wait(ctx, base, "") + } +} diff --git a/internal/processing/dispatcher.go b/internal/processing/dispatcher.go new file mode 100644 index 00000000..376655ec --- /dev/null +++ b/internal/processing/dispatcher.go @@ -0,0 +1,70 @@ +package processing + +import ( + "context" + "encoding/json" + "log/slog" + "time" + + "messenger/internal/broker" + "messenger/internal/pipeline" + "messenger/internal/platform/config" +) + +func retryPolicy(cfg config.Jobs) pipeline.RetryPolicy { + return pipeline.RetryPolicy{MaxAttempts: cfg.MaxAttempts, BaseDelay: time.Second, MaxDelay: time.Minute} +} + +func pause(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) { + var p *broker.Publisher + defer func() { + if p != nil { + p.Close() + } + }() + for ctx.Err() == nil { + _, err := store.RecoverOne(ctx, retryPolicy(cfg)) + if err == nil && p == nil { + p, err = broker.NewPublisher(ctx, cfg.URL) + } + if err == nil { + var sent bool + sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error { + body, err := json.Marshal(j) + if err != nil { + return err + } + if err := p.Publish(ctx, body); err != nil { + return err + } + if barrier != nil { + return barrier(ctx, j, "published") + } + return nil + }) + if err == nil { + if !sent { + pause(ctx, 250*time.Millisecond) + } + continue + } + } + if p != nil { + p.Close() + p = nil + } + if ctx.Err() == nil { + log.Warn("attachment dispatcher will retry", "error", err) + } + pause(ctx, time.Second) + } +} diff --git a/internal/processing/worker.go b/internal/processing/worker.go new file mode 100644 index 00000000..56544598 --- /dev/null +++ b/internal/processing/worker.go @@ -0,0 +1,176 @@ +package processing + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "io" + "log/slog" + "sync" + "time" + + "messenger/internal/attachment" + "messenger/internal/broker" + "messenger/internal/imaging" + "messenger/internal/pipeline" + "messenger/internal/platform/config" +) + +var errSourceMismatch = errors.New("original differs from accepted metadata") + +type Worker struct { + Store *pipeline.Store + Blobs attachment.Blobs + Config config.Jobs + Log *slog.Logger + // An experiment may pause an individual attempt at an exact durability boundary. + Checkpoint func(context.Context, pipeline.Job, string) error +} + +func (w *Worker) Run(ctx context.Context) { + var wg sync.WaitGroup + for range w.Config.Concurrency { + wg.Go(func() { + for ctx.Err() == nil { + err := w.consume(ctx) + if ctx.Err() == nil { + w.Log.Warn("consumer reconnecting", "error", err) + pause(ctx, time.Second) + } + } + }) + } + wg.Wait() +} + +func (w *Worker) consume(ctx context.Context) error { + c, err := broker.NewConsumer(ctx, w.Config.URL) + if err != nil { + return err + } + defer c.Close() + for d := range c.Deliveries { + var j pipeline.Job + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF { + w.Log.Warn("discarding invalid attachment envelope") + if err := d.Reject(false); err != nil { + return err + } + continue + } + if err := w.process(ctx, j); err != nil { + return err + } + if err := d.Ack(false); err != nil { + return err + } + if err := w.checkpoint(ctx, j, "acked"); err != nil { + return err + } + } + return errors.New("broker delivery stream closed") +} + +func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error { + if w.Checkpoint != nil { + return w.Checkpoint(ctx, j, phase) + } + return nil +} + +func (w *Worker) process(ctx context.Context, j pipeline.Job) error { + c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second) + if err != nil { + return err + } + if c == nil { + return w.checkpoint(ctx, j, "duplicate") + } + if err := w.checkpoint(ctx, j, "claimed"); err != nil { + return err + } + jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout) + defer cancel() + variants, err := w.writeObjects(jobCtx, c) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + code := "processing_failed" + permanent := false + switch { + case errors.Is(err, imaging.ErrInvalidImage): + code, permanent = "invalid_image", true + case errors.Is(err, errSourceMismatch): + code, permanent = "source_mismatch", true + case errors.Is(err, attachment.ErrObjectMissing): + code = "source_missing" + } + w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code) + _, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config)) + return err + } + if err := w.checkpoint(ctx, j, "objects"); err != nil { + return err + } + saved, err := w.Store.Complete(ctx, c, variants) + if err != nil || !saved { + return err + } + return w.checkpoint(ctx, j, "committed") +} + +func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) { + if c.SizeBytes < 1 || c.SizeBytes > 10<<20 { + return nil, errSourceMismatch + } + r, err := blobs.Open(ctx, c.SourceKey) + if err != nil { + return nil, err + } + data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1)) + closeErr := r.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + h := sha256.Sum256(data) + if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) { + return nil, errSourceMismatch + } + return data, nil +} + +func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) { + data, err := readOriginal(ctx, w.Blobs, c) + if err != nil { + return nil, err + } + encoded, err := imaging.Render(ctx, data, c.PipelineVersion) + if err != nil { + return nil, err + } + key := attachment.BlobKey(c.Hash) + if c.SourceKey != key { + if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil { + return nil, err + } + } + var variants []pipeline.Variant + for _, v := range encoded { + hash := sha256.Sum256(v.Data) + key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:]) + n := int64(len(v.Data)) + if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil { + return nil, err + } + variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]}) + } + return variants, nil +} diff --git a/internal/processing/worker_test.go b/internal/processing/worker_test.go new file mode 100644 index 00000000..b021d7aa --- /dev/null +++ b/internal/processing/worker_test.go @@ -0,0 +1,37 @@ +package processing + +import ( + "context" + "crypto/sha256" + "errors" + "io" + "strings" + "testing" + + "messenger/internal/attachment" + "messenger/internal/pipeline" +) + +type stagedSource struct { + attachment.Blobs + body string +} + +func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(s.body)), nil +} + +func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) { + h := sha256.Sum256([]byte("accepted")) + c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"} + for _, bad := range []string{"truncated", "altered!", "oversized bytes"} { + _, err := readOriginal(t.Context(), stagedSource{body: bad}, c) + if !errors.Is(err, errSourceMismatch) { + t.Fatalf("body %q: %v", bad, err) + } + } + got, err := readOriginal(t.Context(), stagedSource{body: "accepted"}, c) + if err != nil || string(got) != "accepted" { + t.Fatalf("valid original: %v", err) + } +} diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh index 081d1023..7bc9f22b 100755 --- a/scripts/init-secrets.sh +++ b/scripts/init-secrets.sh @@ -37,6 +37,15 @@ if [ "$present" -eq 0 ]; then printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url fi +if [ ! -e secrets/rabbitmq.conf ] && [ ! -e secrets/broker_url ]; then + rabbit_password=$(random) + printf 'default_user = messenger\ndefault_pass = %s\n' "$rabbit_password" > secrets/rabbitmq.conf + printf 'amqp://messenger:%s@broker:5672/\n' "$rabbit_password" > secrets/broker_url +elif [ ! -s secrets/rabbitmq.conf ] || [ ! -s secrets/broker_url ]; then + echo "restore rabbitmq.conf and broker_url together" >&2 + exit 1 +fi + # Read just the bucket setting, never execute the local .env file as a script. bucket=${S3_BUCKET:-} if [ -z "$bucket" ] && [ -f .env ]; then diff --git a/scripts/restart.sh b/scripts/restart.sh index bc4cdb3e..0a715fde 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -4,9 +4,7 @@ set -eu cd "$(dirname "$0")/.." docker compose stop -# Re-run one-shot jobs against the retained data before restoring replicas. -docker compose up -d --wait postgres valkey s3 -docker compose run --rm migrate -docker compose run --rm s3-init +# Compose starts both one-shot jobs and waits for their successful exit +# before either API replica or the worker can start. docker compose up -d --wait sh scripts/wait-ready.sh