Подключите RabbitMQ и отдельный обработчик заданий

This commit is contained in:
2026-11-18 15:33:33 +03:00
parent 752f29573b
commit 2dc566e6ec
19 changed files with 818 additions and 14 deletions
+2 -3
View File
@@ -6,9 +6,8 @@ APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
# Sessions are stored in Valkey on this stand.
SESSION_STORE_URL_FILE=/run/secrets/session_store_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:8333
+3
View File
@@ -30,8 +30,11 @@ jobs:
with:
python-version: '3.12'
- run: make secrets
- run: chmod +x scripts/restart.sh scripts/stop-one.sh
- run: docker compose up -d --build --wait
- run: mkdir -p evidence-ci
- name: Image codecs in the worker runtime
run: make codec-test | tee evidence-ci/codec.txt
- name: Public smoke
run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt
- name: Persistence across a restart of the API and both stores
+10 -4
View File
@@ -3,17 +3,17 @@
FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download && go mod verify
RUN --mount=type=cache,target=/go/pkg/mod go mod download && go mod verify
COPY cmd ./cmd
COPY internal ./internal
COPY migrations ./migrations
RUN --mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init
RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/worker ./cmd/migrate ./cmd/reconcile ./cmd/s3init
# The load generator gets an image of its own; the API image does not carry it.
FROM build AS loadgen-build
COPY tools ./tools
RUN --mount=type=cache,target=/root/.cache/go-build \
RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen
@@ -22,6 +22,12 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen
USER app
ENTRYPOINT ["/usr/local/bin/loadgen"]
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS worker
RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp && adduser -D -H -u 10001 app
COPY --from=build /out/worker /usr/local/bin/worker
USER app
ENTRYPOINT ["/usr/local/bin/worker"]
# Last, so that `build: .` builds it; it also runs the migrate service.
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
RUN adduser -D -H -u 10001 app
+11
View File
@@ -110,3 +110,14 @@ concurrency:
explain:
docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql
# Set CODEC_STRESS=1 after generating bin/decoder-fixtures to include the 40 MP cases.
.PHONY: codec-test
codec-test: export GOOS = linux
codec-test: export GOARCH = $(shell docker version --format "{{.Server.Arch}}")
codec-test: export CGO_ENABLED = 0
codec-test: export MSYS_NO_PATHCONV = 1
codec-test:
go test -c -o bin/imaging.test ./internal/imaging
docker build --target worker -t mtusi-messenger-worker:lab5 .
docker run --rm --network none --cpus 2 --memory 2g --entrypoint /codec/imaging.test -e CODEC_REQUIRED=1 $(if $(filter 1,$(CODEC_STRESS)),-e CODEC_FIXTURES=/codec/decoder-fixtures,) --mount "type=bind,source=$(CURDIR)/bin,target=/codec,readonly" mtusi-messenger-worker:lab5 -test.v
+23 -3
View File
@@ -19,11 +19,13 @@ import (
"messenger/internal/app"
"messenger/internal/attachment"
"messenger/internal/chat"
"messenger/internal/pipeline"
"messenger/internal/platform/config"
"messenger/internal/platform/ident"
"messenger/internal/platform/postgres"
"messenger/internal/platform/s3store"
"messenger/internal/platform/valkey"
"messenger/internal/processing"
"messenger/internal/server"
)
@@ -85,10 +87,25 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
IdempotencyTTL: cfg.IdempotencyTTL,
Blobs: blobs,
AttachmentsCfg: attachment.Config{
MaxBytes: cfg.S3.MaxBytes,
MaxPixels: cfg.S3.MaxPixels,
BarrierDir: os.Getenv("LAB5_BARRIERS"),
MaxBytes: cfg.S3.MaxBytes,
MaxPixels: cfg.S3.MaxPixels,
},
})
var jobs config.Jobs
var barrier func(context.Context, pipeline.Job, string) error
if dir := os.Getenv("LAB5_BARRIERS"); dir != "" {
if err := os.MkdirAll(dir, 0700); err != nil {
return err
}
barrier = processing.FileBarrier(dir)
}
if blobs != nil {
jobs, err = config.LoadJobs(os.LookupEnv)
if err != nil {
return err
}
}
var debugLn net.Listener
if cfg.DebugAddr != "" {
if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil {
@@ -99,6 +116,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
ctx, cancel := context.WithCancel(ctx)
var background sync.WaitGroup
if blobs != nil {
background.Go(func() { processing.Dispatch(ctx, log, pipeline.NewStore(pool), jobs, barrier) })
}
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
if reconciler != nil {
background.Go(func() {
@@ -151,7 +171,7 @@ func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Looku
if err != nil {
return nil, nil, err
}
reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour)
reconciler, err := attachment.NewReconcilerWithRetention(pool, reconBlobs, time.Hour, reconCfg.StagingRetention)
if err != nil {
return nil, nil, err
}
+155
View File
@@ -0,0 +1,155 @@
// Command worker runs image consumers or an explicit operator action.
package main
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/broker"
"messenger/internal/imaging"
"messenger/internal/pipeline"
"messenger/internal/platform/config"
"messenger/internal/platform/postgres"
"messenger/internal/platform/s3store"
"messenger/internal/processing"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
if err := run(ctx, os.Args[1:]); err != nil {
slog.Error("worker stopped", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) error {
if len(args) == 1 && args[0] == "--prepare-large-png" {
return imaging.PrepareLargePNG(os.Stdin)
}
db, err := config.LoadDatabaseURL(os.LookupEnv)
if err != nil {
return err
}
cfg, err := config.LoadJobs(os.LookupEnv)
if err != nil {
return err
}
s3, err := config.LoadS3(os.LookupEnv)
if err != nil {
return err
}
blobs, err := s3store.New(s3)
if err != nil {
return err
}
pool, err := postgres.NewPool(ctx, db, 5)
if err != nil {
return err
}
defer pool.Close()
store := pipeline.NewStore(pool)
w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()}
if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") {
dir := strings.TrimPrefix(args[0], "--barriers=")
if err := os.MkdirAll(dir, 0700); err != nil {
return err
}
w.Checkpoint = processing.FileBarrier(dir)
args = nil
}
if len(args) == 0 {
w.Run(ctx)
return nil
}
op, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
if len(args) == 1 && args[0] == "failed" {
jobs, err := store.Failed(op)
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
const usage = "use failed, replay <id> <operator>, reprocess <id> <version>, or duplicate <id> <version> <attempt>"
if len(args) < 3 {
return errors.New(usage)
}
id, err := uuid.Parse(args[1])
if err != nil {
return errors.New("invalid attachment UUID")
}
exists := func(ctx context.Context, key string) (bool, error) {
_, err := blobs.Stat(ctx, key)
if errors.Is(err, attachment.ErrObjectMissing) {
return false, nil
}
return err == nil, err
}
switch args[0] {
case "replay":
if len(args) != 3 {
return errors.New(usage)
}
return store.Replay(op, id, args[2], exists)
case "reprocess":
if len(args) != 3 {
return errors.New(usage)
}
v, err := strconv.Atoi(args[2])
if err != nil {
return err
}
return store.Reprocess(op, id, v, exists)
case "duplicate":
if len(args) != 4 {
return errors.New(usage)
}
version, err := strconv.Atoi(args[2])
if err != nil {
return err
}
attempt, err := strconv.Atoi(args[3])
if err != nil {
return err
}
return duplicate(op, pool, cfg.URL, id, version, attempt)
default:
return errors.New(usage)
}
}
// duplicate sends a previously issued envelope through the actual broker. It
// cannot change SQL state or manufacture a future attempt.
func duplicate(ctx context.Context, pool *pgxpool.Pool, url string, id uuid.UUID, version, attempt int) error {
j := pipeline.Job{SchemaVersion: 1, Type: "attachment.process.v1", AttachmentID: id, PipelineVersion: version, Attempt: attempt}
err := pool.QueryRow(ctx, `SELECT job_id,created_at FROM attachment_jobs
WHERE attachment_id=$1 AND pipeline_version=$2 AND $3::integer BETWEEN 1 AND attempt`, id, version, attempt).Scan(&j.JobID, &j.CreatedAt)
if err != nil {
return err
}
p, err := broker.NewPublisher(ctx, url)
if err != nil {
return err
}
defer p.Close()
body, err := json.Marshal(j)
if err != nil {
return err
}
return p.Publish(ctx, body)
}
+59
View File
@@ -39,6 +39,9 @@ services:
environment:
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
@@ -55,6 +58,7 @@ services:
secrets:
- database_url
- session_store_url
- broker_url
- s3_access_key_id
- s3_secret_access_key
- s3_reconcile_access_key_id
@@ -80,6 +84,55 @@ services:
cpus: "1.0"
memory: 2G
broker:
image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
hostname: broker
secrets:
- source: rabbitmq_config
target: /etc/rabbitmq/conf.d/90-lab.conf
volumes:
- rabbitdata:/var/lib/rabbitmq
networks: [data]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 12
deploy:
resources:
limits:
cpus: "1"
memory: 512M
worker:
build:
context: .
target: worker
image: mtusi-messenger-worker:lab5
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "2"
memory: 2G
migrate:
image: mtusi-messenger:lab5
entrypoint: ["/usr/local/bin/migrate"]
@@ -222,6 +275,7 @@ services:
image: mtusi-messenger:lab5
entrypoint: ["/usr/local/bin/reconcile"]
environment:
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
@@ -252,6 +306,7 @@ services:
memory: 128M
volumes:
rabbitdata:
pgdata:
valkeydata:
s3data:
@@ -262,6 +317,10 @@ networks:
internal: true
secrets:
broker_url:
file: ./secrets/broker_url
rabbitmq_config:
file: ./secrets/rabbitmq.conf
database_url:
file: ./secrets/database_url
session_store_url:
+1
View File
@@ -10,6 +10,7 @@ require (
github.com/jackc/pgx/v5 v5.11.0
github.com/minio/minio-go/v7 v7.4.1
github.com/pressly/goose/v3 v3.27.0
github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.23.0
github.com/testcontainers/testcontainers-go v0.39.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0
+2
View File
@@ -171,6 +171,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+115
View File
@@ -0,0 +1,115 @@
// Package broker owns the AMQP connection for one publisher or consumer.
package broker
import (
"context"
"errors"
"net"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const queue = "attachment.jobs"
type link struct {
conn *amqp.Connection
channel *amqp.Channel
}
func open(ctx context.Context, url string) (*link, error) {
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 5 * time.Second, Dial: func(network, addr string) (net.Conn, error) {
d := net.Dialer{Timeout: 3 * time.Second}
c, err := d.DialContext(ctx, network, addr)
if err == nil {
err = c.SetDeadline(time.Now().Add(5 * time.Second))
}
return c, err
}})
// Parse errors can contain the complete connection URL.
if err != nil {
return nil, errors.New("RabbitMQ connection unavailable")
}
l := &link{conn: conn}
l.channel, err = conn.Channel()
if err == nil {
// PostgreSQL limits processing attempts. A broker redelivery can occur
// before PostgreSQL is reachable and must leave the job recoverable.
_, err = l.channel.QueueDeclare(queue, true, false, false, false, amqp.Table{
"x-queue-type": "quorum", "x-delivery-limit": int32(-1),
})
}
if err != nil {
l.Close()
return nil, err
}
return l, nil
}
func (l *link) Close() { _ = l.conn.CloseDeadline(time.Now().Add(time.Second)) }
type Publisher struct {
*link
returned <-chan amqp.Return
}
func NewPublisher(ctx context.Context, url string) (*Publisher, error) {
l, err := open(ctx, url)
if err != nil {
return nil, err
}
if err = l.channel.Confirm(false); err != nil {
l.Close()
return nil, err
}
return &Publisher{l, l.channel.NotifyReturn(make(chan amqp.Return, 1))}, nil
}
// Publish has one in-flight confirm on its channel. An ambiguous result means
// the caller discards this publisher and leaves the outbox row pending.
func (p *Publisher) Publish(ctx context.Context, body []byte) error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
stop := context.AfterFunc(ctx, p.Close)
defer stop()
confirmation, err := p.channel.PublishWithDeferredConfirmWithContext(ctx, "", queue, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body})
if err != nil {
return err
}
ok, err := confirmation.WaitContext(ctx)
if err != nil {
return err
}
if !ok {
return errors.New("broker did not confirm publication")
}
// RabbitMQ sends basic.return before the associated publisher confirm.
select {
case <-p.returned:
return errors.New("broker returned publication")
default:
return nil
}
}
type Consumer struct {
*link
Deliveries <-chan amqp.Delivery
}
func NewConsumer(ctx context.Context, url string) (*Consumer, error) {
l, err := open(ctx, url)
if err != nil {
return nil, err
}
if err = l.channel.Qos(1, 0, false); err != nil {
l.Close()
return nil, err
}
d, err := l.channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
if err != nil {
l.Close()
return nil, err
}
return &Consumer{l, d}, nil
}
+56
View File
@@ -0,0 +1,56 @@
package broker
import (
"context"
"testing"
"time"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
)
func TestConfirmAndRedeliverUnacked(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
defer cancel()
c, err := testcontainers.Run(ctx, "rabbitmq:4.1.4-alpine", testcontainers.WithEnv(map[string]string{"RABBITMQ_DEFAULT_USER": "lab", "RABBITMQ_DEFAULT_PASS": "test-only"}), testcontainers.WithExposedPorts("5672/tcp"), testcontainers.WithWaitStrategy(wait.ForLog("Server startup complete")))
if err != nil {
t.Fatal(err)
}
testcontainers.CleanupContainer(t, c)
endpoint, err := c.PortEndpoint(ctx, "5672/tcp", "")
if err != nil {
t.Fatal(err)
}
url := "amqp://lab:test-only@" + endpoint + "/"
pub, err := NewPublisher(ctx, url)
if err != nil {
t.Fatal(err)
}
defer pub.Close()
if err = pub.Publish(ctx, []byte("work")); err != nil {
t.Fatal(err)
}
// Broker redeliveries while PostgreSQL is unreachable do not consume
// the application budget. Exercise more than the quorum queue default.
for attempt := 0; attempt <= 22; attempt++ {
consumer, err := NewConsumer(ctx, url)
if err != nil {
t.Fatal(err)
}
select {
case msg := <-consumer.Deliveries:
if string(msg.Body) != "work" || msg.Redelivered != (attempt > 0) {
t.Fatalf("unexpected delivery on attempt %d", attempt)
}
if attempt == 22 {
if err := msg.Ack(false); err != nil {
t.Fatal(err)
}
}
case <-time.After(5 * time.Second):
consumer.Close()
t.Fatalf("confirmed job missing on delivery %d", attempt+1)
}
consumer.Close()
}
}
+41
View File
@@ -0,0 +1,41 @@
package config
import (
"fmt"
"strconv"
"time"
)
type Jobs struct {
URL string
Concurrency, MaxAttempts int
Timeout time.Duration
}
func LoadJobs(lookup Lookup) (Jobs, error) {
url, err := secret(lookup, "BROKER_URL_FILE")
if err != nil {
return Jobs{}, err
}
c := Jobs{URL: url, Concurrency: 2, MaxAttempts: 5}
seconds := 60
for _, p := range []struct {
key string
value *int
max int
}{
{"WORKER_CONCURRENCY", &c.Concurrency, 2},
{"JOB_MAX_ATTEMPTS", &c.MaxAttempts, 100},
{"JOB_TIMEOUT_SECONDS", &seconds, 3600},
} {
if s, ok := lookup(p.key); ok {
n, err := strconv.Atoi(s)
if err != nil || n < 1 || n > p.max {
return Jobs{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max)
}
*p.value = n
}
}
c.Timeout = time.Duration(seconds) * time.Second
return c, nil
}
+28
View File
@@ -0,0 +1,28 @@
package config
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestLoadJobs(t *testing.T) {
path := filepath.Join(t.TempDir(), "url")
if err := os.WriteFile(path, []byte("amqp://worker:secret@broker:5672/\n"), 0600); err != nil {
t.Fatal(err)
}
values := map[string]string{"BROKER_URL_FILE": path}
lookup := func(k string) (string, bool) { v, ok := values[k]; return v, ok }
cfg, err := LoadJobs(lookup)
if err != nil || cfg.Timeout != 60*time.Second || cfg.MaxAttempts != 5 || cfg.Concurrency < 1 || cfg.URL != "amqp://worker:secret@broker:5672/" {
t.Fatalf("unexpected defaults or secret read, err=%v", err)
}
for _, key := range []string{"JOB_TIMEOUT_SECONDS", "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS"} {
values[key] = "0"
if _, err := LoadJobs(lookup); err == nil {
t.Errorf("%s accepted zero", key)
}
delete(values, key)
}
}
+18
View File
@@ -0,0 +1,18 @@
package processing
import (
"context"
"fmt"
"path/filepath"
"messenger/internal/pipeline"
"messenger/internal/platform/faultbarrier"
)
// Only the explicit fault experiment enables these one-shot barriers.
func FileBarrier(dir string) func(context.Context, pipeline.Job, string) error {
return func(ctx context.Context, j pipeline.Job, phase string) error {
base := filepath.Join(dir, fmt.Sprintf("%s-v%d-%s", j.AttachmentID, j.PipelineVersion, phase))
return faultbarrier.Wait(ctx, base, "")
}
}
+70
View File
@@ -0,0 +1,70 @@
package processing
import (
"context"
"encoding/json"
"log/slog"
"time"
"messenger/internal/broker"
"messenger/internal/pipeline"
"messenger/internal/platform/config"
)
func retryPolicy(cfg config.Jobs) pipeline.RetryPolicy {
return pipeline.RetryPolicy{MaxAttempts: cfg.MaxAttempts, BaseDelay: time.Second, MaxDelay: time.Minute}
}
func pause(ctx context.Context, d time.Duration) {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) {
var p *broker.Publisher
defer func() {
if p != nil {
p.Close()
}
}()
for ctx.Err() == nil {
_, err := store.RecoverOne(ctx, retryPolicy(cfg))
if err == nil && p == nil {
p, err = broker.NewPublisher(ctx, cfg.URL)
}
if err == nil {
var sent bool
sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error {
body, err := json.Marshal(j)
if err != nil {
return err
}
if err := p.Publish(ctx, body); err != nil {
return err
}
if barrier != nil {
return barrier(ctx, j, "published")
}
return nil
})
if err == nil {
if !sent {
pause(ctx, 250*time.Millisecond)
}
continue
}
}
if p != nil {
p.Close()
p = nil
}
if ctx.Err() == nil {
log.Warn("attachment dispatcher will retry", "error", err)
}
pause(ctx, time.Second)
}
}
+176
View File
@@ -0,0 +1,176 @@
package processing
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"io"
"log/slog"
"sync"
"time"
"messenger/internal/attachment"
"messenger/internal/broker"
"messenger/internal/imaging"
"messenger/internal/pipeline"
"messenger/internal/platform/config"
)
var errSourceMismatch = errors.New("original differs from accepted metadata")
type Worker struct {
Store *pipeline.Store
Blobs attachment.Blobs
Config config.Jobs
Log *slog.Logger
// An experiment may pause an individual attempt at an exact durability boundary.
Checkpoint func(context.Context, pipeline.Job, string) error
}
func (w *Worker) Run(ctx context.Context) {
var wg sync.WaitGroup
for range w.Config.Concurrency {
wg.Go(func() {
for ctx.Err() == nil {
err := w.consume(ctx)
if ctx.Err() == nil {
w.Log.Warn("consumer reconnecting", "error", err)
pause(ctx, time.Second)
}
}
})
}
wg.Wait()
}
func (w *Worker) consume(ctx context.Context) error {
c, err := broker.NewConsumer(ctx, w.Config.URL)
if err != nil {
return err
}
defer c.Close()
for d := range c.Deliveries {
var j pipeline.Job
dec := json.NewDecoder(bytes.NewReader(d.Body))
dec.DisallowUnknownFields()
if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF {
w.Log.Warn("discarding invalid attachment envelope")
if err := d.Reject(false); err != nil {
return err
}
continue
}
if err := w.process(ctx, j); err != nil {
return err
}
if err := d.Ack(false); err != nil {
return err
}
if err := w.checkpoint(ctx, j, "acked"); err != nil {
return err
}
}
return errors.New("broker delivery stream closed")
}
func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error {
if w.Checkpoint != nil {
return w.Checkpoint(ctx, j, phase)
}
return nil
}
func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second)
if err != nil {
return err
}
if c == nil {
return w.checkpoint(ctx, j, "duplicate")
}
if err := w.checkpoint(ctx, j, "claimed"); err != nil {
return err
}
jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
defer cancel()
variants, err := w.writeObjects(jobCtx, c)
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
code := "processing_failed"
permanent := false
switch {
case errors.Is(err, imaging.ErrInvalidImage):
code, permanent = "invalid_image", true
case errors.Is(err, errSourceMismatch):
code, permanent = "source_mismatch", true
case errors.Is(err, attachment.ErrObjectMissing):
code = "source_missing"
}
w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code)
_, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config))
return err
}
if err := w.checkpoint(ctx, j, "objects"); err != nil {
return err
}
saved, err := w.Store.Complete(ctx, c, variants)
if err != nil || !saved {
return err
}
return w.checkpoint(ctx, j, "committed")
}
func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) {
if c.SizeBytes < 1 || c.SizeBytes > 10<<20 {
return nil, errSourceMismatch
}
r, err := blobs.Open(ctx, c.SourceKey)
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1))
closeErr := r.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
h := sha256.Sum256(data)
if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) {
return nil, errSourceMismatch
}
return data, nil
}
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) {
data, err := readOriginal(ctx, w.Blobs, c)
if err != nil {
return nil, err
}
encoded, err := imaging.Render(ctx, data, c.PipelineVersion)
if err != nil {
return nil, err
}
key := attachment.BlobKey(c.Hash)
if c.SourceKey != key {
if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil {
return nil, err
}
}
var variants []pipeline.Variant
for _, v := range encoded {
hash := sha256.Sum256(v.Data)
key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:])
n := int64(len(v.Data))
if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil {
return nil, err
}
variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]})
}
return variants, nil
}
+37
View File
@@ -0,0 +1,37 @@
package processing
import (
"context"
"crypto/sha256"
"errors"
"io"
"strings"
"testing"
"messenger/internal/attachment"
"messenger/internal/pipeline"
)
type stagedSource struct {
attachment.Blobs
body string
}
func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader(s.body)), nil
}
func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
h := sha256.Sum256([]byte("accepted"))
c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"}
for _, bad := range []string{"truncated", "altered!", "oversized bytes"} {
_, err := readOriginal(t.Context(), stagedSource{body: bad}, c)
if !errors.Is(err, errSourceMismatch) {
t.Fatalf("body %q: %v", bad, err)
}
}
got, err := readOriginal(t.Context(), stagedSource{body: "accepted"}, c)
if err != nil || string(got) != "accepted" {
t.Fatalf("valid original: %v", err)
}
}
+9
View File
@@ -37,6 +37,15 @@ if [ "$present" -eq 0 ]; then
printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url
fi
if [ ! -e secrets/rabbitmq.conf ] && [ ! -e secrets/broker_url ]; then
rabbit_password=$(random)
printf 'default_user = messenger\ndefault_pass = %s\n' "$rabbit_password" > secrets/rabbitmq.conf
printf 'amqp://messenger:%s@broker:5672/\n' "$rabbit_password" > secrets/broker_url
elif [ ! -s secrets/rabbitmq.conf ] || [ ! -s secrets/broker_url ]; then
echo "restore rabbitmq.conf and broker_url together" >&2
exit 1
fi
# Read just the bucket setting, never execute the local .env file as a script.
bucket=${S3_BUCKET:-}
if [ -z "$bucket" ] && [ -f .env ]; then
+2 -4
View File
@@ -4,9 +4,7 @@
set -eu
cd "$(dirname "$0")/.."
docker compose stop
# Re-run one-shot jobs against the retained data before restoring replicas.
docker compose up -d --wait postgres valkey s3
docker compose run --rm migrate
docker compose run --rm s3-init
# Compose starts both one-shot jobs and waits for their successful exit
# before either API replica or the worker can start.
docker compose up -d --wait
sh scripts/wait-ready.sh