Подключите RabbitMQ и отдельный обработчик заданий
This commit is contained in:
+2
-3
@@ -6,9 +6,8 @@ APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Sessions are stored in Valkey on this stand.
|
||||
SESSION_STORE_URL_FILE=/run/secrets/session_store_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
|
||||
@@ -30,8 +30,11 @@ jobs:
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: make secrets
|
||||
- run: chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
- run: docker compose up -d --build --wait
|
||||
- run: mkdir -p evidence-ci
|
||||
- name: Image codecs in the worker runtime
|
||||
run: make codec-test | tee evidence-ci/codec.txt
|
||||
- name: Public smoke
|
||||
run: make test PYTHON=python BASE_URL="$BASE_URL" | tee evidence-ci/smoke.txt
|
||||
- name: Persistence across a restart of the API and both stores
|
||||
|
||||
+10
-4
@@ -3,17 +3,17 @@
|
||||
FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download && go mod verify
|
||||
RUN --mount=type=cache,target=/go/pkg/mod go mod download && go mod verify
|
||||
COPY cmd ./cmd
|
||||
COPY internal ./internal
|
||||
COPY migrations ./migrations
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate ./cmd/reconcile ./cmd/s3init
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/worker ./cmd/migrate ./cmd/reconcile ./cmd/s3init
|
||||
|
||||
# The load generator gets an image of its own; the API image does not carry it.
|
||||
FROM build AS loadgen-build
|
||||
COPY tools ./tools
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod \
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/loadgen ./tools/loadgen
|
||||
|
||||
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS loadgen
|
||||
@@ -22,6 +22,12 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen
|
||||
USER app
|
||||
ENTRYPOINT ["/usr/local/bin/loadgen"]
|
||||
|
||||
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS worker
|
||||
RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp && adduser -D -H -u 10001 app
|
||||
COPY --from=build /out/worker /usr/local/bin/worker
|
||||
USER app
|
||||
ENTRYPOINT ["/usr/local/bin/worker"]
|
||||
|
||||
# Last, so that `build: .` builds it; it also runs the migrate service.
|
||||
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
|
||||
RUN adduser -D -H -u 10001 app
|
||||
|
||||
@@ -110,3 +110,14 @@ concurrency:
|
||||
|
||||
explain:
|
||||
docker compose exec -T postgres psql -U messenger -d messenger -X -q -v ON_ERROR_STOP=1 -f - < scripts/explain-history.sql
|
||||
|
||||
# Set CODEC_STRESS=1 after generating bin/decoder-fixtures to include the 40 MP cases.
|
||||
.PHONY: codec-test
|
||||
codec-test: export GOOS = linux
|
||||
codec-test: export GOARCH = $(shell docker version --format "{{.Server.Arch}}")
|
||||
codec-test: export CGO_ENABLED = 0
|
||||
codec-test: export MSYS_NO_PATHCONV = 1
|
||||
codec-test:
|
||||
go test -c -o bin/imaging.test ./internal/imaging
|
||||
docker build --target worker -t mtusi-messenger-worker:lab5 .
|
||||
docker run --rm --network none --cpus 2 --memory 2g --entrypoint /codec/imaging.test -e CODEC_REQUIRED=1 $(if $(filter 1,$(CODEC_STRESS)),-e CODEC_FIXTURES=/codec/decoder-fixtures,) --mount "type=bind,source=$(CURDIR)/bin,target=/codec,readonly" mtusi-messenger-worker:lab5 -test.v
|
||||
|
||||
+23
-3
@@ -19,11 +19,13 @@ import (
|
||||
"messenger/internal/app"
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/chat"
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/ident"
|
||||
"messenger/internal/platform/postgres"
|
||||
"messenger/internal/platform/s3store"
|
||||
"messenger/internal/platform/valkey"
|
||||
"messenger/internal/processing"
|
||||
"messenger/internal/server"
|
||||
)
|
||||
|
||||
@@ -85,10 +87,25 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
IdempotencyTTL: cfg.IdempotencyTTL,
|
||||
Blobs: blobs,
|
||||
AttachmentsCfg: attachment.Config{
|
||||
MaxBytes: cfg.S3.MaxBytes,
|
||||
MaxPixels: cfg.S3.MaxPixels,
|
||||
BarrierDir: os.Getenv("LAB5_BARRIERS"),
|
||||
MaxBytes: cfg.S3.MaxBytes,
|
||||
MaxPixels: cfg.S3.MaxPixels,
|
||||
},
|
||||
})
|
||||
var jobs config.Jobs
|
||||
var barrier func(context.Context, pipeline.Job, string) error
|
||||
if dir := os.Getenv("LAB5_BARRIERS"); dir != "" {
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
barrier = processing.FileBarrier(dir)
|
||||
}
|
||||
if blobs != nil {
|
||||
jobs, err = config.LoadJobs(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var debugLn net.Listener
|
||||
if cfg.DebugAddr != "" {
|
||||
if debugLn, err = net.Listen("tcp", cfg.DebugAddr); err != nil {
|
||||
@@ -99,6 +116,9 @@ func run(ctx context.Context, log *slog.Logger, cfg config.Config, alias string)
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
var background sync.WaitGroup
|
||||
if blobs != nil {
|
||||
background.Go(func() { processing.Dispatch(ctx, log, pipeline.NewStore(pool), jobs, barrier) })
|
||||
}
|
||||
background.Go(func() { chat.RunKeyCleanup(ctx, log, a.Chats, keyCleanupEvery) })
|
||||
if reconciler != nil {
|
||||
background.Go(func() {
|
||||
@@ -151,7 +171,7 @@ func newAttachmentStorage(pool *pgxpool.Pool, cfg config.S3, lookup config.Looku
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
reconciler, err := attachment.NewReconciler(pool, reconBlobs, time.Hour)
|
||||
reconciler, err := attachment.NewReconcilerWithRetention(pool, reconBlobs, time.Hour, reconCfg.StagingRetention)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
// Command worker runs image consumers or an explicit operator action.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/broker"
|
||||
"messenger/internal/imaging"
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/platform/config"
|
||||
"messenger/internal/platform/postgres"
|
||||
"messenger/internal/platform/s3store"
|
||||
"messenger/internal/processing"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Args[1:]); err != nil {
|
||||
slog.Error("worker stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string) error {
|
||||
if len(args) == 1 && args[0] == "--prepare-large-png" {
|
||||
return imaging.PrepareLargePNG(os.Stdin)
|
||||
}
|
||||
db, err := config.LoadDatabaseURL(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg, err := config.LoadJobs(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s3, err := config.LoadS3(os.LookupEnv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobs, err := s3store.New(s3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := postgres.NewPool(ctx, db, 5)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
store := pipeline.NewStore(pool)
|
||||
w := processing.Worker{Store: store, Blobs: blobs, Config: cfg, Log: slog.Default()}
|
||||
if len(args) == 1 && strings.HasPrefix(args[0], "--barriers=") {
|
||||
dir := strings.TrimPrefix(args[0], "--barriers=")
|
||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
w.Checkpoint = processing.FileBarrier(dir)
|
||||
args = nil
|
||||
}
|
||||
if len(args) == 0 {
|
||||
w.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
op, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
if len(args) == 1 && args[0] == "failed" {
|
||||
jobs, err := store.Failed(op)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
}
|
||||
const usage = "use failed, replay <id> <operator>, reprocess <id> <version>, or duplicate <id> <version> <attempt>"
|
||||
if len(args) < 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
id, err := uuid.Parse(args[1])
|
||||
if err != nil {
|
||||
return errors.New("invalid attachment UUID")
|
||||
}
|
||||
exists := func(ctx context.Context, key string) (bool, error) {
|
||||
_, err := blobs.Stat(ctx, key)
|
||||
if errors.Is(err, attachment.ErrObjectMissing) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
switch args[0] {
|
||||
case "replay":
|
||||
if len(args) != 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
return store.Replay(op, id, args[2], exists)
|
||||
case "reprocess":
|
||||
if len(args) != 3 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
v, err := strconv.Atoi(args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Reprocess(op, id, v, exists)
|
||||
case "duplicate":
|
||||
if len(args) != 4 {
|
||||
return errors.New(usage)
|
||||
}
|
||||
version, err := strconv.Atoi(args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attempt, err := strconv.Atoi(args[3])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return duplicate(op, pool, cfg.URL, id, version, attempt)
|
||||
default:
|
||||
return errors.New(usage)
|
||||
}
|
||||
}
|
||||
|
||||
// duplicate sends a previously issued envelope through the actual broker. It
|
||||
// cannot change SQL state or manufacture a future attempt.
|
||||
func duplicate(ctx context.Context, pool *pgxpool.Pool, url string, id uuid.UUID, version, attempt int) error {
|
||||
j := pipeline.Job{SchemaVersion: 1, Type: "attachment.process.v1", AttachmentID: id, PipelineVersion: version, Attempt: attempt}
|
||||
err := pool.QueryRow(ctx, `SELECT job_id,created_at FROM attachment_jobs
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 AND $3::integer BETWEEN 1 AND attempt`, id, version, attempt).Scan(&j.JobID, &j.CreatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p, err := broker.NewPublisher(ctx, url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer p.Close()
|
||||
body, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.Publish(ctx, body)
|
||||
}
|
||||
@@ -39,6 +39,9 @@ services:
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
@@ -55,6 +58,7 @@ services:
|
||||
secrets:
|
||||
- database_url
|
||||
- session_store_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_reconcile_access_key_id
|
||||
@@ -80,6 +84,55 @@ services:
|
||||
cpus: "1.0"
|
||||
memory: 2G
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
||||
hostname: broker
|
||||
secrets:
|
||||
- source: rabbitmq_config
|
||||
target: /etc/rabbitmq/conf.d/90-lab.conf
|
||||
volumes:
|
||||
- rabbitdata:/var/lib/rabbitmq
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1"
|
||||
memory: 512M
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
target: worker
|
||||
image: mtusi-messenger-worker:lab5
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "2"
|
||||
memory: 2G
|
||||
|
||||
migrate:
|
||||
image: mtusi-messenger:lab5
|
||||
entrypoint: ["/usr/local/bin/migrate"]
|
||||
@@ -222,6 +275,7 @@ services:
|
||||
image: mtusi-messenger:lab5
|
||||
entrypoint: ["/usr/local/bin/reconcile"]
|
||||
environment:
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
@@ -252,6 +306,7 @@ services:
|
||||
memory: 128M
|
||||
|
||||
volumes:
|
||||
rabbitdata:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
s3data:
|
||||
@@ -262,6 +317,10 @@ networks:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
rabbitmq_config:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
session_store_url:
|
||||
|
||||
@@ -10,6 +10,7 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/pressly/goose/v3 v3.27.0
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
github.com/testcontainers/testcontainers-go v0.39.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0
|
||||
|
||||
@@ -171,6 +171,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
// Package broker owns the AMQP connection for one publisher or consumer.
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
const queue = "attachment.jobs"
|
||||
|
||||
type link struct {
|
||||
conn *amqp.Connection
|
||||
channel *amqp.Channel
|
||||
}
|
||||
|
||||
func open(ctx context.Context, url string) (*link, error) {
|
||||
conn, err := amqp.DialConfig(url, amqp.Config{Heartbeat: 5 * time.Second, Dial: func(network, addr string) (net.Conn, error) {
|
||||
d := net.Dialer{Timeout: 3 * time.Second}
|
||||
c, err := d.DialContext(ctx, network, addr)
|
||||
if err == nil {
|
||||
err = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
}
|
||||
return c, err
|
||||
}})
|
||||
// Parse errors can contain the complete connection URL.
|
||||
if err != nil {
|
||||
return nil, errors.New("RabbitMQ connection unavailable")
|
||||
}
|
||||
l := &link{conn: conn}
|
||||
l.channel, err = conn.Channel()
|
||||
if err == nil {
|
||||
// PostgreSQL limits processing attempts. A broker redelivery can occur
|
||||
// before PostgreSQL is reachable and must leave the job recoverable.
|
||||
_, err = l.channel.QueueDeclare(queue, true, false, false, false, amqp.Table{
|
||||
"x-queue-type": "quorum", "x-delivery-limit": int32(-1),
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
l.Close()
|
||||
return nil, err
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func (l *link) Close() { _ = l.conn.CloseDeadline(time.Now().Add(time.Second)) }
|
||||
|
||||
type Publisher struct {
|
||||
*link
|
||||
returned <-chan amqp.Return
|
||||
}
|
||||
|
||||
func NewPublisher(ctx context.Context, url string) (*Publisher, error) {
|
||||
l, err := open(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = l.channel.Confirm(false); err != nil {
|
||||
l.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &Publisher{l, l.channel.NotifyReturn(make(chan amqp.Return, 1))}, nil
|
||||
}
|
||||
|
||||
// Publish has one in-flight confirm on its channel. An ambiguous result means
|
||||
// the caller discards this publisher and leaves the outbox row pending.
|
||||
func (p *Publisher) Publish(ctx context.Context, body []byte) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
stop := context.AfterFunc(ctx, p.Close)
|
||||
defer stop()
|
||||
confirmation, err := p.channel.PublishWithDeferredConfirmWithContext(ctx, "", queue, true, false, amqp.Publishing{ContentType: "application/json", DeliveryMode: amqp.Persistent, Body: body})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ok, err := confirmation.WaitContext(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return errors.New("broker did not confirm publication")
|
||||
}
|
||||
// RabbitMQ sends basic.return before the associated publisher confirm.
|
||||
select {
|
||||
case <-p.returned:
|
||||
return errors.New("broker returned publication")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type Consumer struct {
|
||||
*link
|
||||
Deliveries <-chan amqp.Delivery
|
||||
}
|
||||
|
||||
func NewConsumer(ctx context.Context, url string) (*Consumer, error) {
|
||||
l, err := open(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = l.channel.Qos(1, 0, false); err != nil {
|
||||
l.Close()
|
||||
return nil, err
|
||||
}
|
||||
d, err := l.channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
|
||||
if err != nil {
|
||||
l.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &Consumer{l, d}, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
||||
func TestConfirmAndRedeliverUnacked(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
c, err := testcontainers.Run(ctx, "rabbitmq:4.1.4-alpine", testcontainers.WithEnv(map[string]string{"RABBITMQ_DEFAULT_USER": "lab", "RABBITMQ_DEFAULT_PASS": "test-only"}), testcontainers.WithExposedPorts("5672/tcp"), testcontainers.WithWaitStrategy(wait.ForLog("Server startup complete")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
testcontainers.CleanupContainer(t, c)
|
||||
endpoint, err := c.PortEndpoint(ctx, "5672/tcp", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url := "amqp://lab:test-only@" + endpoint + "/"
|
||||
pub, err := NewPublisher(ctx, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pub.Close()
|
||||
if err = pub.Publish(ctx, []byte("work")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Broker redeliveries while PostgreSQL is unreachable do not consume
|
||||
// the application budget. Exercise more than the quorum queue default.
|
||||
for attempt := 0; attempt <= 22; attempt++ {
|
||||
consumer, err := NewConsumer(ctx, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case msg := <-consumer.Deliveries:
|
||||
if string(msg.Body) != "work" || msg.Redelivered != (attempt > 0) {
|
||||
t.Fatalf("unexpected delivery on attempt %d", attempt)
|
||||
}
|
||||
if attempt == 22 {
|
||||
if err := msg.Ack(false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
consumer.Close()
|
||||
t.Fatalf("confirmed job missing on delivery %d", attempt+1)
|
||||
}
|
||||
consumer.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Jobs struct {
|
||||
URL string
|
||||
Concurrency, MaxAttempts int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func LoadJobs(lookup Lookup) (Jobs, error) {
|
||||
url, err := secret(lookup, "BROKER_URL_FILE")
|
||||
if err != nil {
|
||||
return Jobs{}, err
|
||||
}
|
||||
c := Jobs{URL: url, Concurrency: 2, MaxAttempts: 5}
|
||||
seconds := 60
|
||||
for _, p := range []struct {
|
||||
key string
|
||||
value *int
|
||||
max int
|
||||
}{
|
||||
{"WORKER_CONCURRENCY", &c.Concurrency, 2},
|
||||
{"JOB_MAX_ATTEMPTS", &c.MaxAttempts, 100},
|
||||
{"JOB_TIMEOUT_SECONDS", &seconds, 3600},
|
||||
} {
|
||||
if s, ok := lookup(p.key); ok {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil || n < 1 || n > p.max {
|
||||
return Jobs{}, fmt.Errorf("%s must be between 1 and %d", p.key, p.max)
|
||||
}
|
||||
*p.value = n
|
||||
}
|
||||
}
|
||||
c.Timeout = time.Duration(seconds) * time.Second
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadJobs(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "url")
|
||||
if err := os.WriteFile(path, []byte("amqp://worker:secret@broker:5672/\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values := map[string]string{"BROKER_URL_FILE": path}
|
||||
lookup := func(k string) (string, bool) { v, ok := values[k]; return v, ok }
|
||||
cfg, err := LoadJobs(lookup)
|
||||
if err != nil || cfg.Timeout != 60*time.Second || cfg.MaxAttempts != 5 || cfg.Concurrency < 1 || cfg.URL != "amqp://worker:secret@broker:5672/" {
|
||||
t.Fatalf("unexpected defaults or secret read, err=%v", err)
|
||||
}
|
||||
for _, key := range []string{"JOB_TIMEOUT_SECONDS", "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS"} {
|
||||
values[key] = "0"
|
||||
if _, err := LoadJobs(lookup); err == nil {
|
||||
t.Errorf("%s accepted zero", key)
|
||||
}
|
||||
delete(values, key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/platform/faultbarrier"
|
||||
)
|
||||
|
||||
// Only the explicit fault experiment enables these one-shot barriers.
|
||||
func FileBarrier(dir string) func(context.Context, pipeline.Job, string) error {
|
||||
return func(ctx context.Context, j pipeline.Job, phase string) error {
|
||||
base := filepath.Join(dir, fmt.Sprintf("%s-v%d-%s", j.AttachmentID, j.PipelineVersion, phase))
|
||||
return faultbarrier.Wait(ctx, base, "")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"messenger/internal/broker"
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/platform/config"
|
||||
)
|
||||
|
||||
func retryPolicy(cfg config.Jobs) pipeline.RetryPolicy {
|
||||
return pipeline.RetryPolicy{MaxAttempts: cfg.MaxAttempts, BaseDelay: time.Second, MaxDelay: time.Minute}
|
||||
}
|
||||
|
||||
func pause(ctx context.Context, d time.Duration) {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
func Dispatch(ctx context.Context, log *slog.Logger, store *pipeline.Store, cfg config.Jobs, barrier func(context.Context, pipeline.Job, string) error) {
|
||||
var p *broker.Publisher
|
||||
defer func() {
|
||||
if p != nil {
|
||||
p.Close()
|
||||
}
|
||||
}()
|
||||
for ctx.Err() == nil {
|
||||
_, err := store.RecoverOne(ctx, retryPolicy(cfg))
|
||||
if err == nil && p == nil {
|
||||
p, err = broker.NewPublisher(ctx, cfg.URL)
|
||||
}
|
||||
if err == nil {
|
||||
var sent bool
|
||||
sent, err = store.PublishOne(ctx, func(ctx context.Context, j pipeline.Job) error {
|
||||
body, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Publish(ctx, body); err != nil {
|
||||
return err
|
||||
}
|
||||
if barrier != nil {
|
||||
return barrier(ctx, j, "published")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
if !sent {
|
||||
pause(ctx, 250*time.Millisecond)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if p != nil {
|
||||
p.Close()
|
||||
p = nil
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
log.Warn("attachment dispatcher will retry", "error", err)
|
||||
}
|
||||
pause(ctx, time.Second)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/broker"
|
||||
"messenger/internal/imaging"
|
||||
"messenger/internal/pipeline"
|
||||
"messenger/internal/platform/config"
|
||||
)
|
||||
|
||||
var errSourceMismatch = errors.New("original differs from accepted metadata")
|
||||
|
||||
type Worker struct {
|
||||
Store *pipeline.Store
|
||||
Blobs attachment.Blobs
|
||||
Config config.Jobs
|
||||
Log *slog.Logger
|
||||
// An experiment may pause an individual attempt at an exact durability boundary.
|
||||
Checkpoint func(context.Context, pipeline.Job, string) error
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
var wg sync.WaitGroup
|
||||
for range w.Config.Concurrency {
|
||||
wg.Go(func() {
|
||||
for ctx.Err() == nil {
|
||||
err := w.consume(ctx)
|
||||
if ctx.Err() == nil {
|
||||
w.Log.Warn("consumer reconnecting", "error", err)
|
||||
pause(ctx, time.Second)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) consume(ctx context.Context) error {
|
||||
c, err := broker.NewConsumer(ctx, w.Config.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer c.Close()
|
||||
for d := range c.Deliveries {
|
||||
var j pipeline.Job
|
||||
dec := json.NewDecoder(bytes.NewReader(d.Body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&j); err != nil || !j.Valid() || dec.Decode(new(any)) != io.EOF {
|
||||
w.Log.Warn("discarding invalid attachment envelope")
|
||||
if err := d.Reject(false); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := w.process(ctx, j); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.Ack(false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.checkpoint(ctx, j, "acked"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("broker delivery stream closed")
|
||||
}
|
||||
|
||||
func (w *Worker) checkpoint(ctx context.Context, j pipeline.Job, phase string) error {
|
||||
if w.Checkpoint != nil {
|
||||
return w.Checkpoint(ctx, j, phase)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, j pipeline.Job) error {
|
||||
c, err := w.Store.Claim(ctx, j, w.Config.Timeout+5*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c == nil {
|
||||
return w.checkpoint(ctx, j, "duplicate")
|
||||
}
|
||||
if err := w.checkpoint(ctx, j, "claimed"); err != nil {
|
||||
return err
|
||||
}
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
|
||||
defer cancel()
|
||||
variants, err := w.writeObjects(jobCtx, c)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
code := "processing_failed"
|
||||
permanent := false
|
||||
switch {
|
||||
case errors.Is(err, imaging.ErrInvalidImage):
|
||||
code, permanent = "invalid_image", true
|
||||
case errors.Is(err, errSourceMismatch):
|
||||
code, permanent = "source_mismatch", true
|
||||
case errors.Is(err, attachment.ErrObjectMissing):
|
||||
code = "source_missing"
|
||||
}
|
||||
w.Log.Warn("attachment attempt failed", "job_id", j.JobID, "attempt", j.Attempt, "error_code", code)
|
||||
_, err = w.Store.Fail(ctx, c, code, permanent, retryPolicy(w.Config))
|
||||
return err
|
||||
}
|
||||
if err := w.checkpoint(ctx, j, "objects"); err != nil {
|
||||
return err
|
||||
}
|
||||
saved, err := w.Store.Complete(ctx, c, variants)
|
||||
if err != nil || !saved {
|
||||
return err
|
||||
}
|
||||
return w.checkpoint(ctx, j, "committed")
|
||||
}
|
||||
|
||||
func readOriginal(ctx context.Context, blobs attachment.Blobs, c *pipeline.Claim) ([]byte, error) {
|
||||
if c.SizeBytes < 1 || c.SizeBytes > 10<<20 {
|
||||
return nil, errSourceMismatch
|
||||
}
|
||||
r, err := blobs.Open(ctx, c.SourceKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(r, c.SizeBytes+1))
|
||||
closeErr := r.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
h := sha256.Sum256(data)
|
||||
if int64(len(data)) != c.SizeBytes || !bytes.Equal(h[:], c.Hash) {
|
||||
return nil, errSourceMismatch
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (w *Worker) writeObjects(ctx context.Context, c *pipeline.Claim) ([]pipeline.Variant, error) {
|
||||
data, err := readOriginal(ctx, w.Blobs, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoded, err := imaging.Render(ctx, data, c.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := attachment.BlobKey(c.Hash)
|
||||
if c.SourceKey != key {
|
||||
if err := w.Blobs.Put(ctx, key, bytes.NewReader(data), c.SizeBytes, c.MIMEType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var variants []pipeline.Variant
|
||||
for _, v := range encoded {
|
||||
hash := sha256.Sum256(v.Data)
|
||||
key := pipeline.VariantKey(c.AttachmentID, c.PipelineVersion, v.Name, hash[:])
|
||||
n := int64(len(v.Data))
|
||||
if err := w.Blobs.Put(ctx, key, bytes.NewReader(v.Data), n, v.MIMEType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
variants = append(variants, pipeline.Variant{Name: v.Name, Key: key, MIMEType: v.MIMEType, Width: v.Width, Height: v.Height, SizeBytes: n, SHA256: hash[:]})
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"messenger/internal/attachment"
|
||||
"messenger/internal/pipeline"
|
||||
)
|
||||
|
||||
type stagedSource struct {
|
||||
attachment.Blobs
|
||||
body string
|
||||
}
|
||||
|
||||
func (s stagedSource) Open(context.Context, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(strings.NewReader(s.body)), nil
|
||||
}
|
||||
|
||||
func TestReadOriginalValidatesPersistedSizeAndDigest(t *testing.T) {
|
||||
h := sha256.Sum256([]byte("accepted"))
|
||||
c := &pipeline.Claim{SizeBytes: 8, Hash: h[:], SourceKey: "staging/test"}
|
||||
for _, bad := range []string{"truncated", "altered!", "oversized bytes"} {
|
||||
_, err := readOriginal(t.Context(), stagedSource{body: bad}, c)
|
||||
if !errors.Is(err, errSourceMismatch) {
|
||||
t.Fatalf("body %q: %v", bad, err)
|
||||
}
|
||||
}
|
||||
got, err := readOriginal(t.Context(), stagedSource{body: "accepted"}, c)
|
||||
if err != nil || string(got) != "accepted" {
|
||||
t.Fatalf("valid original: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,15 @@ if [ "$present" -eq 0 ]; then
|
||||
printf 'redis://:%s@valkey:6379/0\n' "$kv" > secrets/session_store_url
|
||||
fi
|
||||
|
||||
if [ ! -e secrets/rabbitmq.conf ] && [ ! -e secrets/broker_url ]; then
|
||||
rabbit_password=$(random)
|
||||
printf 'default_user = messenger\ndefault_pass = %s\n' "$rabbit_password" > secrets/rabbitmq.conf
|
||||
printf 'amqp://messenger:%s@broker:5672/\n' "$rabbit_password" > secrets/broker_url
|
||||
elif [ ! -s secrets/rabbitmq.conf ] || [ ! -s secrets/broker_url ]; then
|
||||
echo "restore rabbitmq.conf and broker_url together" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Read just the bucket setting, never execute the local .env file as a script.
|
||||
bucket=${S3_BUCKET:-}
|
||||
if [ -z "$bucket" ] && [ -f .env ]; then
|
||||
|
||||
+2
-4
@@ -4,9 +4,7 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop
|
||||
# Re-run one-shot jobs against the retained data before restoring replicas.
|
||||
docker compose up -d --wait postgres valkey s3
|
||||
docker compose run --rm migrate
|
||||
docker compose run --rm s3-init
|
||||
# Compose starts both one-shot jobs and waits for their successful exit
|
||||
# before either API replica or the worker can start.
|
||||
docker compose up -d --wait
|
||||
sh scripts/wait-ready.sh
|
||||
|
||||
Reference in New Issue
Block a user