forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
+3
-4
@@ -20,12 +20,9 @@ import (
|
||||
"github.com/gofiber/fiber/v3"
|
||||
"github.com/gofiber/fiber/v3/middleware/recover"
|
||||
"github.com/gofiber/fiber/v3/middleware/requestid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber.App, error) {
|
||||
st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL)
|
||||
|
||||
func New(cfg config.Config, st *store.Store, sessions *session.Store) (*fiber.App, error) {
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: cfg.S3Endpoint,
|
||||
Region: cfg.S3Region,
|
||||
@@ -74,6 +71,8 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber
|
||||
attachments.NewValidator(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
|
||||
cfg.MaxUploadBytes,
|
||||
cfg.UploadDrainLimit,
|
||||
cfg.MaxPendingJobs,
|
||||
cfg.JobMaxAttempts,
|
||||
cfg.UploadTimeout,
|
||||
cfg.DownloadTimeout,
|
||||
cfg.UploadTempDir,
|
||||
|
||||
@@ -31,35 +31,31 @@ type Handler struct {
|
||||
validator *Validator
|
||||
maxUploadBytes int64
|
||||
drainLimit int64
|
||||
maxPendingJobs int
|
||||
jobMaxAttempts int
|
||||
uploadTimeout time.Duration
|
||||
downloadTimeout time.Duration
|
||||
tempDir string
|
||||
}
|
||||
|
||||
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
|
||||
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, maxPendingJobs, jobMaxAttempts int, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
|
||||
return &Handler{
|
||||
store: st,
|
||||
objects: client,
|
||||
validator: validator,
|
||||
maxUploadBytes: maxUploadBytes,
|
||||
drainLimit: drainLimit,
|
||||
maxPendingJobs: maxPendingJobs,
|
||||
jobMaxAttempts: jobMaxAttempts,
|
||||
uploadTimeout: uploadTimeout,
|
||||
downloadTimeout: downloadTimeout,
|
||||
tempDir: tempDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Upload stores raw PNG/JPEG bytes. The order is deliberate:
|
||||
// 1. authorize before reading the body;
|
||||
// 2. stream the body to a bounded temporary file while hashing;
|
||||
// 3. validate pixels and decode;
|
||||
// 4. commit a pending metadata row;
|
||||
// 5. PUT the object;
|
||||
// 6. mark the row ready.
|
||||
//
|
||||
// A 201 is therefore returned only after both the object and its metadata are
|
||||
// durable. Every failure between 4 and 6 has a compensation path, and the
|
||||
// reconciler covers crashes that prevent compensation from running.
|
||||
// Upload stages the original, then commits the attachment snapshot and its
|
||||
// outbox job together. The 202 is therefore a promise about durable bytes plus
|
||||
// durable intent, not a promise that resizing already happened.
|
||||
func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
userID := common.MustUserID(c)
|
||||
|
||||
@@ -88,6 +84,19 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
|
||||
}
|
||||
|
||||
// Backpressure: refuse new work before reading its bytes. The check is
|
||||
// approximate across API replicas but bounds staging growth under overload.
|
||||
if h.maxPendingJobs > 0 {
|
||||
pending, countErr := h.store.CountPendingJobs(c.Context())
|
||||
if countErr != nil {
|
||||
return countErr
|
||||
}
|
||||
if pending >= h.maxPendingJobs {
|
||||
c.Set(fiber.HeaderRetryAfter, "5")
|
||||
return common.Unavailable("processing queue is full")
|
||||
}
|
||||
}
|
||||
|
||||
// Reject an obviously oversized request before allocating a temp file or
|
||||
// decoding anything. Chunked requests are caught by the streaming counter
|
||||
// below.
|
||||
@@ -151,6 +160,7 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
}
|
||||
|
||||
attachmentID := uuid.New()
|
||||
jobID := uuid.New()
|
||||
attachment := model.Attachment{
|
||||
ID: attachmentID,
|
||||
ChatID: chatID,
|
||||
@@ -159,55 +169,51 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
ContentType: info.ContentType,
|
||||
SizeBytes: written,
|
||||
SHA256: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Status: "pending",
|
||||
Status: "queued",
|
||||
Variants: []model.Variant{},
|
||||
ErrorCode: nil,
|
||||
CreatedAt: common.NowUTC(),
|
||||
UpdatedAt: common.NowUTC(),
|
||||
// AGENT: the key is server-generated from UUIDs and never contains the
|
||||
// client filename. That keeps traversal and collisions out of the
|
||||
// object store even before application authorization is considered.
|
||||
ObjectKey: objectKey(chatID, attachmentID),
|
||||
}
|
||||
|
||||
if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil {
|
||||
return err
|
||||
// AGENT: keys are server-generated from UUIDs, never the client
|
||||
// filename. staging/ is the durable handoff prefix; the worker copies
|
||||
// the same bytes to the final attachments/ key unchanged.
|
||||
ObjectKey: objectKey(chatID, attachmentID),
|
||||
StagingKey: stagingKey(chatID, attachmentID),
|
||||
PipelineVersion: 1,
|
||||
}
|
||||
|
||||
if _, err = file.Seek(0, 0); err != nil {
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
return common.BadRequest("cannot read upload body")
|
||||
}
|
||||
if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil {
|
||||
slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
if err = h.objects.Put(ctx, attachment.StagingKey, file, written, attachment.ContentType); err != nil {
|
||||
slog.Error("stage attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
return common.Unavailable("object storage unavailable")
|
||||
}
|
||||
|
||||
updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID)
|
||||
if err != nil {
|
||||
slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID)
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
return common.Unavailable("attachment metadata unavailable")
|
||||
job := model.JobRecord{
|
||||
Job: model.Job{
|
||||
JobID: jobID,
|
||||
AttachmentID: attachmentID,
|
||||
PipelineVersion: attachment.PipelineVersion,
|
||||
CreatedAt: attachment.CreatedAt,
|
||||
},
|
||||
MaxAttempts: h.jobMaxAttempts,
|
||||
}
|
||||
if err = h.store.CreateQueuedAttachment(ctx, attachment, job); err != nil {
|
||||
h.compensateStaging(attachment.StagingKey)
|
||||
return err
|
||||
}
|
||||
|
||||
attachment.Status = "ready"
|
||||
attachment.UpdatedAt = common.RFCFormTime(updatedAt)
|
||||
return c.Status(fiber.StatusCreated).JSON(attachment)
|
||||
return c.Status(fiber.StatusAccepted).JSON(attachment)
|
||||
}
|
||||
|
||||
// compensate is best effort by design. If either delete fails, the pending row
|
||||
// or the object remains visible to the reconciler, which retries after the
|
||||
// configured grace period.
|
||||
func (h *Handler) compensate(attachmentID uuid.UUID, key string) {
|
||||
// compensateStaging is best effort. If it fails, the reconciler removes an
|
||||
// object with no metadata row after the grace period.
|
||||
func (h *Handler) compensateStaging(key string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID)
|
||||
}
|
||||
if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil {
|
||||
slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID)
|
||||
slog.Error("compensate: delete staging object", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,10 +229,9 @@ func (h *Handler) Get(c fiber.Ctx) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if attachment.Status != "ready" {
|
||||
return common.Conflict("attachment is not ready")
|
||||
}
|
||||
|
||||
// queued/processing snapshots are part of lab5: the client polls this
|
||||
// endpoint until status becomes ready or failed.
|
||||
return c.JSON(attachment)
|
||||
}
|
||||
|
||||
@@ -242,7 +247,9 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
if variant == "" {
|
||||
variant = "original"
|
||||
}
|
||||
if variant != "original" {
|
||||
switch variant {
|
||||
case "original", "thumbnail", "crop", "optimized":
|
||||
default:
|
||||
return common.BadRequest("unsupported variant")
|
||||
}
|
||||
|
||||
@@ -254,9 +261,28 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
return common.Conflict("attachment is not ready")
|
||||
}
|
||||
|
||||
key := attachment.ObjectKey
|
||||
contentType := attachment.ContentType
|
||||
sizeBytes := attachment.SizeBytes
|
||||
if variant != "original" {
|
||||
found := false
|
||||
for _, candidate := range attachment.Variants {
|
||||
if candidate.Name == variant {
|
||||
key = store.VariantObjectKey(attachment.ObjectKey, candidate.Name)
|
||||
contentType = candidate.ContentType
|
||||
sizeBytes = candidate.SizeBytes
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return common.NotFound("attachment variant not found")
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout)
|
||||
|
||||
body, size, err := h.objects.Get(ctx, attachment.ObjectKey)
|
||||
body, size, err := h.objects.Get(ctx, key)
|
||||
if err != nil {
|
||||
cancel()
|
||||
if errors.Is(err, objectstore.ErrNotFound) {
|
||||
@@ -266,16 +292,16 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
return common.Unavailable("object storage unavailable")
|
||||
}
|
||||
if size != attachment.SizeBytes {
|
||||
if size != sizeBytes {
|
||||
_ = body.Close()
|
||||
cancel()
|
||||
slog.Error("attachment object size mismatch",
|
||||
"attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size)
|
||||
"attachment_id", attachment.ID, "metadata_size", sizeBytes, "object_size", size)
|
||||
return common.Unavailable("attachment content is inconsistent")
|
||||
}
|
||||
|
||||
c.Set(fiber.HeaderContentType, attachment.ContentType)
|
||||
c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10))
|
||||
c.Set(fiber.HeaderContentType, contentType)
|
||||
c.Set(fiber.HeaderContentLength, strconv.FormatInt(sizeBytes, 10))
|
||||
c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
|
||||
c.Set(fiber.HeaderCacheControl, "private, no-store")
|
||||
c.Set("X-Content-Type-Options", "nosniff")
|
||||
@@ -307,6 +333,10 @@ func objectKey(chatID, attachmentID uuid.UUID) string {
|
||||
return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID)
|
||||
}
|
||||
|
||||
func stagingKey(chatID, attachmentID uuid.UUID) string {
|
||||
return fmt.Sprintf("staging/%s/%s", chatID, attachmentID)
|
||||
}
|
||||
|
||||
// CleanupStaleUploads removes temp files abandoned by a previous crash. A
|
||||
// running request removes its own file in a defer; this only handles the
|
||||
// process-died-mid-upload case.
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
// Package broker wraps RabbitMQ delivery and publisher confirms. PostgreSQL is
|
||||
// still the source of truth: a message carries only a job id and a snapshot of
|
||||
// the durable job intent.
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
)
|
||||
|
||||
// ErrRequeue asks the consumer to nack and requeue. The worker uses it only
|
||||
// when its durable state could not be written (for example PostgreSQL is down).
|
||||
var ErrRequeue = errors.New("requeue delivery")
|
||||
|
||||
func dial(url string) (*amqp.Connection, error) {
|
||||
return amqp.DialConfig(url, amqp.Config{
|
||||
Heartbeat: 10 * time.Second,
|
||||
Locale: "en_US",
|
||||
})
|
||||
}
|
||||
|
||||
func declareQueue(ch *amqp.Channel, queue string) error {
|
||||
// A quorum queue uses a durable replicated log on the broker. Delivery is
|
||||
// still at least once; idempotency lives in the job effect key.
|
||||
_, err := ch.QueueDeclare(queue, true, false, false, false, amqp.Table{
|
||||
"x-queue-type": "quorum",
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Publisher is safe for concurrent use. It keeps one confirm-mode channel and
|
||||
// reconnects lazily after a broker restart.
|
||||
type Publisher struct {
|
||||
url string
|
||||
queue string
|
||||
mu sync.Mutex
|
||||
conn *amqp.Connection
|
||||
ch *amqp.Channel
|
||||
}
|
||||
|
||||
func NewPublisher(url, queue string) *Publisher {
|
||||
return &Publisher{url: url, queue: queue}
|
||||
}
|
||||
|
||||
func (p *Publisher) ensure() error {
|
||||
if p.ch != nil && !p.ch.IsClosed() {
|
||||
return nil
|
||||
}
|
||||
p.closeLocked()
|
||||
conn, err := dial(p.url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
if err = ch.Confirm(false); err != nil {
|
||||
_ = ch.Close()
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
if err = declareQueue(ch, p.queue); err != nil {
|
||||
_ = ch.Close()
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
p.conn, p.ch = conn, ch
|
||||
return nil
|
||||
}
|
||||
|
||||
// Publish waits for the broker's publisher confirm. A nil return means the
|
||||
// broker accepted the message, not that a worker executed it.
|
||||
func (p *Publisher) Publish(ctx context.Context, job model.Job) error {
|
||||
body, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if err = p.ensure(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
confirmation, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, "", p.queue, true, false, amqp.Publishing{
|
||||
DeliveryMode: amqp.Persistent,
|
||||
ContentType: "application/json",
|
||||
MessageId: job.JobID.String(),
|
||||
Timestamp: time.Now().UTC(),
|
||||
Type: job.Type,
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
p.closeLocked()
|
||||
return err
|
||||
}
|
||||
confirmed, err := confirmation.WaitContext(ctx)
|
||||
if err != nil {
|
||||
p.closeLocked()
|
||||
return err
|
||||
}
|
||||
if !confirmed {
|
||||
p.closeLocked()
|
||||
return fmt.Errorf("publisher confirm was negative")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Publisher) Close() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.closeLocked()
|
||||
}
|
||||
|
||||
func (p *Publisher) closeLocked() {
|
||||
if p.ch != nil {
|
||||
_ = p.ch.Close()
|
||||
}
|
||||
if p.conn != nil {
|
||||
_ = p.conn.Close()
|
||||
}
|
||||
p.ch, p.conn = nil, nil
|
||||
}
|
||||
|
||||
// Handler must return nil after the durable effect (and optional ack); the
|
||||
// consumer acks then. ErrRequeue leaves the message for redelivery.
|
||||
type Handler func(context.Context, model.Job) error
|
||||
|
||||
type Consumer struct {
|
||||
url string
|
||||
queue string
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewConsumer(url, queue string, logger *slog.Logger) *Consumer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Consumer{url: url, queue: queue, logger: logger}
|
||||
}
|
||||
|
||||
// Run reconnects until ctx is cancelled. concurrency bounds simultaneous
|
||||
// handlers in this process; prefetch bounds unacked messages the broker may
|
||||
// push. The two knobs are intentionally separate for the lab5 experiment.
|
||||
func (c *Consumer) Run(ctx context.Context, concurrency, prefetch int, handler Handler) error {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
err := c.consumeOnce(ctx, concurrency, prefetch, handler)
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
c.logger.Error("rabbitmq consumer disconnected", "err", err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Consumer) consumeOnce(ctx context.Context, concurrency, prefetch int, handler Handler) error {
|
||||
conn, err := dial(c.url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ch.Close()
|
||||
if err = declareQueue(ch, c.queue); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ch.Qos(prefetch, 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
deliveries, err := ch.Consume(c.queue, "", false, false, false, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var workers sync.WaitGroup
|
||||
|
||||
stop := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-stop:
|
||||
}
|
||||
_ = ch.Cancel("", false)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
close(stop)
|
||||
workers.Wait()
|
||||
return nil
|
||||
case delivery, ok := <-deliveries:
|
||||
if !ok {
|
||||
close(stop)
|
||||
workers.Wait()
|
||||
return errors.New("delivery channel closed")
|
||||
}
|
||||
var job model.Job
|
||||
if err = json.Unmarshal(delivery.Body, &job); err != nil {
|
||||
c.logger.Error("drop malformed job message", "err", err)
|
||||
_ = delivery.Ack(false)
|
||||
continue
|
||||
}
|
||||
|
||||
sem <- struct{}{}
|
||||
workers.Add(1)
|
||||
go func(delivery amqp.Delivery, job model.Job) {
|
||||
defer workers.Done()
|
||||
defer func() { <-sem }()
|
||||
handlerErr := handler(ctx, job)
|
||||
switch {
|
||||
case handlerErr == nil:
|
||||
if err := delivery.Ack(false); err != nil {
|
||||
c.logger.Error("ack failed", "job_id", job.JobID, "err", err)
|
||||
}
|
||||
case errors.Is(handlerErr, ErrRequeue):
|
||||
// A short sleep avoids a tight redelivery loop while a
|
||||
// dependency is down. The lease/outbox still own the job.
|
||||
time.Sleep(time.Second)
|
||||
if err := delivery.Nack(false, true); err != nil {
|
||||
c.logger.Error("nack failed", "job_id", job.JobID, "err", err)
|
||||
}
|
||||
default:
|
||||
c.logger.Error("job handler failed", "job_id", job.JobID, "err", handlerErr)
|
||||
_ = delivery.Ack(false)
|
||||
}
|
||||
}(delivery, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -22,6 +23,27 @@ func (t RFCFormTime) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(utc.Format(layout))
|
||||
}
|
||||
|
||||
// UnmarshalJSON accepts the same RFC 3339 strings MarshalJSON emits. The
|
||||
// broker round-trips Job values through JSON, and a named time.Time type does
|
||||
// not inherit encoding/json's time.Time special case on decode.
|
||||
func (t *RFCFormTime) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "null" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parsed, err := time.Parse(time.RFC3339Nano, raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse RFC form time: %w", err)
|
||||
}
|
||||
*t = RFCFormTime(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func NowUTC() RFCFormTime {
|
||||
return RFCFormTime(time.Now().UTC().Truncate(storageTimePrecision))
|
||||
}
|
||||
|
||||
@@ -43,6 +43,15 @@ func TestRFCFormTimeSerializesFractionalSeconds(t *testing.T) {
|
||||
if string(encoded) != tc.want {
|
||||
t.Fatalf("RFC form time = %s, want %s", encoded, tc.want)
|
||||
}
|
||||
|
||||
var decoded RFCFormTime
|
||||
if err = json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal RFC form time: %v", err)
|
||||
}
|
||||
want := tc.value.UTC().Truncate(storageTimePrecision)
|
||||
if !time.Time(decoded).Equal(want) {
|
||||
t.Fatalf("round trip = %v, want %v", time.Time(decoded), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -60,6 +62,26 @@ type Config struct {
|
||||
|
||||
ReconcileGrace time.Duration
|
||||
ReconcileInterval time.Duration
|
||||
|
||||
// Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox
|
||||
// publisher) and the worker (consumer), but not by the reconciler.
|
||||
BrokerURL string
|
||||
WorkerOnly bool
|
||||
JobQueue string
|
||||
JobMaxAttempts int
|
||||
JobTimeout time.Duration
|
||||
JobLease time.Duration
|
||||
RetryBase time.Duration
|
||||
RetryMax time.Duration
|
||||
WorkerConcurrency int
|
||||
WorkerPrefetch int
|
||||
MaxPendingJobs int
|
||||
LeaseSweep time.Duration
|
||||
StagingRetention time.Duration
|
||||
WorkTempDir string
|
||||
WorkerFaultMode string
|
||||
CrashAfterEffect bool
|
||||
CrashAfterClaim bool
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
@@ -141,6 +163,75 @@ func Load() (Config, error) {
|
||||
}
|
||||
cfg.ReconcileInterval = time.Duration(reconcileInterval) * time.Second
|
||||
|
||||
if cfg.WorkerOnly, err = envBool("WORKER_ONLY", false); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.CrashAfterEffect, err = envBool("WORKER_CRASH_AFTER_EFFECT", false); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.CrashAfterClaim, err = envBool("WORKER_CRASH_AFTER_CLAIM", false); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
jobAttempts, err := envInt("JOB_MAX_ATTEMPTS", 5, 1, 100)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
jobTimeout, err := envInt("JOB_TIMEOUT_SECONDS", 60, 5, 3600)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
jobLease, err := envInt("JOB_LEASE_SECONDS", 90, 10, 7200)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
retryBase, err := envInt("JOB_RETRY_BASE_SECONDS", 2, 1, 3600)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
retryMax, err := envInt("JOB_RETRY_MAX_SECONDS", 30, 1, 7200)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
concurrency, err := envInt("WORKER_CONCURRENCY", 2, 1, 128)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
prefetch, err := envInt("WORKER_PREFETCH", 2, 1, 512)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
maxPending, err := envInt("MAX_PENDING_JOBS", 1000, 0, 0)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
leaseSweep, err := envInt("LEASE_SWEEP_SECONDS", 15, 1, 3600)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
stagingRetention, err := envInt("STAGING_RETENTION_SECONDS", 86400, 60, 0)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
cfg.JobQueue = envOr("JOB_QUEUE", model.JobTypeAttachmentProcessV1)
|
||||
cfg.JobMaxAttempts = jobAttempts
|
||||
cfg.JobTimeout = time.Duration(jobTimeout) * time.Second
|
||||
cfg.JobLease = time.Duration(jobLease) * time.Second
|
||||
cfg.RetryBase = time.Duration(retryBase) * time.Second
|
||||
cfg.RetryMax = time.Duration(retryMax) * time.Second
|
||||
cfg.WorkerConcurrency = concurrency
|
||||
cfg.WorkerPrefetch = prefetch
|
||||
cfg.MaxPendingJobs = maxPending
|
||||
cfg.LeaseSweep = time.Duration(leaseSweep) * time.Second
|
||||
cfg.StagingRetention = time.Duration(stagingRetention) * time.Second
|
||||
cfg.WorkTempDir = envOr("WORK_TEMP_DIR", filepath.Join(os.TempDir(), "messenger-work"))
|
||||
cfg.WorkerFaultMode = envOr("WORKER_FAULT_MODE", "none")
|
||||
switch cfg.WorkerFaultMode {
|
||||
case "none", "transient-once", "permanent-once":
|
||||
default:
|
||||
return Config{}, fmt.Errorf("WORKER_FAULT_MODE must be none, transient-once or permanent-once")
|
||||
}
|
||||
|
||||
if cfg.DatabaseURL, err = secret("DATABASE_URL"); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
@@ -175,6 +266,17 @@ func Load() (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
if cfg.BrokerURL, err = secret("BROKER_URL"); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if cfg.BrokerURL == "" {
|
||||
return Config{}, fmt.Errorf("BROKER_URL or BROKER_URL_FILE is required")
|
||||
}
|
||||
|
||||
if cfg.WorkerOnly {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
if cfg.CacheURL, err = secret("CACHE_URL"); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
DROP TABLE IF EXISTS attachment_jobs;
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_variants_array;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_pipeline_version;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
|
||||
|
||||
-- Collapse asynchronous states back to lab4's single in-flight state. Ready
|
||||
-- and failed rows keep their meaning; queued/processing work is abandoned.
|
||||
UPDATE attachments
|
||||
SET status = 'pending', error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP COLUMN IF EXISTS variants,
|
||||
DROP COLUMN IF EXISTS pipeline_version,
|
||||
DROP COLUMN IF EXISTS staging_key,
|
||||
DROP COLUMN IF EXISTS ready_at;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('pending', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status = 'pending' AND error_code IS NULL) OR
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,76 @@
|
||||
-- Lab5 turns attachment processing into a durable asynchronous job.
|
||||
-- The API stages bytes in S3, then in ONE transaction writes the attachment
|
||||
-- snapshot and its outbox/job row. A separate publisher moves the intent to
|
||||
-- RabbitMQ, so a broker outage cannot lose an accepted upload.
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
|
||||
-- Lab4 "pending" meant "object PUT in progress". Nothing can finish it now,
|
||||
-- because lab5 acceptors write "queued" plus a job. Preserve the row for the
|
||||
-- operator instead of silently deleting it.
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'legacy_interrupted', updated_at = now()
|
||||
WHERE status = 'pending';
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS staging_key text,
|
||||
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS ready_at timestamptz;
|
||||
|
||||
UPDATE attachments SET ready_at = updated_at WHERE status = 'ready' AND ready_at IS NULL;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
),
|
||||
ADD CONSTRAINT attachments_variants_array
|
||||
CHECK (jsonb_typeof(variants) = 'array'),
|
||||
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1);
|
||||
|
||||
-- One job per logical attachment effect. published_at IS NULL is the outbox
|
||||
-- marker: a committed intent that has not received a publisher confirm.
|
||||
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
type text NOT NULL,
|
||||
pipeline_version integer NOT NULL,
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL,
|
||||
status text NOT NULL,
|
||||
error_code text,
|
||||
available_at timestamptz NOT NULL,
|
||||
published_at timestamptz,
|
||||
lease_until timestamptz,
|
||||
processing_started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
replay_count integer NOT NULL DEFAULT 0,
|
||||
last_replay_at timestamptz,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
CONSTRAINT attachment_jobs_status
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1),
|
||||
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT attachment_jobs_finished CHECK (
|
||||
(status IN ('queued', 'processing') AND finished_at IS NULL) OR
|
||||
(status IN ('ready', 'failed') AND finished_at IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx
|
||||
ON attachment_jobs (available_at, created_at)
|
||||
WHERE published_at IS NULL AND status = 'queued';
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_lease_idx
|
||||
ON attachment_jobs (lease_until)
|
||||
WHERE status = 'processing';
|
||||
CREATE INDEX IF NOT EXISTS attachment_jobs_status_idx
|
||||
ON attachment_jobs (status, created_at);
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Returning to the stricter 0004 shape requires clearing retry reasons first.
|
||||
UPDATE attachment_jobs
|
||||
SET error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
UPDATE attachments
|
||||
SET error_code = NULL
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL)
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
-- ScheduleRetry records the transient reason on the job while it waits for the
|
||||
-- next attempt. The original 0004 constraint only allowed error_code on failed
|
||||
-- rows, so a jittered retry update was rejected and the broker redelivery was
|
||||
-- then acknowledged as "busy", leaving the job processing until lease expiry.
|
||||
-- Ready stays clean; failed stays explainable; queued/processing may carry the
|
||||
-- most recent transient reason.
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT IF EXISTS attachments_error_state;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_error_state CHECK (
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL) OR
|
||||
(status IN ('queued', 'processing'))
|
||||
);
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD CONSTRAINT attachment_jobs_error_state CHECK (
|
||||
(status = 'ready' AND error_code IS NULL) OR
|
||||
(status = 'failed' AND error_code IS NOT NULL) OR
|
||||
(status IN ('queued', 'processing'))
|
||||
);
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
// latestMigrationVersion is the only value to bump when a migration is added.
|
||||
// The rest of the test derives filenames and ordering from the directory.
|
||||
const latestMigrationVersion = 3
|
||||
const latestMigrationVersion = 5
|
||||
|
||||
var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`)
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Job is the broker payload described by contracts/jobs.schema.json. It never
|
||||
// contains image bytes, bucket keys or credentials: the worker reloads trusted
|
||||
// metadata from PostgreSQL by attachment_id.
|
||||
type Job struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID uuid.UUID `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID uuid.UUID `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt common.RFCFormTime `json:"created_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
JobTypeAttachmentProcessV1 = "attachment.process.v1"
|
||||
|
||||
JobStatusQueued = "queued"
|
||||
JobStatusProcessing = "processing"
|
||||
JobStatusReady = "ready"
|
||||
JobStatusFailed = "failed"
|
||||
)
|
||||
|
||||
// JobRecord is the durable outbox row plus the operator-visible job state.
|
||||
type JobRecord struct {
|
||||
Job
|
||||
Status string
|
||||
// AvailableAt is the earliest time the outbox may publish a retry.
|
||||
AvailableAt time.Time
|
||||
// PublishedAt nil means the outbox has not received a publisher confirm
|
||||
// yet. This is the recovery marker for a crash between COMMIT and publish.
|
||||
PublishedAt *time.Time
|
||||
LeaseUntil *time.Time
|
||||
ProcessingStartedAt *time.Time
|
||||
FinishedAt *time.Time
|
||||
MaxAttempts int
|
||||
ErrorCode *string
|
||||
ReplayCount int
|
||||
}
|
||||
@@ -56,8 +56,10 @@ type Attachment struct {
|
||||
CreatedAt common.RFCFormTime `json:"created_at"`
|
||||
UpdatedAt common.RFCFormTime `json:"updated_at"`
|
||||
|
||||
// ObjectKey is internal and never serialized.
|
||||
ObjectKey string `json:"-"`
|
||||
// ObjectKey and StagingKey are internal and never serialized.
|
||||
ObjectKey string `json:"-"`
|
||||
StagingKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
func (a Attachment) GetID() uuid.UUID { return a.ID }
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Package outbox publishes committed job intents to RabbitMQ. The database row
|
||||
// is the outbox: published_at IS NULL means "this intent still needs a broker
|
||||
// confirm". A crash between COMMIT and publish is recovered by the next loop.
|
||||
package outbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
||||
)
|
||||
|
||||
const batchSize = 20
|
||||
|
||||
type Publisher struct {
|
||||
store *store.Store
|
||||
broker *broker.Publisher
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func New(st *store.Store, b *broker.Publisher, logger *slog.Logger) *Publisher {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Publisher{store: st, broker: b, logger: logger}
|
||||
}
|
||||
|
||||
func (p *Publisher) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(250 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := p.PublishBatch(ctx); err != nil {
|
||||
p.logger.Error("outbox publish pass failed", "err", err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Publisher) PublishBatch(ctx context.Context) error {
|
||||
jobs, err := p.store.ListPublishableJobs(ctx, batchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range jobs {
|
||||
if err = p.publish(ctx, record); err != nil {
|
||||
return fmt.Errorf("publish job %s: %w", record.JobID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Publisher) publish(ctx context.Context, record model.JobRecord) error {
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
message := model.Job{
|
||||
SchemaVersion: 1,
|
||||
JobID: record.JobID,
|
||||
Type: record.Type,
|
||||
AttachmentID: record.AttachmentID,
|
||||
PipelineVersion: record.PipelineVersion,
|
||||
Attempt: record.Attempt + 1,
|
||||
CreatedAt: record.CreatedAt,
|
||||
}
|
||||
if err := p.broker.Publish(publishCtx, message); err != nil {
|
||||
return err
|
||||
}
|
||||
// Compare-and-set on attempt/status: a worker may have claimed it, and a
|
||||
// retry may have been scheduled while the confirm was in flight.
|
||||
return p.store.MarkJobPublished(publishCtx, record.JobID, record.Attempt, time.Now().UTC())
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"image"
|
||||
"io"
|
||||
)
|
||||
|
||||
// jpegOrientation returns the EXIF orientation (1..8) of a JPEG, or 1 when the
|
||||
// file has no usable EXIF block. Go's image/jpeg intentionally ignores EXIF,
|
||||
// so the worker applies the transform after decoding. The original bytes are
|
||||
// never rewritten.
|
||||
func jpegOrientation(r io.ReadSeeker) int {
|
||||
defer func() { _, _ = r.Seek(0, 0) }()
|
||||
if _, err := r.Seek(0, 0); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
var soi [2]byte
|
||||
if _, err := io.ReadFull(r, soi[:]); err != nil || soi[0] != 0xff || soi[1] != 0xd8 {
|
||||
return 1
|
||||
}
|
||||
|
||||
for {
|
||||
var prefix [1]byte
|
||||
if _, err := io.ReadFull(r, prefix[:]); err != nil || prefix[0] != 0xff {
|
||||
return 1
|
||||
}
|
||||
marker := byte(0xff)
|
||||
for marker == 0xff {
|
||||
if _, err := io.ReadFull(r, prefix[:]); err != nil {
|
||||
return 1
|
||||
}
|
||||
marker = prefix[0]
|
||||
}
|
||||
|
||||
// Markers without a length field, and start-of-scan where metadata
|
||||
// after it cannot be an APP1 segment we care about.
|
||||
if marker == 0xd8 || marker == 0xd9 || marker == 0xda ||
|
||||
marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7) {
|
||||
return 1
|
||||
}
|
||||
|
||||
var lengthBytes [2]byte
|
||||
if _, err := io.ReadFull(r, lengthBytes[:]); err != nil {
|
||||
return 1
|
||||
}
|
||||
length := int(binary.BigEndian.Uint16(lengthBytes[:]))
|
||||
if length < 2 {
|
||||
return 1
|
||||
}
|
||||
payloadLength := length - 2
|
||||
if marker != 0xe1 || payloadLength < 6 {
|
||||
if _, err := r.Seek(int64(payloadLength), io.SeekCurrent); err != nil {
|
||||
return 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
payload := make([]byte, payloadLength)
|
||||
if _, err := io.ReadFull(r, payload); err != nil {
|
||||
return 1
|
||||
}
|
||||
if string(payload[:6]) == "Exif\x00\x00" {
|
||||
return parseTIFFOrientation(payload[6:])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseTIFFOrientation(data []byte) int {
|
||||
if len(data) < 8 {
|
||||
return 1
|
||||
}
|
||||
var order binary.ByteOrder
|
||||
switch string(data[:2]) {
|
||||
case "II":
|
||||
order = binary.LittleEndian
|
||||
case "MM":
|
||||
order = binary.BigEndian
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
if order.Uint16(data[2:4]) != 42 {
|
||||
return 1
|
||||
}
|
||||
|
||||
ifdOffset := int(order.Uint32(data[4:8]))
|
||||
if ifdOffset < 8 || ifdOffset+2 > len(data) {
|
||||
return 1
|
||||
}
|
||||
count := int(order.Uint16(data[ifdOffset : ifdOffset+2]))
|
||||
position := ifdOffset + 2
|
||||
for index := 0; index < count; index++ {
|
||||
if position+12 > len(data) {
|
||||
return 1
|
||||
}
|
||||
entry := data[position : position+12]
|
||||
tag := order.Uint16(entry[0:2])
|
||||
fieldType := order.Uint16(entry[2:4])
|
||||
items := order.Uint32(entry[4:8])
|
||||
if tag == 0x0112 && fieldType == 3 && items >= 1 {
|
||||
if items == 1 {
|
||||
value := int(order.Uint16(entry[8:10]))
|
||||
if value >= 1 && value <= 8 {
|
||||
return value
|
||||
}
|
||||
return 1
|
||||
}
|
||||
offset := int(order.Uint32(entry[8:12]))
|
||||
if offset+2 > len(data) {
|
||||
return 1
|
||||
}
|
||||
value := int(order.Uint16(data[offset : offset+2]))
|
||||
if value >= 1 && value <= 8 {
|
||||
return value
|
||||
}
|
||||
return 1
|
||||
}
|
||||
position += 12
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// orient applies the eight EXIF orientation transforms. Every branch returns a
|
||||
// new image except orientation 1, so the caller can treat the result as
|
||||
// read-only.
|
||||
func orient(src image.Image, orientation int) image.Image {
|
||||
if orientation <= 1 || orientation > 8 {
|
||||
return src
|
||||
}
|
||||
bounds := src.Bounds()
|
||||
width, height := bounds.Dx(), bounds.Dy()
|
||||
swap := orientation >= 5
|
||||
outWidth, outHeight := width, height
|
||||
if swap {
|
||||
outWidth, outHeight = height, width
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, outWidth, outHeight))
|
||||
for y := 0; y < height; y++ {
|
||||
for x := 0; x < width; x++ {
|
||||
var nx, ny int
|
||||
switch orientation {
|
||||
case 2: // mirror horizontal
|
||||
nx, ny = width-1-x, y
|
||||
case 3: // rotate 180
|
||||
nx, ny = width-1-x, height-1-y
|
||||
case 4: // mirror vertical
|
||||
nx, ny = x, height-1-y
|
||||
case 5: // transpose
|
||||
nx, ny = y, x
|
||||
case 6: // rotate 90 clockwise
|
||||
nx, ny = height-1-y, x
|
||||
case 7: // transverse
|
||||
nx, ny = height-1-y, width-1-x
|
||||
case 8: // rotate 270 clockwise
|
||||
nx, ny = y, width-1-x
|
||||
}
|
||||
dst.Set(nx, ny, src.At(bounds.Min.X+x, bounds.Min.Y+y))
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// Package pipeline turns one validated original image into the required lab5
|
||||
// variants. It is deterministic and side-effect free so a broker redelivery can
|
||||
// simply overwrite the same keys with the same bytes.
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"os"
|
||||
|
||||
"github.com/gen2brain/webp"
|
||||
"golang.org/x/image/draw"
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
const (
|
||||
ThumbnailMax = 256
|
||||
CropSize = 128
|
||||
OptimizedMax = 1280
|
||||
|
||||
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
|
||||
// copied byte-for-byte and keeps its metadata and checksum.
|
||||
JPEGQuality = 85
|
||||
WebPQuality = 82
|
||||
WebPMethod = 4
|
||||
)
|
||||
|
||||
type Processor struct {
|
||||
maxPixels int64
|
||||
budgetBytes int64
|
||||
decodeBudget *semaphore.Weighted
|
||||
}
|
||||
|
||||
// Encoded is one variant ready to be uploaded.
|
||||
type Encoded struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Bytes []byte
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func New(maxPixels, decodeBudgetBytes int64) *Processor {
|
||||
return &Processor{
|
||||
maxPixels: maxPixels,
|
||||
budgetBytes: decodeBudgetBytes,
|
||||
decodeBudget: semaphore.NewWeighted(decodeBudgetBytes),
|
||||
}
|
||||
}
|
||||
|
||||
// Failure classifies pipeline errors for the retry policy. A corrupt image is
|
||||
// permanent (retrying cannot help); I/O, lease expiry and dependency errors are
|
||||
// transient.
|
||||
type Failure struct {
|
||||
Permanent bool
|
||||
Code string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (f *Failure) Error() string {
|
||||
if f.Err == nil {
|
||||
return f.Code
|
||||
}
|
||||
return f.Code + ": " + f.Err.Error()
|
||||
}
|
||||
|
||||
func (f *Failure) Unwrap() error { return f.Err }
|
||||
|
||||
func permanent(code string, err error) *Failure {
|
||||
return &Failure{Permanent: true, Code: code, Err: err}
|
||||
}
|
||||
|
||||
func transient(code string, err error) *Failure {
|
||||
return &Failure{Permanent: false, Code: code, Err: err}
|
||||
}
|
||||
|
||||
// Process decodes the file once and returns all variants in memory. Worker
|
||||
// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous
|
||||
// decoded images, which is the memory-heavy part of the experiment.
|
||||
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) {
|
||||
img, err := p.decode(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax))
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
}
|
||||
crop, err := renderJPEG("crop", centerSquare(img, CropSize))
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
}
|
||||
optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax))
|
||||
if err != nil {
|
||||
return nil, permanent("processing_failed", err)
|
||||
}
|
||||
|
||||
return []Encoded{thumbnail, crop, optimized}, nil
|
||||
}
|
||||
|
||||
func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) {
|
||||
if _, err := file.Seek(0, 0); err != nil {
|
||||
return nil, transient("storage_read_failed", err)
|
||||
}
|
||||
cfg, format, err := image.DecodeConfig(file)
|
||||
if err != nil {
|
||||
return nil, permanent("invalid_image", err)
|
||||
}
|
||||
if format != "png" && format != "jpeg" {
|
||||
return nil, permanent("invalid_image", fmt.Errorf("unsupported format %q", format))
|
||||
}
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return nil, permanent("invalid_image", errors.New("empty image bounds"))
|
||||
}
|
||||
pixels := int64(cfg.Width) * int64(cfg.Height)
|
||||
if pixels > p.maxPixels {
|
||||
return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS"))
|
||||
}
|
||||
|
||||
// AGENT: the semaphore is acquired before Decode, not after. A decoder
|
||||
// allocates roughly width*height*4 bytes; without this gate several 40 MP
|
||||
// files can OOM the worker even with a small HTTP concurrency.
|
||||
weight := pixels * 4
|
||||
if weight > p.budgetBytes {
|
||||
return nil, permanent("invalid_image", errors.New("image exceeds decode memory budget"))
|
||||
}
|
||||
if err = p.decodeBudget.Acquire(ctx, weight); err != nil {
|
||||
return nil, transient("processing_timeout", err)
|
||||
}
|
||||
defer p.decodeBudget.Release(weight)
|
||||
|
||||
orientation := jpegOrientation(file)
|
||||
if _, err = file.Seek(0, 0); err != nil {
|
||||
return nil, transient("storage_read_failed", err)
|
||||
}
|
||||
img, decodeErr := safeDecode(file)
|
||||
if decodeErr != nil {
|
||||
return nil, permanent("invalid_image", decodeErr)
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
if bounds.Dx() != cfg.Width || bounds.Dy() != cfg.Height {
|
||||
return nil, permanent("invalid_image", errors.New("decoded bounds differ from header"))
|
||||
}
|
||||
return orient(img, orientation), nil
|
||||
}
|
||||
|
||||
func safeDecode(file *os.File) (img image.Image, err error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
img = nil
|
||||
err = fmt.Errorf("image decoder panic: %v", recovered)
|
||||
}
|
||||
}()
|
||||
img, _, err = image.Decode(file)
|
||||
return img, err
|
||||
}
|
||||
|
||||
// fitOnWhite scales down to fit maxWidth x maxHeight, never up. The source is
|
||||
// composited over white, which is the contract for JPEG variants with alpha.
|
||||
func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA {
|
||||
bounds := src.Bounds()
|
||||
width, height := fitSize(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight)
|
||||
dst := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), src, bounds, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the
|
||||
// contract allows upscaling tiny sources for the crop variant.
|
||||
func centerSquare(src image.Image, size int) *image.RGBA {
|
||||
bounds := src.Bounds()
|
||||
side := bounds.Dx()
|
||||
if bounds.Dy() < side {
|
||||
side = bounds.Dy()
|
||||
}
|
||||
x := bounds.Min.X + (bounds.Dx()-side)/2
|
||||
y := bounds.Min.Y + (bounds.Dy()-side)/2
|
||||
cropRect := image.Rect(x, y, x+side, y+side)
|
||||
|
||||
dst := image.NewRGBA(image.Rect(0, 0, size, size))
|
||||
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), src, cropRect, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// fitSize preserves aspect ratio, rounds down and clamps to at least 1 px.
|
||||
// It deliberately never enlarges a small source.
|
||||
func fitSize(width, height, maxWidth, maxHeight int) (int, int) {
|
||||
if width <= maxWidth && height <= maxHeight {
|
||||
return width, height
|
||||
}
|
||||
if width*maxHeight >= height*maxWidth {
|
||||
newHeight := height * maxWidth / width
|
||||
if newHeight < 1 {
|
||||
newHeight = 1
|
||||
}
|
||||
return maxWidth, newHeight
|
||||
}
|
||||
newWidth := width * maxHeight / height
|
||||
if newWidth < 1 {
|
||||
newWidth = 1
|
||||
}
|
||||
return newWidth, maxHeight
|
||||
}
|
||||
|
||||
func renderJPEG(name string, img image.Image) (Encoded, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
||||
return Encoded{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return Encoded{
|
||||
Name: name,
|
||||
ContentType: "image/jpeg",
|
||||
Bytes: buf.Bytes(),
|
||||
Width: bounds.Dx(),
|
||||
Height: bounds.Dy(),
|
||||
SHA256: digest(buf.Bytes()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func renderWebP(img image.Image) (Encoded, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
|
||||
return Encoded{}, err
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
return Encoded{
|
||||
Name: "optimized",
|
||||
ContentType: "image/webp",
|
||||
Bytes: buf.Bytes(),
|
||||
Width: bounds.Dx(),
|
||||
Height: bounds.Dy(),
|
||||
SHA256: digest(buf.Bytes()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func digest(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) {
|
||||
cases := []struct {
|
||||
w, h, maxW, maxH int
|
||||
wantW, wantH int
|
||||
}{
|
||||
{320, 200, 256, 256, 256, 160},
|
||||
{200, 320, 256, 256, 160, 256},
|
||||
{100, 50, 256, 256, 100, 50},
|
||||
{4000, 1000, 1280, 1280, 1280, 320},
|
||||
{1000, 4000, 256, 256, 64, 256},
|
||||
{1, 4000, 256, 256, 1, 256},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
w, h := fitSize(tc.w, tc.h, tc.maxW, tc.maxH)
|
||||
if w != tc.wantW || h != tc.wantH {
|
||||
t.Fatalf("fitSize(%d,%d,%d,%d) = %dx%d, want %dx%d",
|
||||
tc.w, tc.h, tc.maxW, tc.maxH, w, h, tc.wantW, tc.wantH)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessProducesContractVariants(t *testing.T) {
|
||||
file := pngFile(t, 320, 200, false)
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
if len(variants) != 3 {
|
||||
t.Fatalf("got %d variants, want 3", len(variants))
|
||||
}
|
||||
|
||||
byName := map[string]Encoded{}
|
||||
for _, variant := range variants {
|
||||
byName[variant.Name] = variant
|
||||
}
|
||||
thumbnail := byName["thumbnail"]
|
||||
if thumbnail.ContentType != "image/jpeg" || thumbnail.Width != 256 || thumbnail.Height != 160 {
|
||||
t.Fatalf("thumbnail = %s %dx%d, want image/jpeg 256x160",
|
||||
thumbnail.ContentType, thumbnail.Width, thumbnail.Height)
|
||||
}
|
||||
decoded, err := jpeg.Decode(bytes.NewReader(thumbnail.Bytes))
|
||||
if err != nil {
|
||||
t.Fatalf("decode thumbnail: %v", err)
|
||||
}
|
||||
if decoded.Bounds().Dx() != 256 || decoded.Bounds().Dy() != 160 {
|
||||
t.Fatalf("actual thumbnail bounds = %v", decoded.Bounds())
|
||||
}
|
||||
|
||||
crop := byName["crop"]
|
||||
if crop.ContentType != "image/jpeg" || crop.Width != 128 || crop.Height != 128 {
|
||||
t.Fatalf("crop = %s %dx%d", crop.ContentType, crop.Width, crop.Height)
|
||||
}
|
||||
optimized := byName["optimized"]
|
||||
if optimized.ContentType != "image/webp" || optimized.Width != 320 || optimized.Height != 200 {
|
||||
t.Fatalf("optimized = %s %dx%d, want image/webp 320x200", optimized.ContentType, optimized.Width, optimized.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailCompositesAlphaOnWhite(t *testing.T) {
|
||||
file := pngFile(t, 64, 64, true)
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
decoded, err := jpeg.Decode(bytes.NewReader(variants[0].Bytes))
|
||||
if err != nil {
|
||||
t.Fatalf("decode thumbnail: %v", err)
|
||||
}
|
||||
r, g, b, _ := decoded.At(32, 32).RGBA()
|
||||
if r < 0xf000 || g < 0xf000 || b < 0xf000 {
|
||||
t.Fatalf("transparent pixel is not white: %04x %04x %04x", r, g, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEXIFOrientationRotatesBeforeResize(t *testing.T) {
|
||||
file := jpegFileWithOrientation(t, 320, 200, 6)
|
||||
if got := jpegOrientation(file); got != 6 {
|
||||
t.Fatalf("jpegOrientation = %d, want 6", got)
|
||||
}
|
||||
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
|
||||
if err != nil {
|
||||
t.Fatalf("Process: %v", err)
|
||||
}
|
||||
if variants[0].Width != 160 || variants[0].Height != 256 {
|
||||
t.Fatalf("oriented thumbnail = %dx%d, want 160x256", variants[0].Width, variants[0].Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTIFFOrientationBigEndian(t *testing.T) {
|
||||
tiff := []byte{
|
||||
'M', 'M', 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08,
|
||||
0x00, 0x01,
|
||||
0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
}
|
||||
if got := parseTIFFOrientation(tiff); got != 8 {
|
||||
t.Fatalf("parseTIFFOrientation = %d, want 8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func pngFile(t *testing.T, width, height int, transparent bool) *os.File {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
for y := 0; y < height; y++ {
|
||||
for x := 0; x < width; x++ {
|
||||
alpha := uint8(0xff)
|
||||
if transparent {
|
||||
alpha = 0
|
||||
}
|
||||
img.Set(x, y, color.RGBA{R: 10, G: 120, B: 200, A: alpha})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode png: %v", err)
|
||||
}
|
||||
return tempFile(t, "image.png", buf.Bytes())
|
||||
}
|
||||
|
||||
func jpegFileWithOrientation(t *testing.T, width, height int, orientation uint16) *os.File {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, nil); err != nil {
|
||||
t.Fatalf("encode jpeg: %v", err)
|
||||
}
|
||||
|
||||
payload := []byte{'E', 'x', 'i', 'f', 0, 0}
|
||||
tiff := make([]byte, 26)
|
||||
copy(tiff, []byte{'I', 'I'})
|
||||
binary.LittleEndian.PutUint16(tiff[2:4], 42)
|
||||
binary.LittleEndian.PutUint32(tiff[4:8], 8)
|
||||
binary.LittleEndian.PutUint16(tiff[8:10], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
|
||||
binary.LittleEndian.PutUint16(tiff[12:14], 3)
|
||||
binary.LittleEndian.PutUint32(tiff[14:18], 1)
|
||||
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
|
||||
payload = append(payload, tiff...)
|
||||
|
||||
segment := append([]byte{0xff, 0xe1}, 0, 0)
|
||||
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
|
||||
segment = append(segment, payload...)
|
||||
|
||||
data := append([]byte{}, buf.Bytes()[:2]...)
|
||||
data = append(data, segment...)
|
||||
data = append(data, buf.Bytes()[2:]...)
|
||||
return tempFile(t, "oriented.jpg", data)
|
||||
}
|
||||
|
||||
func tempFile(t *testing.T, name string, data []byte) *os.File {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = file.Close() })
|
||||
return file
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
// Package reconcile repairs the two failure windows between PostgreSQL
|
||||
// metadata and S3 bytes:
|
||||
// Package reconcile repairs the two storage/metadata gaps that a crash can
|
||||
// leave behind:
|
||||
//
|
||||
// - a pending metadata row whose upload never finished or whose 201 was
|
||||
// never returned;
|
||||
// - an S3 object whose metadata transaction never committed.
|
||||
// - a staging object whose metadata row never committed;
|
||||
// - a terminal attachment whose staging copy was not deleted yet.
|
||||
//
|
||||
// Both repairs use a grace period so an upload that is still in flight is
|
||||
// never touched.
|
||||
// It never touches the final original or a variant while PostgreSQL still owns
|
||||
// it. The grace period protects in-flight work.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
@@ -19,7 +18,7 @@ import (
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
||||
)
|
||||
|
||||
const objectPrefix = "attachments/"
|
||||
var prefixes = []string{"attachments/", "staging/"}
|
||||
|
||||
type Reconciler struct {
|
||||
store *store.Store
|
||||
@@ -29,9 +28,9 @@ type Reconciler struct {
|
||||
}
|
||||
|
||||
type Stats struct {
|
||||
StalePending int `json:"stale_pending"`
|
||||
StaleStaging int `json:"stale_staging"`
|
||||
OrphanObjects int `json:"orphan_objects"`
|
||||
DeletedRows int `json:"deleted_rows"`
|
||||
DeletedStaging int `json:"deleted_staging"`
|
||||
DeletedObjects int `json:"deleted_objects"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
@@ -47,68 +46,64 @@ func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
|
||||
var stats Stats
|
||||
cutoff := time.Now().UTC().Add(-r.grace)
|
||||
|
||||
// Phase 1: rows that were committed as pending but never reached ready.
|
||||
stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500)
|
||||
// Phase 1: ready/failed rows eventually lose their staging copy. The
|
||||
// worker usually deletes it immediately after ready commits; this covers a
|
||||
// crash or a transient S3 error.
|
||||
stale, err := r.store.ListStaleStagingAttachments(ctx, cutoff, 500)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for _, attachment := range stale {
|
||||
stats.StalePending++
|
||||
stats.StaleStaging++
|
||||
if dryRun {
|
||||
r.logger.Info("reconcile: would remove stale pending attachment",
|
||||
"attachment_id", attachment.ID, "object_key", attachment.ObjectKey,
|
||||
"created_at", attachment.CreatedAt)
|
||||
r.logger.Info("reconcile: would remove stale staging",
|
||||
"attachment_id", attachment.ID, "staging_key", attachment.StagingKey)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
// Keep the row for the next pass. Deleting it now would lose the
|
||||
// only pointer to an object that might still exist.
|
||||
r.logger.Error("reconcile: delete stale object",
|
||||
if err = r.objects.Delete(ctx, attachment.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
r.logger.Error("reconcile: delete staging",
|
||||
"err", err, "attachment_id", attachment.ID)
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil {
|
||||
r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID)
|
||||
if err = r.store.ClearStagingKey(ctx, attachment.ID); err != nil {
|
||||
r.logger.Error("reconcile: clear staging key", "err", err, "attachment_id", attachment.ID)
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
stats.DeletedRows++
|
||||
stats.DeletedObjects++
|
||||
stats.DeletedStaging++
|
||||
}
|
||||
|
||||
// Phase 2: objects with no metadata row at all. The pending row is written
|
||||
// before the PUT in the normal path, so this only happens after a crash in
|
||||
// a narrow window (or manual bucket tampering).
|
||||
listed, err := r.objects.List(ctx, objectPrefix)
|
||||
// Phase 2: anything under our prefixes with no database owner. The API
|
||||
// writes staging before the metadata commit, so this window is real.
|
||||
known, err := r.store.ListStorageKeys(ctx)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
known, err := r.store.ListAttachmentObjectKeys(ctx)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for _, object := range listed {
|
||||
if _, ok := known[object.Key]; ok {
|
||||
continue
|
||||
for _, prefix := range prefixes {
|
||||
listed, listErr := r.objects.List(ctx, prefix)
|
||||
if listErr != nil {
|
||||
return stats, listErr
|
||||
}
|
||||
// A missing LastModified must not be treated as ancient: skipping is
|
||||
// safer than deleting an object that may have just been created.
|
||||
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
|
||||
continue
|
||||
for _, object := range listed {
|
||||
if _, ok := known[object.Key]; ok {
|
||||
continue
|
||||
}
|
||||
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
stats.OrphanObjects++
|
||||
if dryRun {
|
||||
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
|
||||
continue
|
||||
}
|
||||
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
stats.DeletedObjects++
|
||||
}
|
||||
stats.OrphanObjects++
|
||||
if dryRun {
|
||||
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
|
||||
continue
|
||||
}
|
||||
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
stats.DeletedObjects++
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
|
||||
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
@@ -12,10 +13,11 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// CreatePendingAttachment records the intent to store an object before the S3
|
||||
// PUT happens. Committing this row first is what makes the reconciler able to
|
||||
// distinguish "never uploaded" from "uploaded but process died".
|
||||
func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachment) error {
|
||||
// CreateQueuedAttachment is the durability boundary of the upload endpoint.
|
||||
// The attachment snapshot and the outbox job commit together, so there is no
|
||||
// observable state in which the API promised work but PostgreSQL lost the
|
||||
// intent. Publication to RabbitMQ happens later and may be retried safely.
|
||||
func (s *Store) CreateQueuedAttachment(ctx context.Context, att model.Attachment, job model.JobRecord) error {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
@@ -23,58 +25,74 @@ func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachmen
|
||||
if err != nil {
|
||||
return common.BadRequest("invalid sha256")
|
||||
}
|
||||
if att.PipelineVersion < 1 {
|
||||
att.PipelineVersion = 1
|
||||
}
|
||||
variants, err := json.Marshal([]model.Variant{})
|
||||
if err != nil {
|
||||
return common.Internal()
|
||||
}
|
||||
|
||||
_, err = s.pool.Exec(ctx, `
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, error_code, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL, $9, $9)`,
|
||||
object_key, staging_key, pipeline_version, variants, status, error_code,
|
||||
created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued', NULL, $12, $12)`,
|
||||
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes,
|
||||
hash, att.ObjectKey, time.Time(att.CreatedAt))
|
||||
return dbError(err)
|
||||
}
|
||||
|
||||
// MarkAttachmentReady publishes metadata only after the object bytes are known
|
||||
// to be in S3. A 201 is therefore a promise about both stores.
|
||||
func (s *Store) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (time.Time, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
updatedAt := time.Now().UTC()
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = $2
|
||||
WHERE id = $1 AND status = 'pending'`, id, updatedAt)
|
||||
hash, att.ObjectKey, att.StagingKey, att.PipelineVersion, variants, time.Time(att.CreatedAt))
|
||||
if err != nil {
|
||||
return time.Time{}, dbError(err)
|
||||
return dbError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return time.Time{}, common.Conflict("attachment is not pending")
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
|
||||
status, error_code, available_at, published_at, lease_until,
|
||||
processing_started_at, finished_at, replay_count, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, 0, $5, 'queued', NULL, $6, NULL, NULL,
|
||||
NULL, NULL, 0, $6, $6)`,
|
||||
job.JobID, att.ID, model.JobTypeAttachmentProcessV1, att.PipelineVersion,
|
||||
job.MaxAttempts, time.Time(att.CreatedAt))
|
||||
if err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
return updatedAt, nil
|
||||
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePendingAttachment removes a row only while it is still pending. The
|
||||
// status guard is the safety property: if an upload completed concurrently, the
|
||||
// compensation or reconciler pass must not delete a ready attachment.
|
||||
func (s *Store) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error {
|
||||
// CountPendingJobs is an admission control signal, not a queue size proof.
|
||||
// Two API replicas may observe slightly stale counts, but the durable cap is
|
||||
// still useful: it stops an overload from growing staging and PostgreSQL
|
||||
// without limit. Rejections are 503 because the client may retry later.
|
||||
func (s *Store) CountPendingJobs(ctx context.Context) (int, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
|
||||
return dbError(err)
|
||||
var count int
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM attachment_jobs
|
||||
WHERE status IN ('queued', 'processing')`).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, dbError(err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
att, err := scanAttachment(s.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, a.status, a.error_code,
|
||||
a.created_at, a.updated_at, a.object_key
|
||||
FROM attachments a
|
||||
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+`
|
||||
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2
|
||||
WHERE a.id = $1`, attachmentID, userID))
|
||||
if err != nil {
|
||||
@@ -83,20 +101,76 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID u
|
||||
return att, nil
|
||||
}
|
||||
|
||||
// ListStalePendingAttachments is the entry point for crash recovery. cutoff is
|
||||
// an instant, not a duration: rows created before it are eligible. The caller
|
||||
// computes it as now - grace so the boundary can be tuned and tested.
|
||||
func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
|
||||
// GetAttachment is the worker's trusted read. The worker never accepts bucket
|
||||
// keys or filenames from the broker message.
|
||||
func (s *Store) GetAttachment(ctx context.Context, attachmentID uuid.UUID) (model.Attachment, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, a.status, a.error_code,
|
||||
a.created_at, a.updated_at, a.object_key
|
||||
FROM attachments a
|
||||
WHERE a.status = 'pending' AND a.created_at < $1
|
||||
ORDER BY a.created_at
|
||||
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1`, attachmentID))
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
return att, nil
|
||||
}
|
||||
|
||||
const attachmentSelect = `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, a.status, a.error_code,
|
||||
a.created_at, a.updated_at, a.object_key, COALESCE(a.staging_key, ''),
|
||||
a.pipeline_version, a.variants
|
||||
FROM attachments a`
|
||||
|
||||
func scanAttachment(row rowScanner) (model.Attachment, error) {
|
||||
var (
|
||||
att model.Attachment
|
||||
hash []byte
|
||||
errCode *string
|
||||
createdAt time.Time
|
||||
updatedAt time.Time
|
||||
variants []byte
|
||||
)
|
||||
|
||||
if err := row.Scan(
|
||||
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
|
||||
&att.SizeBytes, &hash, &att.Status, &errCode,
|
||||
&createdAt, &updatedAt, &att.ObjectKey, &att.StagingKey,
|
||||
&att.PipelineVersion, &variants,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, common.NotFound("attachment not found")
|
||||
}
|
||||
return model.Attachment{}, dbError(err)
|
||||
}
|
||||
|
||||
att.SHA256 = hex.EncodeToString(hash)
|
||||
att.ErrorCode = errCode
|
||||
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
|
||||
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
|
||||
att.Variants = []model.Variant{}
|
||||
if len(variants) > 0 {
|
||||
if err := json.Unmarshal(variants, &att.Variants); err != nil {
|
||||
return model.Attachment{}, dbError(err)
|
||||
}
|
||||
}
|
||||
if att.Variants == nil {
|
||||
att.Variants = []model.Variant{}
|
||||
}
|
||||
return att, nil
|
||||
}
|
||||
|
||||
// ListStaleStagingAttachments returns terminal attachments whose staging copy
|
||||
// may be removed. The grace period is applied to updated_at so a just-finished
|
||||
// job is never cleaned while its commit is still settling.
|
||||
func (s *Store) ListStaleStagingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(ctx, attachmentSelect+`
|
||||
WHERE a.status IN ('ready', 'failed')
|
||||
AND a.updated_at < $1
|
||||
AND a.staging_key IS NOT NULL
|
||||
ORDER BY a.updated_at
|
||||
LIMIT $2`, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, dbError(err)
|
||||
@@ -107,7 +181,7 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim
|
||||
for rows.Next() {
|
||||
att, scanErr := scanAttachment(rows)
|
||||
if scanErr != nil {
|
||||
return nil, dbError(scanErr)
|
||||
return nil, scanErr
|
||||
}
|
||||
result = append(result, att)
|
||||
}
|
||||
@@ -117,11 +191,24 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct{}, error) {
|
||||
func (s *Store) ClearStagingKey(ctx context.Context, attachmentID uuid.UUID) error {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(ctx, `SELECT object_key FROM attachments`)
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachments SET staging_key = NULL WHERE id = $1`, attachmentID)
|
||||
return dbError(err)
|
||||
}
|
||||
|
||||
// ListStorageKeys returns every object the database still owns. It includes
|
||||
// variant keys derived from the original key; the reconciler must never delete
|
||||
// a thumbnail that has no separate row.
|
||||
func (s *Store) ListStorageKeys(ctx context.Context) (map[string]struct{}, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT object_key, COALESCE(staging_key, ''), variants FROM attachments`)
|
||||
if err != nil {
|
||||
return nil, dbError(err)
|
||||
}
|
||||
@@ -129,11 +216,25 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct
|
||||
|
||||
keys := make(map[string]struct{})
|
||||
for rows.Next() {
|
||||
var key string
|
||||
if err = rows.Scan(&key); err != nil {
|
||||
var (
|
||||
objectKey string
|
||||
staging string
|
||||
variants []byte
|
||||
)
|
||||
if err = rows.Scan(&objectKey, &staging, &variants); err != nil {
|
||||
return nil, dbError(err)
|
||||
}
|
||||
keys[key] = struct{}{}
|
||||
keys[objectKey] = struct{}{}
|
||||
if staging != "" {
|
||||
keys[staging] = struct{}{}
|
||||
}
|
||||
var list []model.Variant
|
||||
if err = json.Unmarshal(variants, &list); err != nil {
|
||||
return nil, dbError(err)
|
||||
}
|
||||
for _, variant := range list {
|
||||
keys[VariantObjectKey(objectKey, variant.Name)] = struct{}{}
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, dbError(err)
|
||||
@@ -141,30 +242,9 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func scanAttachment(row rowScanner) (model.Attachment, error) {
|
||||
var (
|
||||
att model.Attachment
|
||||
hash []byte
|
||||
errCode *string
|
||||
createdAt time.Time
|
||||
updatedAt time.Time
|
||||
)
|
||||
|
||||
if err := row.Scan(
|
||||
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
|
||||
&att.SizeBytes, &hash, &att.Status, &errCode,
|
||||
&createdAt, &updatedAt, &att.ObjectKey,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.Attachment{}, common.NotFound("attachment not found")
|
||||
}
|
||||
return model.Attachment{}, dbError(err)
|
||||
}
|
||||
|
||||
att.SHA256 = hex.EncodeToString(hash)
|
||||
att.ErrorCode = errCode
|
||||
att.Variants = []model.Variant{}
|
||||
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
|
||||
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
|
||||
return att, nil
|
||||
// VariantObjectKey is the one place that names variant objects. Keeping it
|
||||
// deterministic makes a retried job overwrite the same keys instead of
|
||||
// creating a second variant set.
|
||||
func VariantObjectKey(originalKey, variant string) string {
|
||||
return originalKey + "/" + variant
|
||||
}
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ClaimResult tells the consumer whether this delivery owns the work.
|
||||
type ClaimResult struct {
|
||||
Job model.JobRecord
|
||||
Attachment model.Attachment
|
||||
// Terminal means the logical work is already ready/failed. Ack it.
|
||||
Terminal bool
|
||||
// Busy means another attempt owns an active lease, or a retry is not due.
|
||||
// The publisher/lease sweeper will make the job visible again.
|
||||
Busy bool
|
||||
}
|
||||
|
||||
// ClaimJob acquires or renews the logical job lease in one transaction.
|
||||
// A broker redelivery of the same job_id therefore cannot become two workers
|
||||
// writing results at the same time.
|
||||
func (s *Store) ClaimJob(ctx context.Context, jobID uuid.UUID, lease time.Duration) (ClaimResult, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
job, err := scanJob(tx.QueryRow(ctx, `
|
||||
SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
|
||||
status, error_code, created_at, available_at, published_at,
|
||||
lease_until, processing_started_at, finished_at, replay_count
|
||||
FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID))
|
||||
if err != nil {
|
||||
return ClaimResult{}, err
|
||||
}
|
||||
|
||||
att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, job.AttachmentID))
|
||||
if err != nil {
|
||||
return ClaimResult{}, err
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if job.Status == model.JobStatusReady || job.Status == model.JobStatusFailed || att.Status == "ready" {
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
|
||||
}
|
||||
if att.Status == "failed" {
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
|
||||
}
|
||||
if job.Status == model.JobStatusProcessing && job.LeaseUntil != nil && job.LeaseUntil.After(now) {
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
return ClaimResult{Job: job, Attachment: att, Busy: true}, nil
|
||||
}
|
||||
if job.Status == model.JobStatusQueued && job.AvailableAt.After(now) {
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
return ClaimResult{Job: job, Attachment: att, Busy: true}, nil
|
||||
}
|
||||
|
||||
if job.Attempt >= job.MaxAttempts {
|
||||
_, failErr := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', error_code = 'retry_exhausted',
|
||||
finished_at = $2, lease_until = NULL, updated_at = $2
|
||||
WHERE job_id = $1`, jobID, now)
|
||||
if failErr != nil {
|
||||
return ClaimResult{}, dbError(failErr)
|
||||
}
|
||||
_, failErr = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'retry_exhausted', updated_at = $2
|
||||
WHERE id = $1 AND status <> 'ready'`, job.AttachmentID, now)
|
||||
if failErr != nil {
|
||||
return ClaimResult{}, dbError(failErr)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
|
||||
}
|
||||
|
||||
attempt := job.Attempt + 1
|
||||
leaseUntil := now.Add(lease)
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing', attempt = $2, error_code = NULL,
|
||||
lease_until = $3, processing_started_at = COALESCE(processing_started_at, $4),
|
||||
published_at = COALESCE(published_at, $4), updated_at = $4
|
||||
WHERE job_id = $1`, jobID, attempt, leaseUntil, now)
|
||||
if err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = $2
|
||||
WHERE id = $1 AND status IN ('queued', 'processing')`, job.AttachmentID, now)
|
||||
if err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return ClaimResult{}, dbError(err)
|
||||
}
|
||||
|
||||
job.Attempt = attempt
|
||||
job.Status = model.JobStatusProcessing
|
||||
job.ErrorCode = nil
|
||||
job.LeaseUntil = &leaseUntil
|
||||
if job.ProcessingStartedAt == nil {
|
||||
job.ProcessingStartedAt = &now
|
||||
}
|
||||
att.Status = "processing"
|
||||
att.ErrorCode = nil
|
||||
att.UpdatedAt = common.RFCFormTime(now)
|
||||
return ClaimResult{Job: job, Attachment: att}, nil
|
||||
}
|
||||
|
||||
// CompleteJob commits the ready snapshot only for the attempt/version that
|
||||
// still owns the lease. An older duplicate can therefore never roll a newer
|
||||
// result back or overwrite it.
|
||||
func (s *Store) CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, variants []model.Variant) (bool, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
if variants == nil {
|
||||
variants = []model.Variant{}
|
||||
}
|
||||
encoded, err := json.Marshal(variants)
|
||||
if err != nil {
|
||||
return false, common.Internal()
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
var attachmentID uuid.UUID
|
||||
var pipelineVersion int
|
||||
var status string
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT attachment_id, pipeline_version, status
|
||||
FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID).
|
||||
Scan(&attachmentID, &pipelineVersion, &status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, common.NotFound("job not found")
|
||||
}
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if status != model.JobStatusProcessing {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, attachmentID))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if att.PipelineVersion != pipelineVersion || att.Status == "ready" || att.Status == "failed" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', error_code = NULL, finished_at = $3,
|
||||
lease_until = NULL, updated_at = $3
|
||||
WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`,
|
||||
jobID, attempt, now)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'ready', variants = $2, error_code = NULL,
|
||||
ready_at = $3, updated_at = $3
|
||||
WHERE id = $1 AND pipeline_version = $4 AND status IN ('queued', 'processing')`,
|
||||
attachmentID, encoded, now, pipelineVersion)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// ScheduleRetry keeps the job in the durable outbox with a jittered delay.
|
||||
// The caller acks the broker only after this commit, so the retry intent is
|
||||
// never lost between "try later" and ack.
|
||||
func (s *Store) ScheduleRetry(ctx context.Context, jobID uuid.UUID, attempt int, code string, delay time.Duration) (bool, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
now := time.Now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', error_code = $3, available_at = $4,
|
||||
published_at = NULL, lease_until = NULL, updated_at = $5
|
||||
WHERE job_id = $1 AND status = 'processing' AND attempt = $2`,
|
||||
jobID, attempt, code, now.Add(delay), now)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = $2
|
||||
WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1)
|
||||
AND status <> 'ready'`, jobID, now)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
return true, dbError(tx.Commit(ctx))
|
||||
}
|
||||
|
||||
// FailJob records a terminal reason. It is idempotent: a later stale attempt
|
||||
// that no longer matches the current attempt is ignored.
|
||||
func (s *Store) FailJob(ctx context.Context, jobID uuid.UUID, attempt int, code string) error {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
now := time.Now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', error_code = $3, finished_at = $4,
|
||||
lease_until = NULL, updated_at = $4
|
||||
WHERE job_id = $1 AND status = 'processing' AND attempt = $2`,
|
||||
jobID, attempt, code, now)
|
||||
if err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $2, updated_at = $3
|
||||
WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1)
|
||||
AND status <> 'ready'`, jobID, code, now)
|
||||
if err != nil {
|
||||
return dbError(err)
|
||||
}
|
||||
return dbError(tx.Commit(ctx))
|
||||
}
|
||||
|
||||
// RecoverExpiredLeases is the stuck-processing guard. A dead worker leaves a
|
||||
// processing row whose lease expires; the sweeper requeues it or marks it
|
||||
// retry_exhausted. It never touches ready work.
|
||||
func (s *Store) RecoverExpiredLeases(ctx context.Context) (int, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
WITH expired AS (
|
||||
UPDATE attachment_jobs j
|
||||
SET status = CASE WHEN j.attempt >= j.max_attempts THEN 'failed' ELSE 'queued' END,
|
||||
error_code = CASE WHEN j.attempt >= j.max_attempts THEN 'retry_exhausted' ELSE NULL END,
|
||||
available_at = now(),
|
||||
published_at = NULL,
|
||||
lease_until = NULL,
|
||||
finished_at = CASE WHEN j.attempt >= j.max_attempts THEN now() ELSE NULL END,
|
||||
updated_at = now()
|
||||
WHERE j.status = 'processing'
|
||||
AND j.lease_until IS NOT NULL
|
||||
AND j.lease_until < now()
|
||||
RETURNING j.attachment_id, j.status, j.error_code
|
||||
)
|
||||
UPDATE attachments a
|
||||
SET status = CASE WHEN e.status = 'failed' THEN 'failed' ELSE 'queued' END,
|
||||
error_code = CASE WHEN e.status = 'failed' THEN e.error_code ELSE NULL END,
|
||||
updated_at = now()
|
||||
FROM expired e
|
||||
WHERE a.id = e.attachment_id AND a.status <> 'ready'`)
|
||||
if err != nil {
|
||||
return 0, dbError(err)
|
||||
}
|
||||
return int(tag.RowsAffected()), nil
|
||||
}
|
||||
|
||||
// ListPublishableJobs is the outbox read side. It intentionally does not lock
|
||||
// rows: two API replicas may publish the same message, and the idempotent
|
||||
// claim/effect makes that duplicate harmless.
|
||||
func (s *Store) ListPublishableJobs(ctx context.Context, limit int) ([]model.JobRecord, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
|
||||
status, error_code, created_at, available_at, published_at,
|
||||
lease_until, processing_started_at, finished_at, replay_count
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL AND status = 'queued' AND available_at <= now()
|
||||
ORDER BY available_at, created_at
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, dbError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []model.JobRecord
|
||||
for rows.Next() {
|
||||
job, scanErr := scanJob(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, dbError(err)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// MarkJobPublished uses status+attempt as a compare-and-set. If a worker has
|
||||
// already claimed the job, or a retry has superseded this publish, the update
|
||||
// is a no-op and the outbox will publish the current intent later.
|
||||
func (s *Store) MarkJobPublished(ctx context.Context, jobID uuid.UUID, attempt int, at time.Time) error {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET published_at = $3, updated_at = $3
|
||||
WHERE job_id = $1 AND attempt = $2 AND status = 'queued' AND published_at IS NULL`,
|
||||
jobID, attempt, at.UTC())
|
||||
return dbError(err)
|
||||
}
|
||||
|
||||
// ReplayFailed is the explicit operator action. The status guard means a
|
||||
// second click, or a replay after success, cannot duplicate a ready effect.
|
||||
func (s *Store) ReplayFailed(ctx context.Context, attachmentID uuid.UUID) (bool, error) {
|
||||
ctx, cancel := s.withTimeout(ctx)
|
||||
defer cancel()
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
defer rollback(tx)
|
||||
|
||||
now := time.Now().UTC()
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = 0, error_code = NULL,
|
||||
available_at = $2, published_at = NULL, lease_until = NULL,
|
||||
processing_started_at = NULL, finished_at = NULL,
|
||||
replay_count = replay_count + 1, last_replay_at = $2, updated_at = $2
|
||||
WHERE attachment_id = $1 AND status = 'failed'
|
||||
AND EXISTS (SELECT 1 FROM attachments
|
||||
WHERE id = $1 AND status = 'failed')`,
|
||||
attachmentID, now)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = $2
|
||||
WHERE id = $1 AND status = 'failed'`, attachmentID, now)
|
||||
if err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return false, dbError(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func scanJob(row rowScanner) (model.JobRecord, error) {
|
||||
var (
|
||||
job model.JobRecord
|
||||
created time.Time
|
||||
errCode *string
|
||||
)
|
||||
if err := row.Scan(
|
||||
&job.JobID, &job.AttachmentID, &job.Type, &job.PipelineVersion,
|
||||
&job.Attempt, &job.MaxAttempts, &job.Status, &errCode, &created,
|
||||
&job.AvailableAt, &job.PublishedAt, &job.LeaseUntil,
|
||||
&job.ProcessingStartedAt, &job.FinishedAt, &job.ReplayCount,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return model.JobRecord{}, common.NotFound("job not found")
|
||||
}
|
||||
return model.JobRecord{}, dbError(err)
|
||||
}
|
||||
job.SchemaVersion = 1
|
||||
job.CreatedAt = common.RFCFormTime(created.UTC())
|
||||
job.ErrorCode = errCode
|
||||
return job, nil
|
||||
}
|
||||
@@ -78,9 +78,11 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
|
||||
return model.Message{}, dbError(scanErr)
|
||||
}
|
||||
found++
|
||||
if status != "ready" {
|
||||
// Lab5: a queued/processing attachment is already linked to its
|
||||
// chat and may be sent. Only a terminal failed one is refused.
|
||||
if status == "failed" {
|
||||
rows.Close()
|
||||
return model.Message{}, common.Conflict("attachment is not ready")
|
||||
return model.Message{}, common.Conflict("attachment failed processing")
|
||||
}
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// Package worker runs the asynchronous attachment pipeline. It consumes one
|
||||
// logical job, reads trusted metadata from PostgreSQL, loads the staged object,
|
||||
// writes the original and variants, commits ready atomically, and only then
|
||||
// lets the broker ack the message.
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline"
|
||||
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
cfg config.Config
|
||||
store *store.Store
|
||||
objects *objectstore.Client
|
||||
pipeline *pipeline.Processor
|
||||
logger *slog.Logger
|
||||
faultOnce atomic.Bool
|
||||
}
|
||||
|
||||
func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
w := &Worker{
|
||||
cfg: cfg,
|
||||
store: st,
|
||||
objects: objects,
|
||||
pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
|
||||
logger: logger,
|
||||
}
|
||||
w.faultOnce.Store(true)
|
||||
return w
|
||||
}
|
||||
|
||||
// StartLeaseSweeper recovers a worker that died after claiming a job. Without
|
||||
// this, the database would show "processing" forever and no message would be
|
||||
// left to redeliver.
|
||||
func (w *Worker) StartLeaseSweeper(ctx context.Context) {
|
||||
ticker := time.NewTicker(w.cfg.LeaseSweep)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
count, err := w.store.RecoverExpiredLeases(ctx)
|
||||
if err != nil {
|
||||
w.logger.Error("lease sweep failed", "err", err)
|
||||
} else if count > 0 {
|
||||
w.logger.Warn("recovered expired job leases", "count", count)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle is the broker callback. Returning nil means "durable effect or
|
||||
// deliberate no-op; ack". ErrRequeue means the database itself was unavailable.
|
||||
func (w *Worker) Handle(ctx context.Context, message model.Job) error {
|
||||
claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease)
|
||||
if err != nil {
|
||||
if common.IsCode(err, common.ErrorCodeNotFound) {
|
||||
w.logger.Warn("job no longer exists; acking", "job_id", message.JobID)
|
||||
return nil
|
||||
}
|
||||
w.logger.Error("claim job failed", "job_id", message.JobID, "err", err)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
if claim.Terminal || claim.Busy {
|
||||
w.logger.Info("job is already finished or owned",
|
||||
"job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy,
|
||||
"attempt", claim.Job.Attempt)
|
||||
return nil
|
||||
}
|
||||
if claim.Attachment.ID != message.AttachmentID {
|
||||
w.logger.Error("message attachment mismatch; acking poison message",
|
||||
"job_id", message.JobID, "message_attachment", message.AttachmentID,
|
||||
"database_attachment", claim.Attachment.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
log := w.logger.With(
|
||||
"job_id", claim.Job.JobID,
|
||||
"attachment_id", claim.Attachment.ID,
|
||||
"attempt", claim.Job.Attempt,
|
||||
"image_bytes", claim.Attachment.SizeBytes,
|
||||
)
|
||||
if w.cfg.CrashAfterClaim {
|
||||
// Deterministic fault injection for the "claimed but never finished"
|
||||
// window. The lease sweeper requeues the durable job after it expires;
|
||||
// it does not depend on the broker redelivery still being around.
|
||||
w.logger.Error("fault injection: exiting after claim before processing",
|
||||
"job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil)
|
||||
os.Exit(10)
|
||||
}
|
||||
|
||||
log.Info("processing attachment")
|
||||
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err = w.process(jobCtx, claim, log); err != nil {
|
||||
if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) {
|
||||
err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err}
|
||||
}
|
||||
return w.handleFailure(ctx, claim, err, log)
|
||||
}
|
||||
|
||||
log.Info("attachment ready")
|
||||
if w.cfg.CrashAfterEffect {
|
||||
// Deterministic fault injection for the "written but not acked" window.
|
||||
// The unacked delivery is redelivered after the connection dies.
|
||||
w.logger.Error("fault injection: exiting after durable effect before ack",
|
||||
"job_id", claim.Job.JobID)
|
||||
os.Exit(9)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error {
|
||||
if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) {
|
||||
if mode == "transient-once" {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed",
|
||||
Err: errors.New("injected transient failure")}
|
||||
}
|
||||
return &pipeline.Failure{Permanent: true, Code: "invalid_image",
|
||||
Err: errors.New("injected permanent failure")}
|
||||
}
|
||||
|
||||
att := claim.Attachment
|
||||
if att.StagingKey == "" {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_missing",
|
||||
Err: errors.New("attachment has no staging key")}
|
||||
}
|
||||
|
||||
reader, size, err := w.objects.Get(ctx, att.StagingKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, objectstore.ErrNotFound) {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err}
|
||||
}
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
defer reader.Close()
|
||||
if size != att.SizeBytes {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt",
|
||||
Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)}
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part")
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
path := file.Name()
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
_ = os.Remove(path)
|
||||
}()
|
||||
|
||||
hasher := sha256.New()
|
||||
// The same 10 MiB contract limit protects the worker from a corrupt row
|
||||
// pointing at an unexpectedly large object.
|
||||
written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024))
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
if written > w.cfg.MaxUploadBytes {
|
||||
return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")}
|
||||
}
|
||||
if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 {
|
||||
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
|
||||
}
|
||||
|
||||
encoded, err := w.pipeline.Process(ctx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Original bytes are copied unchanged from staging to the final key.
|
||||
if _, err = file.Seek(0, 0); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
|
||||
variants := make([]model.Variant, 0, len(encoded))
|
||||
for _, item := range encoded {
|
||||
key := store.VariantObjectKey(att.ObjectKey, item.Name)
|
||||
if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
|
||||
}
|
||||
variants = append(variants, model.Variant{
|
||||
Name: item.Name,
|
||||
ContentType: item.ContentType,
|
||||
SizeBytes: int64(len(item.Bytes)),
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
SHA256: item.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants)
|
||||
if err != nil {
|
||||
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
if !committed {
|
||||
log.Warn("ready snapshot was superseded; leaving newer state alone")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Staging is removed only after ready commits. A crash before this delete
|
||||
// leaves an object the reconciler will clean after retention.
|
||||
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
log.Warn("staging delete failed; reconciler will retry", "err", err)
|
||||
return nil
|
||||
}
|
||||
if err = w.store.ClearStagingKey(ctx, att.ID); err != nil {
|
||||
log.Warn("clear staging key failed; reconciler will retry", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error {
|
||||
var failure *pipeline.Failure
|
||||
if !errors.As(err, &failure) {
|
||||
failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
|
||||
}
|
||||
|
||||
if failure.Permanent {
|
||||
log.Error("permanent job failure", "code", failure.Code, "err", failure.Err)
|
||||
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil {
|
||||
log.Error("record permanent failure failed; requeueing", "err", failErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if claim.Job.Attempt >= claim.Job.MaxAttempts {
|
||||
log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err)
|
||||
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil {
|
||||
log.Error("record exhausted retry failed; requeueing", "err", failErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
delay := w.backoff(claim.Job.Attempt)
|
||||
log.Warn("transient job failure; scheduling retry",
|
||||
"code", failure.Code, "err", failure.Err, "retry_in", delay)
|
||||
scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay)
|
||||
if retryErr != nil {
|
||||
log.Error("schedule retry failed; requeueing", "err", retryErr)
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
if !scheduled {
|
||||
// This attempt no longer owns the row (for example a lease sweep or a
|
||||
// replay changed it). Requeue rather than ack a job whose future work
|
||||
// was not durably recorded.
|
||||
log.Warn("retry scheduling lost ownership; requeueing")
|
||||
return broker.ErrRequeue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS.
|
||||
// Jitter prevents many retries from hitting a recovered dependency together.
|
||||
func (w *Worker) backoff(attempt int) time.Duration {
|
||||
delay := w.cfg.RetryBase
|
||||
for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ {
|
||||
delay *= 2
|
||||
}
|
||||
if delay > w.cfg.RetryMax {
|
||||
delay = w.cfg.RetryMax
|
||||
}
|
||||
jitter := 0.5 + rand.Float64() // 0.5 .. 1.5
|
||||
return time.Duration(float64(delay) * jitter)
|
||||
}
|
||||
Reference in New Issue
Block a user