feat(lab5): durable async attachment pipeline through grade 4

- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
Mikhail Verkovykh
2026-11-15 00:30:36 +03:00
parent 28d68dc30b
commit 9e43faa536
48 changed files with 4077 additions and 333 deletions
+3 -4
View File
@@ -20,12 +20,9 @@ import (
"github.com/gofiber/fiber/v3"
"github.com/gofiber/fiber/v3/middleware/recover"
"github.com/gofiber/fiber/v3/middleware/requestid"
"github.com/jackc/pgx/v5/pgxpool"
)
func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber.App, error) {
st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL)
func New(cfg config.Config, st *store.Store, sessions *session.Store) (*fiber.App, error) {
objects, err := objectstore.New(objectstore.Config{
Endpoint: cfg.S3Endpoint,
Region: cfg.S3Region,
@@ -74,6 +71,8 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber
attachments.NewValidator(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
cfg.MaxUploadBytes,
cfg.UploadDrainLimit,
cfg.MaxPendingJobs,
cfg.JobMaxAttempts,
cfg.UploadTimeout,
cfg.DownloadTimeout,
cfg.UploadTempDir,
+81 -51
View File
@@ -31,35 +31,31 @@ type Handler struct {
validator *Validator
maxUploadBytes int64
drainLimit int64
maxPendingJobs int
jobMaxAttempts int
uploadTimeout time.Duration
downloadTimeout time.Duration
tempDir string
}
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, maxPendingJobs, jobMaxAttempts int, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
return &Handler{
store: st,
objects: client,
validator: validator,
maxUploadBytes: maxUploadBytes,
drainLimit: drainLimit,
maxPendingJobs: maxPendingJobs,
jobMaxAttempts: jobMaxAttempts,
uploadTimeout: uploadTimeout,
downloadTimeout: downloadTimeout,
tempDir: tempDir,
}
}
// Upload stores raw PNG/JPEG bytes. The order is deliberate:
// 1. authorize before reading the body;
// 2. stream the body to a bounded temporary file while hashing;
// 3. validate pixels and decode;
// 4. commit a pending metadata row;
// 5. PUT the object;
// 6. mark the row ready.
//
// A 201 is therefore returned only after both the object and its metadata are
// durable. Every failure between 4 and 6 has a compensation path, and the
// reconciler covers crashes that prevent compensation from running.
// Upload stages the original, then commits the attachment snapshot and its
// outbox job together. The 202 is therefore a promise about durable bytes plus
// durable intent, not a promise that resizing already happened.
func (h *Handler) Upload(c fiber.Ctx) error {
userID := common.MustUserID(c)
@@ -88,6 +84,19 @@ func (h *Handler) Upload(c fiber.Ctx) error {
return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
}
// Backpressure: refuse new work before reading its bytes. The check is
// approximate across API replicas but bounds staging growth under overload.
if h.maxPendingJobs > 0 {
pending, countErr := h.store.CountPendingJobs(c.Context())
if countErr != nil {
return countErr
}
if pending >= h.maxPendingJobs {
c.Set(fiber.HeaderRetryAfter, "5")
return common.Unavailable("processing queue is full")
}
}
// Reject an obviously oversized request before allocating a temp file or
// decoding anything. Chunked requests are caught by the streaming counter
// below.
@@ -151,6 +160,7 @@ func (h *Handler) Upload(c fiber.Ctx) error {
}
attachmentID := uuid.New()
jobID := uuid.New()
attachment := model.Attachment{
ID: attachmentID,
ChatID: chatID,
@@ -159,55 +169,51 @@ func (h *Handler) Upload(c fiber.Ctx) error {
ContentType: info.ContentType,
SizeBytes: written,
SHA256: hex.EncodeToString(hasher.Sum(nil)),
Status: "pending",
Status: "queued",
Variants: []model.Variant{},
ErrorCode: nil,
CreatedAt: common.NowUTC(),
UpdatedAt: common.NowUTC(),
// AGENT: the key is server-generated from UUIDs and never contains the
// client filename. That keeps traversal and collisions out of the
// object store even before application authorization is considered.
ObjectKey: objectKey(chatID, attachmentID),
}
if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil {
return err
// AGENT: keys are server-generated from UUIDs, never the client
// filename. staging/ is the durable handoff prefix; the worker copies
// the same bytes to the final attachments/ key unchanged.
ObjectKey: objectKey(chatID, attachmentID),
StagingKey: stagingKey(chatID, attachmentID),
PipelineVersion: 1,
}
if _, err = file.Seek(0, 0); err != nil {
h.compensate(attachment.ID, attachment.ObjectKey)
return common.BadRequest("cannot read upload body")
}
if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil {
slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
if err = h.objects.Put(ctx, attachment.StagingKey, file, written, attachment.ContentType); err != nil {
slog.Error("stage attachment object", "err", err, "attachment_id", attachment.ID)
return common.Unavailable("object storage unavailable")
}
updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID)
if err != nil {
slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
return common.Unavailable("attachment metadata unavailable")
job := model.JobRecord{
Job: model.Job{
JobID: jobID,
AttachmentID: attachmentID,
PipelineVersion: attachment.PipelineVersion,
CreatedAt: attachment.CreatedAt,
},
MaxAttempts: h.jobMaxAttempts,
}
if err = h.store.CreateQueuedAttachment(ctx, attachment, job); err != nil {
h.compensateStaging(attachment.StagingKey)
return err
}
attachment.Status = "ready"
attachment.UpdatedAt = common.RFCFormTime(updatedAt)
return c.Status(fiber.StatusCreated).JSON(attachment)
return c.Status(fiber.StatusAccepted).JSON(attachment)
}
// compensate is best effort by design. If either delete fails, the pending row
// or the object remains visible to the reconciler, which retries after the
// configured grace period.
func (h *Handler) compensate(attachmentID uuid.UUID, key string) {
// compensateStaging is best effort. If it fails, the reconciler removes an
// object with no metadata row after the grace period.
func (h *Handler) compensateStaging(key string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID)
}
if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil {
slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID)
slog.Error("compensate: delete staging object", "err", err)
}
}
@@ -223,10 +229,9 @@ func (h *Handler) Get(c fiber.Ctx) error {
if err != nil {
return err
}
if attachment.Status != "ready" {
return common.Conflict("attachment is not ready")
}
// queued/processing snapshots are part of lab5: the client polls this
// endpoint until status becomes ready or failed.
return c.JSON(attachment)
}
@@ -242,7 +247,9 @@ func (h *Handler) Download(c fiber.Ctx) error {
if variant == "" {
variant = "original"
}
if variant != "original" {
switch variant {
case "original", "thumbnail", "crop", "optimized":
default:
return common.BadRequest("unsupported variant")
}
@@ -254,9 +261,28 @@ func (h *Handler) Download(c fiber.Ctx) error {
return common.Conflict("attachment is not ready")
}
key := attachment.ObjectKey
contentType := attachment.ContentType
sizeBytes := attachment.SizeBytes
if variant != "original" {
found := false
for _, candidate := range attachment.Variants {
if candidate.Name == variant {
key = store.VariantObjectKey(attachment.ObjectKey, candidate.Name)
contentType = candidate.ContentType
sizeBytes = candidate.SizeBytes
found = true
break
}
}
if !found {
return common.NotFound("attachment variant not found")
}
}
ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout)
body, size, err := h.objects.Get(ctx, attachment.ObjectKey)
body, size, err := h.objects.Get(ctx, key)
if err != nil {
cancel()
if errors.Is(err, objectstore.ErrNotFound) {
@@ -266,16 +292,16 @@ func (h *Handler) Download(c fiber.Ctx) error {
slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID)
return common.Unavailable("object storage unavailable")
}
if size != attachment.SizeBytes {
if size != sizeBytes {
_ = body.Close()
cancel()
slog.Error("attachment object size mismatch",
"attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size)
"attachment_id", attachment.ID, "metadata_size", sizeBytes, "object_size", size)
return common.Unavailable("attachment content is inconsistent")
}
c.Set(fiber.HeaderContentType, attachment.ContentType)
c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10))
c.Set(fiber.HeaderContentType, contentType)
c.Set(fiber.HeaderContentLength, strconv.FormatInt(sizeBytes, 10))
c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
c.Set(fiber.HeaderCacheControl, "private, no-store")
c.Set("X-Content-Type-Options", "nosniff")
@@ -307,6 +333,10 @@ func objectKey(chatID, attachmentID uuid.UUID) string {
return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID)
}
func stagingKey(chatID, attachmentID uuid.UUID) string {
return fmt.Sprintf("staging/%s/%s", chatID, attachmentID)
}
// CleanupStaleUploads removes temp files abandoned by a previous crash. A
// running request removes its own file in a defer; this only handles the
// process-died-mid-upload case.
+253
View File
@@ -0,0 +1,253 @@
// Package broker wraps RabbitMQ delivery and publisher confirms. PostgreSQL is
// still the source of truth: a message carries only a job id and a snapshot of
// the durable job intent.
package broker
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"sync"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
)
// ErrRequeue asks the consumer to nack and requeue. The worker uses it only
// when its durable state could not be written (for example PostgreSQL is down).
var ErrRequeue = errors.New("requeue delivery")
func dial(url string) (*amqp.Connection, error) {
return amqp.DialConfig(url, amqp.Config{
Heartbeat: 10 * time.Second,
Locale: "en_US",
})
}
func declareQueue(ch *amqp.Channel, queue string) error {
// A quorum queue uses a durable replicated log on the broker. Delivery is
// still at least once; idempotency lives in the job effect key.
_, err := ch.QueueDeclare(queue, true, false, false, false, amqp.Table{
"x-queue-type": "quorum",
})
return err
}
// Publisher is safe for concurrent use. It keeps one confirm-mode channel and
// reconnects lazily after a broker restart.
type Publisher struct {
url string
queue string
mu sync.Mutex
conn *amqp.Connection
ch *amqp.Channel
}
func NewPublisher(url, queue string) *Publisher {
return &Publisher{url: url, queue: queue}
}
func (p *Publisher) ensure() error {
if p.ch != nil && !p.ch.IsClosed() {
return nil
}
p.closeLocked()
conn, err := dial(p.url)
if err != nil {
return err
}
ch, err := conn.Channel()
if err != nil {
_ = conn.Close()
return err
}
if err = ch.Confirm(false); err != nil {
_ = ch.Close()
_ = conn.Close()
return err
}
if err = declareQueue(ch, p.queue); err != nil {
_ = ch.Close()
_ = conn.Close()
return err
}
p.conn, p.ch = conn, ch
return nil
}
// Publish waits for the broker's publisher confirm. A nil return means the
// broker accepted the message, not that a worker executed it.
func (p *Publisher) Publish(ctx context.Context, job model.Job) error {
body, err := json.Marshal(job)
if err != nil {
return err
}
p.mu.Lock()
defer p.mu.Unlock()
if err = p.ensure(); err != nil {
return err
}
confirmation, err := p.ch.PublishWithDeferredConfirmWithContext(ctx, "", p.queue, true, false, amqp.Publishing{
DeliveryMode: amqp.Persistent,
ContentType: "application/json",
MessageId: job.JobID.String(),
Timestamp: time.Now().UTC(),
Type: job.Type,
Body: body,
})
if err != nil {
p.closeLocked()
return err
}
confirmed, err := confirmation.WaitContext(ctx)
if err != nil {
p.closeLocked()
return err
}
if !confirmed {
p.closeLocked()
return fmt.Errorf("publisher confirm was negative")
}
return nil
}
func (p *Publisher) Close() {
p.mu.Lock()
defer p.mu.Unlock()
p.closeLocked()
}
func (p *Publisher) closeLocked() {
if p.ch != nil {
_ = p.ch.Close()
}
if p.conn != nil {
_ = p.conn.Close()
}
p.ch, p.conn = nil, nil
}
// Handler must return nil after the durable effect (and optional ack); the
// consumer acks then. ErrRequeue leaves the message for redelivery.
type Handler func(context.Context, model.Job) error
type Consumer struct {
url string
queue string
logger *slog.Logger
}
func NewConsumer(url, queue string, logger *slog.Logger) *Consumer {
if logger == nil {
logger = slog.Default()
}
return &Consumer{url: url, queue: queue, logger: logger}
}
// Run reconnects until ctx is cancelled. concurrency bounds simultaneous
// handlers in this process; prefetch bounds unacked messages the broker may
// push. The two knobs are intentionally separate for the lab5 experiment.
func (c *Consumer) Run(ctx context.Context, concurrency, prefetch int, handler Handler) error {
for {
if ctx.Err() != nil {
return nil
}
err := c.consumeOnce(ctx, concurrency, prefetch, handler)
if ctx.Err() != nil {
return nil
}
c.logger.Error("rabbitmq consumer disconnected", "err", err)
select {
case <-ctx.Done():
return nil
case <-time.After(2 * time.Second):
}
}
}
func (c *Consumer) consumeOnce(ctx context.Context, concurrency, prefetch int, handler Handler) error {
conn, err := dial(c.url)
if err != nil {
return err
}
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
return err
}
defer ch.Close()
if err = declareQueue(ch, c.queue); err != nil {
return err
}
if err = ch.Qos(prefetch, 0, false); err != nil {
return err
}
deliveries, err := ch.Consume(c.queue, "", false, false, false, false, nil)
if err != nil {
return err
}
sem := make(chan struct{}, concurrency)
var workers sync.WaitGroup
stop := make(chan struct{})
go func() {
select {
case <-ctx.Done():
case <-stop:
}
_ = ch.Cancel("", false)
}()
for {
select {
case <-ctx.Done():
close(stop)
workers.Wait()
return nil
case delivery, ok := <-deliveries:
if !ok {
close(stop)
workers.Wait()
return errors.New("delivery channel closed")
}
var job model.Job
if err = json.Unmarshal(delivery.Body, &job); err != nil {
c.logger.Error("drop malformed job message", "err", err)
_ = delivery.Ack(false)
continue
}
sem <- struct{}{}
workers.Add(1)
go func(delivery amqp.Delivery, job model.Job) {
defer workers.Done()
defer func() { <-sem }()
handlerErr := handler(ctx, job)
switch {
case handlerErr == nil:
if err := delivery.Ack(false); err != nil {
c.logger.Error("ack failed", "job_id", job.JobID, "err", err)
}
case errors.Is(handlerErr, ErrRequeue):
// A short sleep avoids a tight redelivery loop while a
// dependency is down. The lease/outbox still own the job.
time.Sleep(time.Second)
if err := delivery.Nack(false, true); err != nil {
c.logger.Error("nack failed", "job_id", job.JobID, "err", err)
}
default:
c.logger.Error("job handler failed", "job_id", job.JobID, "err", handlerErr)
_ = delivery.Ack(false)
}
}(delivery, job)
}
}
}
+22
View File
@@ -2,6 +2,7 @@ package common
import (
"encoding/json"
"fmt"
"time"
)
@@ -22,6 +23,27 @@ func (t RFCFormTime) MarshalJSON() ([]byte, error) {
return json.Marshal(utc.Format(layout))
}
// UnmarshalJSON accepts the same RFC 3339 strings MarshalJSON emits. The
// broker round-trips Job values through JSON, and a named time.Time type does
// not inherit encoding/json's time.Time special case on decode.
func (t *RFCFormTime) UnmarshalJSON(data []byte) error {
if string(data) == "null" {
return nil
}
var raw string
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
parsed, err := time.Parse(time.RFC3339Nano, raw)
if err != nil {
return fmt.Errorf("parse RFC form time: %w", err)
}
*t = RFCFormTime(parsed)
return nil
}
func NowUTC() RFCFormTime {
return RFCFormTime(time.Now().UTC().Truncate(storageTimePrecision))
}
+9
View File
@@ -43,6 +43,15 @@ func TestRFCFormTimeSerializesFractionalSeconds(t *testing.T) {
if string(encoded) != tc.want {
t.Fatalf("RFC form time = %s, want %s", encoded, tc.want)
}
var decoded RFCFormTime
if err = json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("unmarshal RFC form time: %v", err)
}
want := tc.value.UTC().Truncate(storageTimePrecision)
if !time.Time(decoded).Equal(want) {
t.Fatalf("round trip = %v, want %v", time.Time(decoded), want)
}
})
}
}
+102
View File
@@ -9,6 +9,8 @@ import (
"strconv"
"strings"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
)
const (
@@ -60,6 +62,26 @@ type Config struct {
ReconcileGrace time.Duration
ReconcileInterval time.Duration
// Lab5 asynchronous pipeline. BrokerURL is required by the API (outbox
// publisher) and the worker (consumer), but not by the reconciler.
BrokerURL string
WorkerOnly bool
JobQueue string
JobMaxAttempts int
JobTimeout time.Duration
JobLease time.Duration
RetryBase time.Duration
RetryMax time.Duration
WorkerConcurrency int
WorkerPrefetch int
MaxPendingJobs int
LeaseSweep time.Duration
StagingRetention time.Duration
WorkTempDir string
WorkerFaultMode string
CrashAfterEffect bool
CrashAfterClaim bool
}
func Load() (Config, error) {
@@ -141,6 +163,75 @@ func Load() (Config, error) {
}
cfg.ReconcileInterval = time.Duration(reconcileInterval) * time.Second
if cfg.WorkerOnly, err = envBool("WORKER_ONLY", false); err != nil {
return Config{}, err
}
if cfg.CrashAfterEffect, err = envBool("WORKER_CRASH_AFTER_EFFECT", false); err != nil {
return Config{}, err
}
if cfg.CrashAfterClaim, err = envBool("WORKER_CRASH_AFTER_CLAIM", false); err != nil {
return Config{}, err
}
jobAttempts, err := envInt("JOB_MAX_ATTEMPTS", 5, 1, 100)
if err != nil {
return Config{}, err
}
jobTimeout, err := envInt("JOB_TIMEOUT_SECONDS", 60, 5, 3600)
if err != nil {
return Config{}, err
}
jobLease, err := envInt("JOB_LEASE_SECONDS", 90, 10, 7200)
if err != nil {
return Config{}, err
}
retryBase, err := envInt("JOB_RETRY_BASE_SECONDS", 2, 1, 3600)
if err != nil {
return Config{}, err
}
retryMax, err := envInt("JOB_RETRY_MAX_SECONDS", 30, 1, 7200)
if err != nil {
return Config{}, err
}
concurrency, err := envInt("WORKER_CONCURRENCY", 2, 1, 128)
if err != nil {
return Config{}, err
}
prefetch, err := envInt("WORKER_PREFETCH", 2, 1, 512)
if err != nil {
return Config{}, err
}
maxPending, err := envInt("MAX_PENDING_JOBS", 1000, 0, 0)
if err != nil {
return Config{}, err
}
leaseSweep, err := envInt("LEASE_SWEEP_SECONDS", 15, 1, 3600)
if err != nil {
return Config{}, err
}
stagingRetention, err := envInt("STAGING_RETENTION_SECONDS", 86400, 60, 0)
if err != nil {
return Config{}, err
}
cfg.JobQueue = envOr("JOB_QUEUE", model.JobTypeAttachmentProcessV1)
cfg.JobMaxAttempts = jobAttempts
cfg.JobTimeout = time.Duration(jobTimeout) * time.Second
cfg.JobLease = time.Duration(jobLease) * time.Second
cfg.RetryBase = time.Duration(retryBase) * time.Second
cfg.RetryMax = time.Duration(retryMax) * time.Second
cfg.WorkerConcurrency = concurrency
cfg.WorkerPrefetch = prefetch
cfg.MaxPendingJobs = maxPending
cfg.LeaseSweep = time.Duration(leaseSweep) * time.Second
cfg.StagingRetention = time.Duration(stagingRetention) * time.Second
cfg.WorkTempDir = envOr("WORK_TEMP_DIR", filepath.Join(os.TempDir(), "messenger-work"))
cfg.WorkerFaultMode = envOr("WORKER_FAULT_MODE", "none")
switch cfg.WorkerFaultMode {
case "none", "transient-once", "permanent-once":
default:
return Config{}, fmt.Errorf("WORKER_FAULT_MODE must be none, transient-once or permanent-once")
}
if cfg.DatabaseURL, err = secret("DATABASE_URL"); err != nil {
return Config{}, err
}
@@ -175,6 +266,17 @@ func Load() (Config, error) {
return cfg, nil
}
if cfg.BrokerURL, err = secret("BROKER_URL"); err != nil {
return Config{}, err
}
if cfg.BrokerURL == "" {
return Config{}, fmt.Errorf("BROKER_URL or BROKER_URL_FILE is required")
}
if cfg.WorkerOnly {
return cfg, nil
}
if cfg.CacheURL, err = secret("CACHE_URL"); err != nil {
return Config{}, err
}
@@ -0,0 +1,27 @@
DROP TABLE IF EXISTS attachment_jobs;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_variants_array;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_pipeline_version;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
-- Collapse asynchronous states back to lab4's single in-flight state. Ready
-- and failed rows keep their meaning; queued/processing work is abandoned.
UPDATE attachments
SET status = 'pending', error_code = NULL
WHERE status IN ('queued', 'processing');
ALTER TABLE attachments
DROP COLUMN IF EXISTS variants,
DROP COLUMN IF EXISTS pipeline_version,
DROP COLUMN IF EXISTS staging_key,
DROP COLUMN IF EXISTS ready_at;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status
CHECK (status IN ('pending', 'ready', 'failed')),
ADD CONSTRAINT attachments_error_state CHECK (
(status = 'pending' AND error_code IS NULL) OR
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
@@ -0,0 +1,76 @@
-- Lab5 turns attachment processing into a durable asynchronous job.
-- The API stages bytes in S3, then in ONE transaction writes the attachment
-- snapshot and its outbox/job row. A separate publisher moves the intent to
-- RabbitMQ, so a broker outage cannot lose an accepted upload.
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status;
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_error_state;
-- Lab4 "pending" meant "object PUT in progress". Nothing can finish it now,
-- because lab5 acceptors write "queued" plus a job. Preserve the row for the
-- operator instead of silently deleting it.
UPDATE attachments
SET status = 'failed', error_code = 'legacy_interrupted', updated_at = now()
WHERE status = 'pending';
ALTER TABLE attachments
ADD COLUMN IF NOT EXISTS staging_key text,
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN IF NOT EXISTS variants jsonb NOT NULL DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS ready_at timestamptz;
UPDATE attachments SET ready_at = updated_at WHERE status = 'ready' AND ready_at IS NULL;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD CONSTRAINT attachments_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
),
ADD CONSTRAINT attachments_variants_array
CHECK (jsonb_typeof(variants) = 'array'),
ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1);
-- One job per logical attachment effect. published_at IS NULL is the outbox
-- marker: a committed intent that has not received a publisher confirm.
CREATE TABLE IF NOT EXISTS attachment_jobs (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE,
type text NOT NULL,
pipeline_version integer NOT NULL,
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL,
status text NOT NULL,
error_code text,
available_at timestamptz NOT NULL,
published_at timestamptz,
lease_until timestamptz,
processing_started_at timestamptz,
finished_at timestamptz,
replay_count integer NOT NULL DEFAULT 0,
last_replay_at timestamptz,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
CONSTRAINT attachment_jobs_status
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachment_jobs_attempts CHECK (attempt >= 0 AND max_attempts >= 1),
CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT attachment_jobs_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
),
CONSTRAINT attachment_jobs_finished CHECK (
(status IN ('queued', 'processing') AND finished_at IS NULL) OR
(status IN ('ready', 'failed') AND finished_at IS NOT NULL)
)
);
CREATE INDEX IF NOT EXISTS attachment_jobs_outbox_idx
ON attachment_jobs (available_at, created_at)
WHERE published_at IS NULL AND status = 'queued';
CREATE INDEX IF NOT EXISTS attachment_jobs_lease_idx
ON attachment_jobs (lease_until)
WHERE status = 'processing';
CREATE INDEX IF NOT EXISTS attachment_jobs_status_idx
ON attachment_jobs (status, created_at);
@@ -0,0 +1,24 @@
-- Returning to the stricter 0004 shape requires clearing retry reasons first.
UPDATE attachment_jobs
SET error_code = NULL
WHERE status IN ('queued', 'processing');
UPDATE attachments
SET error_code = NULL
WHERE status IN ('queued', 'processing');
ALTER TABLE attachments
DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments
ADD CONSTRAINT attachments_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
ALTER TABLE attachment_jobs
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
ALTER TABLE attachment_jobs
ADD CONSTRAINT attachment_jobs_error_state CHECK (
(status IN ('queued', 'processing', 'ready') AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
);
@@ -0,0 +1,23 @@
-- ScheduleRetry records the transient reason on the job while it waits for the
-- next attempt. The original 0004 constraint only allowed error_code on failed
-- rows, so a jittered retry update was rejected and the broker redelivery was
-- then acknowledged as "busy", leaving the job processing until lease expiry.
-- Ready stays clean; failed stays explainable; queued/processing may carry the
-- most recent transient reason.
ALTER TABLE attachments
DROP CONSTRAINT IF EXISTS attachments_error_state;
ALTER TABLE attachments
ADD CONSTRAINT attachments_error_state CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
(status IN ('queued', 'processing'))
);
ALTER TABLE attachment_jobs
DROP CONSTRAINT IF EXISTS attachment_jobs_error_state;
ALTER TABLE attachment_jobs
ADD CONSTRAINT attachment_jobs_error_state CHECK (
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL) OR
(status IN ('queued', 'processing'))
);
+1 -1
View File
@@ -12,7 +12,7 @@ import (
// latestMigrationVersion is the only value to bump when a migration is added.
// The rest of the test derives filenames and ordering from the directory.
const latestMigrationVersion = 3
const latestMigrationVersion = 5
var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`)
+47
View File
@@ -0,0 +1,47 @@
package model
import (
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"github.com/google/uuid"
)
// Job is the broker payload described by contracts/jobs.schema.json. It never
// contains image bytes, bucket keys or credentials: the worker reloads trusted
// metadata from PostgreSQL by attachment_id.
type Job struct {
SchemaVersion int `json:"schema_version"`
JobID uuid.UUID `json:"job_id"`
Type string `json:"type"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt common.RFCFormTime `json:"created_at"`
}
const (
JobTypeAttachmentProcessV1 = "attachment.process.v1"
JobStatusQueued = "queued"
JobStatusProcessing = "processing"
JobStatusReady = "ready"
JobStatusFailed = "failed"
)
// JobRecord is the durable outbox row plus the operator-visible job state.
type JobRecord struct {
Job
Status string
// AvailableAt is the earliest time the outbox may publish a retry.
AvailableAt time.Time
// PublishedAt nil means the outbox has not received a publisher confirm
// yet. This is the recovery marker for a crash between COMMIT and publish.
PublishedAt *time.Time
LeaseUntil *time.Time
ProcessingStartedAt *time.Time
FinishedAt *time.Time
MaxAttempts int
ErrorCode *string
ReplayCount int
}
+4 -2
View File
@@ -56,8 +56,10 @@ type Attachment struct {
CreatedAt common.RFCFormTime `json:"created_at"`
UpdatedAt common.RFCFormTime `json:"updated_at"`
// ObjectKey is internal and never serialized.
ObjectKey string `json:"-"`
// ObjectKey and StagingKey are internal and never serialized.
ObjectKey string `json:"-"`
StagingKey string `json:"-"`
PipelineVersion int `json:"-"`
}
func (a Attachment) GetID() uuid.UUID { return a.ID }
+85
View File
@@ -0,0 +1,85 @@
// Package outbox publishes committed job intents to RabbitMQ. The database row
// is the outbox: published_at IS NULL means "this intent still needs a broker
// confirm". A crash between COMMIT and publish is recovered by the next loop.
package outbox
import (
"context"
"fmt"
"log/slog"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
const batchSize = 20
type Publisher struct {
store *store.Store
broker *broker.Publisher
logger *slog.Logger
}
func New(st *store.Store, b *broker.Publisher, logger *slog.Logger) *Publisher {
if logger == nil {
logger = slog.Default()
}
return &Publisher{store: st, broker: b, logger: logger}
}
func (p *Publisher) Run(ctx context.Context) {
ticker := time.NewTicker(250 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := p.PublishBatch(ctx); err != nil {
p.logger.Error("outbox publish pass failed", "err", err)
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}
}
}
func (p *Publisher) PublishBatch(ctx context.Context) error {
jobs, err := p.store.ListPublishableJobs(ctx, batchSize)
if err != nil {
return err
}
for _, record := range jobs {
if err = p.publish(ctx, record); err != nil {
return fmt.Errorf("publish job %s: %w", record.JobID, err)
}
}
return nil
}
func (p *Publisher) publish(ctx context.Context, record model.JobRecord) error {
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
message := model.Job{
SchemaVersion: 1,
JobID: record.JobID,
Type: record.Type,
AttachmentID: record.AttachmentID,
PipelineVersion: record.PipelineVersion,
Attempt: record.Attempt + 1,
CreatedAt: record.CreatedAt,
}
if err := p.broker.Publish(publishCtx, message); err != nil {
return err
}
// Compare-and-set on attempt/status: a worker may have claimed it, and a
// retry may have been scheduled while the confirm was in flight.
return p.store.MarkJobPublished(publishCtx, record.JobID, record.Attempt, time.Now().UTC())
}
+162
View File
@@ -0,0 +1,162 @@
package pipeline
import (
"encoding/binary"
"image"
"io"
)
// jpegOrientation returns the EXIF orientation (1..8) of a JPEG, or 1 when the
// file has no usable EXIF block. Go's image/jpeg intentionally ignores EXIF,
// so the worker applies the transform after decoding. The original bytes are
// never rewritten.
func jpegOrientation(r io.ReadSeeker) int {
defer func() { _, _ = r.Seek(0, 0) }()
if _, err := r.Seek(0, 0); err != nil {
return 1
}
var soi [2]byte
if _, err := io.ReadFull(r, soi[:]); err != nil || soi[0] != 0xff || soi[1] != 0xd8 {
return 1
}
for {
var prefix [1]byte
if _, err := io.ReadFull(r, prefix[:]); err != nil || prefix[0] != 0xff {
return 1
}
marker := byte(0xff)
for marker == 0xff {
if _, err := io.ReadFull(r, prefix[:]); err != nil {
return 1
}
marker = prefix[0]
}
// Markers without a length field, and start-of-scan where metadata
// after it cannot be an APP1 segment we care about.
if marker == 0xd8 || marker == 0xd9 || marker == 0xda ||
marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7) {
return 1
}
var lengthBytes [2]byte
if _, err := io.ReadFull(r, lengthBytes[:]); err != nil {
return 1
}
length := int(binary.BigEndian.Uint16(lengthBytes[:]))
if length < 2 {
return 1
}
payloadLength := length - 2
if marker != 0xe1 || payloadLength < 6 {
if _, err := r.Seek(int64(payloadLength), io.SeekCurrent); err != nil {
return 1
}
continue
}
payload := make([]byte, payloadLength)
if _, err := io.ReadFull(r, payload); err != nil {
return 1
}
if string(payload[:6]) == "Exif\x00\x00" {
return parseTIFFOrientation(payload[6:])
}
}
}
func parseTIFFOrientation(data []byte) int {
if len(data) < 8 {
return 1
}
var order binary.ByteOrder
switch string(data[:2]) {
case "II":
order = binary.LittleEndian
case "MM":
order = binary.BigEndian
default:
return 1
}
if order.Uint16(data[2:4]) != 42 {
return 1
}
ifdOffset := int(order.Uint32(data[4:8]))
if ifdOffset < 8 || ifdOffset+2 > len(data) {
return 1
}
count := int(order.Uint16(data[ifdOffset : ifdOffset+2]))
position := ifdOffset + 2
for index := 0; index < count; index++ {
if position+12 > len(data) {
return 1
}
entry := data[position : position+12]
tag := order.Uint16(entry[0:2])
fieldType := order.Uint16(entry[2:4])
items := order.Uint32(entry[4:8])
if tag == 0x0112 && fieldType == 3 && items >= 1 {
if items == 1 {
value := int(order.Uint16(entry[8:10]))
if value >= 1 && value <= 8 {
return value
}
return 1
}
offset := int(order.Uint32(entry[8:12]))
if offset+2 > len(data) {
return 1
}
value := int(order.Uint16(data[offset : offset+2]))
if value >= 1 && value <= 8 {
return value
}
return 1
}
position += 12
}
return 1
}
// orient applies the eight EXIF orientation transforms. Every branch returns a
// new image except orientation 1, so the caller can treat the result as
// read-only.
func orient(src image.Image, orientation int) image.Image {
if orientation <= 1 || orientation > 8 {
return src
}
bounds := src.Bounds()
width, height := bounds.Dx(), bounds.Dy()
swap := orientation >= 5
outWidth, outHeight := width, height
if swap {
outWidth, outHeight = height, width
}
dst := image.NewRGBA(image.Rect(0, 0, outWidth, outHeight))
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
var nx, ny int
switch orientation {
case 2: // mirror horizontal
nx, ny = width-1-x, y
case 3: // rotate 180
nx, ny = width-1-x, height-1-y
case 4: // mirror vertical
nx, ny = x, height-1-y
case 5: // transpose
nx, ny = y, x
case 6: // rotate 90 clockwise
nx, ny = height-1-y, x
case 7: // transverse
nx, ny = height-1-y, width-1-x
case 8: // rotate 270 clockwise
nx, ny = y, width-1-x
}
dst.Set(nx, ny, src.At(bounds.Min.X+x, bounds.Min.Y+y))
}
}
return dst
}
+252
View File
@@ -0,0 +1,252 @@
// Package pipeline turns one validated original image into the required lab5
// variants. It is deterministic and side-effect free so a broker redelivery can
// simply overwrite the same keys with the same bytes.
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"image"
"image/color"
"image/jpeg"
_ "image/png"
"os"
"github.com/gen2brain/webp"
"golang.org/x/image/draw"
"golang.org/x/sync/semaphore"
)
const (
ThumbnailMax = 256
CropSize = 128
OptimizedMax = 1280
// Re-encoding drops EXIF/XMP/ICC payloads from variants. The original is
// copied byte-for-byte and keeps its metadata and checksum.
JPEGQuality = 85
WebPQuality = 82
WebPMethod = 4
)
type Processor struct {
maxPixels int64
budgetBytes int64
decodeBudget *semaphore.Weighted
}
// Encoded is one variant ready to be uploaded.
type Encoded struct {
Name string
ContentType string
Bytes []byte
Width int
Height int
SHA256 string
}
func New(maxPixels, decodeBudgetBytes int64) *Processor {
return &Processor{
maxPixels: maxPixels,
budgetBytes: decodeBudgetBytes,
decodeBudget: semaphore.NewWeighted(decodeBudgetBytes),
}
}
// Failure classifies pipeline errors for the retry policy. A corrupt image is
// permanent (retrying cannot help); I/O, lease expiry and dependency errors are
// transient.
type Failure struct {
Permanent bool
Code string
Err error
}
func (f *Failure) Error() string {
if f.Err == nil {
return f.Code
}
return f.Code + ": " + f.Err.Error()
}
func (f *Failure) Unwrap() error { return f.Err }
func permanent(code string, err error) *Failure {
return &Failure{Permanent: true, Code: code, Err: err}
}
func transient(code string, err error) *Failure {
return &Failure{Permanent: false, Code: code, Err: err}
}
// Process decodes the file once and returns all variants in memory. Worker
// concurrency and IMAGE_DECODE_MEMORY_MB bound the number of simultaneous
// decoded images, which is the memory-heavy part of the experiment.
func (p *Processor) Process(ctx context.Context, file *os.File) ([]Encoded, error) {
img, err := p.decode(ctx, file)
if err != nil {
return nil, err
}
thumbnail, err := renderJPEG("thumbnail", fitOnWhite(img, ThumbnailMax, ThumbnailMax))
if err != nil {
return nil, permanent("processing_failed", err)
}
crop, err := renderJPEG("crop", centerSquare(img, CropSize))
if err != nil {
return nil, permanent("processing_failed", err)
}
optimized, err := renderWebP(fitOnWhite(img, OptimizedMax, OptimizedMax))
if err != nil {
return nil, permanent("processing_failed", err)
}
return []Encoded{thumbnail, crop, optimized}, nil
}
func (p *Processor) decode(ctx context.Context, file *os.File) (image.Image, error) {
if _, err := file.Seek(0, 0); err != nil {
return nil, transient("storage_read_failed", err)
}
cfg, format, err := image.DecodeConfig(file)
if err != nil {
return nil, permanent("invalid_image", err)
}
if format != "png" && format != "jpeg" {
return nil, permanent("invalid_image", fmt.Errorf("unsupported format %q", format))
}
if cfg.Width <= 0 || cfg.Height <= 0 {
return nil, permanent("invalid_image", errors.New("empty image bounds"))
}
pixels := int64(cfg.Width) * int64(cfg.Height)
if pixels > p.maxPixels {
return nil, permanent("invalid_image", errors.New("image exceeds MAX_IMAGE_PIXELS"))
}
// AGENT: the semaphore is acquired before Decode, not after. A decoder
// allocates roughly width*height*4 bytes; without this gate several 40 MP
// files can OOM the worker even with a small HTTP concurrency.
weight := pixels * 4
if weight > p.budgetBytes {
return nil, permanent("invalid_image", errors.New("image exceeds decode memory budget"))
}
if err = p.decodeBudget.Acquire(ctx, weight); err != nil {
return nil, transient("processing_timeout", err)
}
defer p.decodeBudget.Release(weight)
orientation := jpegOrientation(file)
if _, err = file.Seek(0, 0); err != nil {
return nil, transient("storage_read_failed", err)
}
img, decodeErr := safeDecode(file)
if decodeErr != nil {
return nil, permanent("invalid_image", decodeErr)
}
bounds := img.Bounds()
if bounds.Dx() != cfg.Width || bounds.Dy() != cfg.Height {
return nil, permanent("invalid_image", errors.New("decoded bounds differ from header"))
}
return orient(img, orientation), nil
}
func safeDecode(file *os.File) (img image.Image, err error) {
defer func() {
if recovered := recover(); recovered != nil {
img = nil
err = fmt.Errorf("image decoder panic: %v", recovered)
}
}()
img, _, err = image.Decode(file)
return img, err
}
// fitOnWhite scales down to fit maxWidth x maxHeight, never up. The source is
// composited over white, which is the contract for JPEG variants with alpha.
func fitOnWhite(src image.Image, maxWidth, maxHeight int) *image.RGBA {
bounds := src.Bounds()
width, height := fitSize(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight)
dst := image.NewRGBA(image.Rect(0, 0, width, height))
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
draw.CatmullRom.Scale(dst, dst.Bounds(), src, bounds, draw.Over, nil)
return dst
}
// centerSquare always produces CropSize x CropSize. Unlike thumbnails, the
// contract allows upscaling tiny sources for the crop variant.
func centerSquare(src image.Image, size int) *image.RGBA {
bounds := src.Bounds()
side := bounds.Dx()
if bounds.Dy() < side {
side = bounds.Dy()
}
x := bounds.Min.X + (bounds.Dx()-side)/2
y := bounds.Min.Y + (bounds.Dy()-side)/2
cropRect := image.Rect(x, y, x+side, y+side)
dst := image.NewRGBA(image.Rect(0, 0, size, size))
draw.Draw(dst, dst.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
draw.CatmullRom.Scale(dst, dst.Bounds(), src, cropRect, draw.Over, nil)
return dst
}
// fitSize preserves aspect ratio, rounds down and clamps to at least 1 px.
// It deliberately never enlarges a small source.
func fitSize(width, height, maxWidth, maxHeight int) (int, int) {
if width <= maxWidth && height <= maxHeight {
return width, height
}
if width*maxHeight >= height*maxWidth {
newHeight := height * maxWidth / width
if newHeight < 1 {
newHeight = 1
}
return maxWidth, newHeight
}
newWidth := width * maxHeight / height
if newWidth < 1 {
newWidth = 1
}
return newWidth, maxHeight
}
func renderJPEG(name string, img image.Image) (Encoded, error) {
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: JPEGQuality}); err != nil {
return Encoded{}, err
}
bounds := img.Bounds()
return Encoded{
Name: name,
ContentType: "image/jpeg",
Bytes: buf.Bytes(),
Width: bounds.Dx(),
Height: bounds.Dy(),
SHA256: digest(buf.Bytes()),
}, nil
}
func renderWebP(img image.Image) (Encoded, error) {
var buf bytes.Buffer
if err := webp.Encode(&buf, img, webp.Options{Quality: WebPQuality, Method: WebPMethod}); err != nil {
return Encoded{}, err
}
bounds := img.Bounds()
return Encoded{
Name: "optimized",
ContentType: "image/webp",
Bytes: buf.Bytes(),
Width: bounds.Dx(),
Height: bounds.Dy(),
SHA256: digest(buf.Bytes()),
}, nil
}
func digest(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
+177
View File
@@ -0,0 +1,177 @@
package pipeline
import (
"bytes"
"context"
"encoding/binary"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"path/filepath"
"testing"
)
func TestFitSizePreservesAspectAndNeverUpscales(t *testing.T) {
cases := []struct {
w, h, maxW, maxH int
wantW, wantH int
}{
{320, 200, 256, 256, 256, 160},
{200, 320, 256, 256, 160, 256},
{100, 50, 256, 256, 100, 50},
{4000, 1000, 1280, 1280, 1280, 320},
{1000, 4000, 256, 256, 64, 256},
{1, 4000, 256, 256, 1, 256},
}
for _, tc := range cases {
w, h := fitSize(tc.w, tc.h, tc.maxW, tc.maxH)
if w != tc.wantW || h != tc.wantH {
t.Fatalf("fitSize(%d,%d,%d,%d) = %dx%d, want %dx%d",
tc.w, tc.h, tc.maxW, tc.maxH, w, h, tc.wantW, tc.wantH)
}
}
}
func TestProcessProducesContractVariants(t *testing.T) {
file := pngFile(t, 320, 200, false)
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
if len(variants) != 3 {
t.Fatalf("got %d variants, want 3", len(variants))
}
byName := map[string]Encoded{}
for _, variant := range variants {
byName[variant.Name] = variant
}
thumbnail := byName["thumbnail"]
if thumbnail.ContentType != "image/jpeg" || thumbnail.Width != 256 || thumbnail.Height != 160 {
t.Fatalf("thumbnail = %s %dx%d, want image/jpeg 256x160",
thumbnail.ContentType, thumbnail.Width, thumbnail.Height)
}
decoded, err := jpeg.Decode(bytes.NewReader(thumbnail.Bytes))
if err != nil {
t.Fatalf("decode thumbnail: %v", err)
}
if decoded.Bounds().Dx() != 256 || decoded.Bounds().Dy() != 160 {
t.Fatalf("actual thumbnail bounds = %v", decoded.Bounds())
}
crop := byName["crop"]
if crop.ContentType != "image/jpeg" || crop.Width != 128 || crop.Height != 128 {
t.Fatalf("crop = %s %dx%d", crop.ContentType, crop.Width, crop.Height)
}
optimized := byName["optimized"]
if optimized.ContentType != "image/webp" || optimized.Width != 320 || optimized.Height != 200 {
t.Fatalf("optimized = %s %dx%d, want image/webp 320x200", optimized.ContentType, optimized.Width, optimized.Height)
}
}
func TestThumbnailCompositesAlphaOnWhite(t *testing.T) {
file := pngFile(t, 64, 64, true)
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
decoded, err := jpeg.Decode(bytes.NewReader(variants[0].Bytes))
if err != nil {
t.Fatalf("decode thumbnail: %v", err)
}
r, g, b, _ := decoded.At(32, 32).RGBA()
if r < 0xf000 || g < 0xf000 || b < 0xf000 {
t.Fatalf("transparent pixel is not white: %04x %04x %04x", r, g, b)
}
}
func TestEXIFOrientationRotatesBeforeResize(t *testing.T) {
file := jpegFileWithOrientation(t, 320, 200, 6)
if got := jpegOrientation(file); got != 6 {
t.Fatalf("jpegOrientation = %d, want 6", got)
}
variants, err := New(40_000_000, 256<<20).Process(context.Background(), file)
if err != nil {
t.Fatalf("Process: %v", err)
}
if variants[0].Width != 160 || variants[0].Height != 256 {
t.Fatalf("oriented thumbnail = %dx%d, want 160x256", variants[0].Width, variants[0].Height)
}
}
func TestParseTIFFOrientationBigEndian(t *testing.T) {
tiff := []byte{
'M', 'M', 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08,
0x00, 0x01,
0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x08, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
}
if got := parseTIFFOrientation(tiff); got != 8 {
t.Fatalf("parseTIFFOrientation = %d, want 8", got)
}
}
func pngFile(t *testing.T, width, height int, transparent bool) *os.File {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
alpha := uint8(0xff)
if transparent {
alpha = 0
}
img.Set(x, y, color.RGBA{R: 10, G: 120, B: 200, A: alpha})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return tempFile(t, "image.png", buf.Bytes())
}
func jpegFileWithOrientation(t *testing.T, width, height int, orientation uint16) *os.File {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
var buf bytes.Buffer
if err := jpeg.Encode(&buf, img, nil); err != nil {
t.Fatalf("encode jpeg: %v", err)
}
payload := []byte{'E', 'x', 'i', 'f', 0, 0}
tiff := make([]byte, 26)
copy(tiff, []byte{'I', 'I'})
binary.LittleEndian.PutUint16(tiff[2:4], 42)
binary.LittleEndian.PutUint32(tiff[4:8], 8)
binary.LittleEndian.PutUint16(tiff[8:10], 1)
binary.LittleEndian.PutUint16(tiff[10:12], 0x0112)
binary.LittleEndian.PutUint16(tiff[12:14], 3)
binary.LittleEndian.PutUint32(tiff[14:18], 1)
binary.LittleEndian.PutUint16(tiff[18:20], orientation)
payload = append(payload, tiff...)
segment := append([]byte{0xff, 0xe1}, 0, 0)
binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2))
segment = append(segment, payload...)
data := append([]byte{}, buf.Bytes()[:2]...)
data = append(data, segment...)
data = append(data, buf.Bytes()[2:]...)
return tempFile(t, "oriented.jpg", data)
}
func tempFile(t *testing.T, name string, data []byte) *os.File {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatalf("write %s: %v", path, err)
}
file, err := os.Open(path)
if err != nil {
t.Fatalf("open %s: %v", path, err)
}
t.Cleanup(func() { _ = file.Close() })
return file
}
+46 -51
View File
@@ -1,12 +1,11 @@
// Package reconcile repairs the two failure windows between PostgreSQL
// metadata and S3 bytes:
// Package reconcile repairs the two storage/metadata gaps that a crash can
// leave behind:
//
// - a pending metadata row whose upload never finished or whose 201 was
// never returned;
// - an S3 object whose metadata transaction never committed.
// - a staging object whose metadata row never committed;
// - a terminal attachment whose staging copy was not deleted yet.
//
// Both repairs use a grace period so an upload that is still in flight is
// never touched.
// It never touches the final original or a variant while PostgreSQL still owns
// it. The grace period protects in-flight work.
package reconcile
import (
@@ -19,7 +18,7 @@ import (
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
const objectPrefix = "attachments/"
var prefixes = []string{"attachments/", "staging/"}
type Reconciler struct {
store *store.Store
@@ -29,9 +28,9 @@ type Reconciler struct {
}
type Stats struct {
StalePending int `json:"stale_pending"`
StaleStaging int `json:"stale_staging"`
OrphanObjects int `json:"orphan_objects"`
DeletedRows int `json:"deleted_rows"`
DeletedStaging int `json:"deleted_staging"`
DeletedObjects int `json:"deleted_objects"`
Skipped int `json:"skipped"`
}
@@ -47,68 +46,64 @@ func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
var stats Stats
cutoff := time.Now().UTC().Add(-r.grace)
// Phase 1: rows that were committed as pending but never reached ready.
stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500)
// Phase 1: ready/failed rows eventually lose their staging copy. The
// worker usually deletes it immediately after ready commits; this covers a
// crash or a transient S3 error.
stale, err := r.store.ListStaleStagingAttachments(ctx, cutoff, 500)
if err != nil {
return stats, err
}
for _, attachment := range stale {
stats.StalePending++
stats.StaleStaging++
if dryRun {
r.logger.Info("reconcile: would remove stale pending attachment",
"attachment_id", attachment.ID, "object_key", attachment.ObjectKey,
"created_at", attachment.CreatedAt)
r.logger.Info("reconcile: would remove stale staging",
"attachment_id", attachment.ID, "staging_key", attachment.StagingKey)
continue
}
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
// Keep the row for the next pass. Deleting it now would lose the
// only pointer to an object that might still exist.
r.logger.Error("reconcile: delete stale object",
if err = r.objects.Delete(ctx, attachment.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete staging",
"err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil {
r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID)
if err = r.store.ClearStagingKey(ctx, attachment.ID); err != nil {
r.logger.Error("reconcile: clear staging key", "err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
stats.DeletedRows++
stats.DeletedObjects++
stats.DeletedStaging++
}
// Phase 2: objects with no metadata row at all. The pending row is written
// before the PUT in the normal path, so this only happens after a crash in
// a narrow window (or manual bucket tampering).
listed, err := r.objects.List(ctx, objectPrefix)
// Phase 2: anything under our prefixes with no database owner. The API
// writes staging before the metadata commit, so this window is real.
known, err := r.store.ListStorageKeys(ctx)
if err != nil {
return stats, err
}
known, err := r.store.ListAttachmentObjectKeys(ctx)
if err != nil {
return stats, err
}
for _, object := range listed {
if _, ok := known[object.Key]; ok {
continue
for _, prefix := range prefixes {
listed, listErr := r.objects.List(ctx, prefix)
if listErr != nil {
return stats, listErr
}
// A missing LastModified must not be treated as ancient: skipping is
// safer than deleting an object that may have just been created.
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
continue
for _, object := range listed {
if _, ok := known[object.Key]; ok {
continue
}
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
continue
}
stats.OrphanObjects++
if dryRun {
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
continue
}
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
stats.Skipped++
continue
}
stats.DeletedObjects++
}
stats.OrphanObjects++
if dryRun {
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
continue
}
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
stats.Skipped++
continue
}
stats.DeletedObjects++
}
return stats, nil
+161 -81
View File
@@ -3,6 +3,7 @@ package store
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"time"
@@ -12,10 +13,11 @@ import (
"github.com/jackc/pgx/v5"
)
// CreatePendingAttachment records the intent to store an object before the S3
// PUT happens. Committing this row first is what makes the reconciler able to
// distinguish "never uploaded" from "uploaded but process died".
func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachment) error {
// CreateQueuedAttachment is the durability boundary of the upload endpoint.
// The attachment snapshot and the outbox job commit together, so there is no
// observable state in which the API promised work but PostgreSQL lost the
// intent. Publication to RabbitMQ happens later and may be retried safely.
func (s *Store) CreateQueuedAttachment(ctx context.Context, att model.Attachment, job model.JobRecord) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
@@ -23,58 +25,74 @@ func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachmen
if err != nil {
return common.BadRequest("invalid sha256")
}
if att.PipelineVersion < 1 {
att.PipelineVersion = 1
}
variants, err := json.Marshal([]model.Variant{})
if err != nil {
return common.Internal()
}
_, err = s.pool.Exec(ctx, `
tx, err := s.pool.Begin(ctx)
if err != nil {
return dbError(err)
}
defer rollback(tx)
_, err = tx.Exec(ctx, `
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, error_code, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL, $9, $9)`,
object_key, staging_key, pipeline_version, variants, status, error_code,
created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued', NULL, $12, $12)`,
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes,
hash, att.ObjectKey, time.Time(att.CreatedAt))
return dbError(err)
}
// MarkAttachmentReady publishes metadata only after the object bytes are known
// to be in S3. A 201 is therefore a promise about both stores.
func (s *Store) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (time.Time, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
updatedAt := time.Now().UTC()
tag, err := s.pool.Exec(ctx, `
UPDATE attachments
SET status = 'ready', updated_at = $2
WHERE id = $1 AND status = 'pending'`, id, updatedAt)
hash, att.ObjectKey, att.StagingKey, att.PipelineVersion, variants, time.Time(att.CreatedAt))
if err != nil {
return time.Time{}, dbError(err)
return dbError(err)
}
if tag.RowsAffected() == 0 {
return time.Time{}, common.Conflict("attachment is not pending")
_, err = tx.Exec(ctx, `
INSERT INTO attachment_jobs
(job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
status, error_code, available_at, published_at, lease_until,
processing_started_at, finished_at, replay_count, created_at, updated_at)
VALUES ($1, $2, $3, $4, 0, $5, 'queued', NULL, $6, NULL, NULL,
NULL, NULL, 0, $6, $6)`,
job.JobID, att.ID, model.JobTypeAttachmentProcessV1, att.PipelineVersion,
job.MaxAttempts, time.Time(att.CreatedAt))
if err != nil {
return dbError(err)
}
return updatedAt, nil
if err = tx.Commit(ctx); err != nil {
return dbError(err)
}
return nil
}
// DeletePendingAttachment removes a row only while it is still pending. The
// status guard is the safety property: if an upload completed concurrently, the
// compensation or reconciler pass must not delete a ready attachment.
func (s *Store) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error {
// CountPendingJobs is an admission control signal, not a queue size proof.
// Two API replicas may observe slightly stale counts, but the durable cap is
// still useful: it stops an overload from growing staging and PostgreSQL
// without limit. Rejections are 503 because the client may retry later.
func (s *Store) CountPendingJobs(ctx context.Context) (int, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
_, err := s.pool.Exec(ctx,
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
return dbError(err)
var count int
err := s.pool.QueryRow(ctx, `
SELECT count(*) FROM attachment_jobs
WHERE status IN ('queued', 'processing')`).Scan(&count)
if err != nil {
return 0, dbError(err)
}
return count, nil
}
func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
att, err := scanAttachment(s.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code,
a.created_at, a.updated_at, a.object_key
FROM attachments a
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+`
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2
WHERE a.id = $1`, attachmentID, userID))
if err != nil {
@@ -83,20 +101,76 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID u
return att, nil
}
// ListStalePendingAttachments is the entry point for crash recovery. cutoff is
// an instant, not a duration: rows created before it are eligible. The caller
// computes it as now - grace so the boundary can be tuned and tested.
func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
// GetAttachment is the worker's trusted read. The worker never accepts bucket
// keys or filenames from the broker message.
func (s *Store) GetAttachment(ctx context.Context, attachmentID uuid.UUID) (model.Attachment, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code,
a.created_at, a.updated_at, a.object_key
FROM attachments a
WHERE a.status = 'pending' AND a.created_at < $1
ORDER BY a.created_at
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1`, attachmentID))
if err != nil {
return model.Attachment{}, err
}
return att, nil
}
const attachmentSelect = `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code,
a.created_at, a.updated_at, a.object_key, COALESCE(a.staging_key, ''),
a.pipeline_version, a.variants
FROM attachments a`
func scanAttachment(row rowScanner) (model.Attachment, error) {
var (
att model.Attachment
hash []byte
errCode *string
createdAt time.Time
updatedAt time.Time
variants []byte
)
if err := row.Scan(
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
&att.SizeBytes, &hash, &att.Status, &errCode,
&createdAt, &updatedAt, &att.ObjectKey, &att.StagingKey,
&att.PipelineVersion, &variants,
); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, common.NotFound("attachment not found")
}
return model.Attachment{}, dbError(err)
}
att.SHA256 = hex.EncodeToString(hash)
att.ErrorCode = errCode
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
att.Variants = []model.Variant{}
if len(variants) > 0 {
if err := json.Unmarshal(variants, &att.Variants); err != nil {
return model.Attachment{}, dbError(err)
}
}
if att.Variants == nil {
att.Variants = []model.Variant{}
}
return att, nil
}
// ListStaleStagingAttachments returns terminal attachments whose staging copy
// may be removed. The grace period is applied to updated_at so a just-finished
// job is never cleaned while its commit is still settling.
func (s *Store) ListStaleStagingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, attachmentSelect+`
WHERE a.status IN ('ready', 'failed')
AND a.updated_at < $1
AND a.staging_key IS NOT NULL
ORDER BY a.updated_at
LIMIT $2`, cutoff, limit)
if err != nil {
return nil, dbError(err)
@@ -107,7 +181,7 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim
for rows.Next() {
att, scanErr := scanAttachment(rows)
if scanErr != nil {
return nil, dbError(scanErr)
return nil, scanErr
}
result = append(result, att)
}
@@ -117,11 +191,24 @@ func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Tim
return result, nil
}
func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct{}, error) {
func (s *Store) ClearStagingKey(ctx context.Context, attachmentID uuid.UUID) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, `SELECT object_key FROM attachments`)
_, err := s.pool.Exec(ctx,
`UPDATE attachments SET staging_key = NULL WHERE id = $1`, attachmentID)
return dbError(err)
}
// ListStorageKeys returns every object the database still owns. It includes
// variant keys derived from the original key; the reconciler must never delete
// a thumbnail that has no separate row.
func (s *Store) ListStorageKeys(ctx context.Context) (map[string]struct{}, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx,
`SELECT object_key, COALESCE(staging_key, ''), variants FROM attachments`)
if err != nil {
return nil, dbError(err)
}
@@ -129,11 +216,25 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct
keys := make(map[string]struct{})
for rows.Next() {
var key string
if err = rows.Scan(&key); err != nil {
var (
objectKey string
staging string
variants []byte
)
if err = rows.Scan(&objectKey, &staging, &variants); err != nil {
return nil, dbError(err)
}
keys[key] = struct{}{}
keys[objectKey] = struct{}{}
if staging != "" {
keys[staging] = struct{}{}
}
var list []model.Variant
if err = json.Unmarshal(variants, &list); err != nil {
return nil, dbError(err)
}
for _, variant := range list {
keys[VariantObjectKey(objectKey, variant.Name)] = struct{}{}
}
}
if err = rows.Err(); err != nil {
return nil, dbError(err)
@@ -141,30 +242,9 @@ func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct
return keys, nil
}
func scanAttachment(row rowScanner) (model.Attachment, error) {
var (
att model.Attachment
hash []byte
errCode *string
createdAt time.Time
updatedAt time.Time
)
if err := row.Scan(
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
&att.SizeBytes, &hash, &att.Status, &errCode,
&createdAt, &updatedAt, &att.ObjectKey,
); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, common.NotFound("attachment not found")
}
return model.Attachment{}, dbError(err)
}
att.SHA256 = hex.EncodeToString(hash)
att.ErrorCode = errCode
att.Variants = []model.Variant{}
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
return att, nil
// VariantObjectKey is the one place that names variant objects. Keeping it
// deterministic makes a retried job overwrite the same keys instead of
// creating a second variant set.
func VariantObjectKey(originalKey, variant string) string {
return originalKey + "/" + variant
}
+433
View File
@@ -0,0 +1,433 @@
package store
import (
"context"
"encoding/json"
"errors"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// ClaimResult tells the consumer whether this delivery owns the work.
type ClaimResult struct {
Job model.JobRecord
Attachment model.Attachment
// Terminal means the logical work is already ready/failed. Ack it.
Terminal bool
// Busy means another attempt owns an active lease, or a retry is not due.
// The publisher/lease sweeper will make the job visible again.
Busy bool
}
// ClaimJob acquires or renews the logical job lease in one transaction.
// A broker redelivery of the same job_id therefore cannot become two workers
// writing results at the same time.
func (s *Store) ClaimJob(ctx context.Context, jobID uuid.UUID, lease time.Duration) (ClaimResult, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
tx, err := s.pool.Begin(ctx)
if err != nil {
return ClaimResult{}, dbError(err)
}
defer rollback(tx)
job, err := scanJob(tx.QueryRow(ctx, `
SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
status, error_code, created_at, available_at, published_at,
lease_until, processing_started_at, finished_at, replay_count
FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID))
if err != nil {
return ClaimResult{}, err
}
att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, job.AttachmentID))
if err != nil {
return ClaimResult{}, err
}
now := time.Now().UTC()
if job.Status == model.JobStatusReady || job.Status == model.JobStatusFailed || att.Status == "ready" {
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
}
if att.Status == "failed" {
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
}
if job.Status == model.JobStatusProcessing && job.LeaseUntil != nil && job.LeaseUntil.After(now) {
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
return ClaimResult{Job: job, Attachment: att, Busy: true}, nil
}
if job.Status == model.JobStatusQueued && job.AvailableAt.After(now) {
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
return ClaimResult{Job: job, Attachment: att, Busy: true}, nil
}
if job.Attempt >= job.MaxAttempts {
_, failErr := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'failed', error_code = 'retry_exhausted',
finished_at = $2, lease_until = NULL, updated_at = $2
WHERE job_id = $1`, jobID, now)
if failErr != nil {
return ClaimResult{}, dbError(failErr)
}
_, failErr = tx.Exec(ctx, `
UPDATE attachments
SET status = 'failed', error_code = 'retry_exhausted', updated_at = $2
WHERE id = $1 AND status <> 'ready'`, job.AttachmentID, now)
if failErr != nil {
return ClaimResult{}, dbError(failErr)
}
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
return ClaimResult{Job: job, Attachment: att, Terminal: true}, nil
}
attempt := job.Attempt + 1
leaseUntil := now.Add(lease)
_, err = tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'processing', attempt = $2, error_code = NULL,
lease_until = $3, processing_started_at = COALESCE(processing_started_at, $4),
published_at = COALESCE(published_at, $4), updated_at = $4
WHERE job_id = $1`, jobID, attempt, leaseUntil, now)
if err != nil {
return ClaimResult{}, dbError(err)
}
_, err = tx.Exec(ctx, `
UPDATE attachments
SET status = 'processing', error_code = NULL, updated_at = $2
WHERE id = $1 AND status IN ('queued', 'processing')`, job.AttachmentID, now)
if err != nil {
return ClaimResult{}, dbError(err)
}
if err = tx.Commit(ctx); err != nil {
return ClaimResult{}, dbError(err)
}
job.Attempt = attempt
job.Status = model.JobStatusProcessing
job.ErrorCode = nil
job.LeaseUntil = &leaseUntil
if job.ProcessingStartedAt == nil {
job.ProcessingStartedAt = &now
}
att.Status = "processing"
att.ErrorCode = nil
att.UpdatedAt = common.RFCFormTime(now)
return ClaimResult{Job: job, Attachment: att}, nil
}
// CompleteJob commits the ready snapshot only for the attempt/version that
// still owns the lease. An older duplicate can therefore never roll a newer
// result back or overwrite it.
func (s *Store) CompleteJob(ctx context.Context, jobID uuid.UUID, attempt int, variants []model.Variant) (bool, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
if variants == nil {
variants = []model.Variant{}
}
encoded, err := json.Marshal(variants)
if err != nil {
return false, common.Internal()
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return false, dbError(err)
}
defer rollback(tx)
var attachmentID uuid.UUID
var pipelineVersion int
var status string
err = tx.QueryRow(ctx, `
SELECT attachment_id, pipeline_version, status
FROM attachment_jobs WHERE job_id = $1 FOR UPDATE`, jobID).
Scan(&attachmentID, &pipelineVersion, &status)
if errors.Is(err, pgx.ErrNoRows) {
return false, common.NotFound("job not found")
}
if err != nil {
return false, dbError(err)
}
if status != model.JobStatusProcessing {
return false, nil
}
att, err := scanAttachment(tx.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1 FOR UPDATE`, attachmentID))
if err != nil {
return false, err
}
if att.PipelineVersion != pipelineVersion || att.Status == "ready" || att.Status == "failed" {
return false, nil
}
now := time.Now().UTC()
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'ready', error_code = NULL, finished_at = $3,
lease_until = NULL, updated_at = $3
WHERE job_id = $1 AND attempt = $2 AND status = 'processing'`,
jobID, attempt, now)
if err != nil {
return false, dbError(err)
}
if tag.RowsAffected() == 0 {
return false, nil
}
_, err = tx.Exec(ctx, `
UPDATE attachments
SET status = 'ready', variants = $2, error_code = NULL,
ready_at = $3, updated_at = $3
WHERE id = $1 AND pipeline_version = $4 AND status IN ('queued', 'processing')`,
attachmentID, encoded, now, pipelineVersion)
if err != nil {
return false, dbError(err)
}
if err = tx.Commit(ctx); err != nil {
return false, dbError(err)
}
return true, nil
}
// ScheduleRetry keeps the job in the durable outbox with a jittered delay.
// The caller acks the broker only after this commit, so the retry intent is
// never lost between "try later" and ack.
func (s *Store) ScheduleRetry(ctx context.Context, jobID uuid.UUID, attempt int, code string, delay time.Duration) (bool, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
tx, err := s.pool.Begin(ctx)
if err != nil {
return false, dbError(err)
}
defer rollback(tx)
now := time.Now().UTC()
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'queued', error_code = $3, available_at = $4,
published_at = NULL, lease_until = NULL, updated_at = $5
WHERE job_id = $1 AND status = 'processing' AND attempt = $2`,
jobID, attempt, code, now.Add(delay), now)
if err != nil {
return false, dbError(err)
}
if tag.RowsAffected() == 0 {
return false, nil
}
_, err = tx.Exec(ctx, `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = $2
WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1)
AND status <> 'ready'`, jobID, now)
if err != nil {
return false, dbError(err)
}
return true, dbError(tx.Commit(ctx))
}
// FailJob records a terminal reason. It is idempotent: a later stale attempt
// that no longer matches the current attempt is ignored.
func (s *Store) FailJob(ctx context.Context, jobID uuid.UUID, attempt int, code string) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
tx, err := s.pool.Begin(ctx)
if err != nil {
return dbError(err)
}
defer rollback(tx)
now := time.Now().UTC()
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'failed', error_code = $3, finished_at = $4,
lease_until = NULL, updated_at = $4
WHERE job_id = $1 AND status = 'processing' AND attempt = $2`,
jobID, attempt, code, now)
if err != nil {
return dbError(err)
}
if tag.RowsAffected() == 0 {
return nil
}
_, err = tx.Exec(ctx, `
UPDATE attachments
SET status = 'failed', error_code = $2, updated_at = $3
WHERE id = (SELECT attachment_id FROM attachment_jobs WHERE job_id = $1)
AND status <> 'ready'`, jobID, code, now)
if err != nil {
return dbError(err)
}
return dbError(tx.Commit(ctx))
}
// RecoverExpiredLeases is the stuck-processing guard. A dead worker leaves a
// processing row whose lease expires; the sweeper requeues it or marks it
// retry_exhausted. It never touches ready work.
func (s *Store) RecoverExpiredLeases(ctx context.Context) (int, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
tag, err := s.pool.Exec(ctx, `
WITH expired AS (
UPDATE attachment_jobs j
SET status = CASE WHEN j.attempt >= j.max_attempts THEN 'failed' ELSE 'queued' END,
error_code = CASE WHEN j.attempt >= j.max_attempts THEN 'retry_exhausted' ELSE NULL END,
available_at = now(),
published_at = NULL,
lease_until = NULL,
finished_at = CASE WHEN j.attempt >= j.max_attempts THEN now() ELSE NULL END,
updated_at = now()
WHERE j.status = 'processing'
AND j.lease_until IS NOT NULL
AND j.lease_until < now()
RETURNING j.attachment_id, j.status, j.error_code
)
UPDATE attachments a
SET status = CASE WHEN e.status = 'failed' THEN 'failed' ELSE 'queued' END,
error_code = CASE WHEN e.status = 'failed' THEN e.error_code ELSE NULL END,
updated_at = now()
FROM expired e
WHERE a.id = e.attachment_id AND a.status <> 'ready'`)
if err != nil {
return 0, dbError(err)
}
return int(tag.RowsAffected()), nil
}
// ListPublishableJobs is the outbox read side. It intentionally does not lock
// rows: two API replicas may publish the same message, and the idempotent
// claim/effect makes that duplicate harmless.
func (s *Store) ListPublishableJobs(ctx context.Context, limit int) ([]model.JobRecord, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, `
SELECT job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
status, error_code, created_at, available_at, published_at,
lease_until, processing_started_at, finished_at, replay_count
FROM attachment_jobs
WHERE published_at IS NULL AND status = 'queued' AND available_at <= now()
ORDER BY available_at, created_at
LIMIT $1`, limit)
if err != nil {
return nil, dbError(err)
}
defer rows.Close()
var jobs []model.JobRecord
for rows.Next() {
job, scanErr := scanJob(rows)
if scanErr != nil {
return nil, scanErr
}
jobs = append(jobs, job)
}
if err = rows.Err(); err != nil {
return nil, dbError(err)
}
return jobs, nil
}
// MarkJobPublished uses status+attempt as a compare-and-set. If a worker has
// already claimed the job, or a retry has superseded this publish, the update
// is a no-op and the outbox will publish the current intent later.
func (s *Store) MarkJobPublished(ctx context.Context, jobID uuid.UUID, attempt int, at time.Time) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
_, err := s.pool.Exec(ctx, `
UPDATE attachment_jobs
SET published_at = $3, updated_at = $3
WHERE job_id = $1 AND attempt = $2 AND status = 'queued' AND published_at IS NULL`,
jobID, attempt, at.UTC())
return dbError(err)
}
// ReplayFailed is the explicit operator action. The status guard means a
// second click, or a replay after success, cannot duplicate a ready effect.
func (s *Store) ReplayFailed(ctx context.Context, attachmentID uuid.UUID) (bool, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
tx, err := s.pool.Begin(ctx)
if err != nil {
return false, dbError(err)
}
defer rollback(tx)
now := time.Now().UTC()
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status = 'queued', attempt = 0, error_code = NULL,
available_at = $2, published_at = NULL, lease_until = NULL,
processing_started_at = NULL, finished_at = NULL,
replay_count = replay_count + 1, last_replay_at = $2, updated_at = $2
WHERE attachment_id = $1 AND status = 'failed'
AND EXISTS (SELECT 1 FROM attachments
WHERE id = $1 AND status = 'failed')`,
attachmentID, now)
if err != nil {
return false, dbError(err)
}
if tag.RowsAffected() == 0 {
return false, nil
}
_, err = tx.Exec(ctx, `
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = $2
WHERE id = $1 AND status = 'failed'`, attachmentID, now)
if err != nil {
return false, dbError(err)
}
if err = tx.Commit(ctx); err != nil {
return false, dbError(err)
}
return true, nil
}
func scanJob(row rowScanner) (model.JobRecord, error) {
var (
job model.JobRecord
created time.Time
errCode *string
)
if err := row.Scan(
&job.JobID, &job.AttachmentID, &job.Type, &job.PipelineVersion,
&job.Attempt, &job.MaxAttempts, &job.Status, &errCode, &created,
&job.AvailableAt, &job.PublishedAt, &job.LeaseUntil,
&job.ProcessingStartedAt, &job.FinishedAt, &job.ReplayCount,
); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.JobRecord{}, common.NotFound("job not found")
}
return model.JobRecord{}, dbError(err)
}
job.SchemaVersion = 1
job.CreatedAt = common.RFCFormTime(created.UTC())
job.ErrorCode = errCode
return job, nil
}
+4 -2
View File
@@ -78,9 +78,11 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
return model.Message{}, dbError(scanErr)
}
found++
if status != "ready" {
// Lab5: a queued/processing attachment is already linked to its
// chat and may be sent. Only a terminal failed one is refused.
if status == "failed" {
rows.Close()
return model.Message{}, common.Conflict("attachment is not ready")
return model.Message{}, common.Conflict("attachment failed processing")
}
}
if err = rows.Err(); err != nil {
+298
View File
@@ -0,0 +1,298 @@
// Package worker runs the asynchronous attachment pipeline. It consumes one
// logical job, reads trusted metadata from PostgreSQL, loads the staged object,
// writes the original and variants, commits ready atomically, and only then
// lets the broker ack the message.
package worker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"sync/atomic"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/broker"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/pipeline"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
type Worker struct {
cfg config.Config
store *store.Store
objects *objectstore.Client
pipeline *pipeline.Processor
logger *slog.Logger
faultOnce atomic.Bool
}
func New(cfg config.Config, st *store.Store, objects *objectstore.Client, logger *slog.Logger) *Worker {
if logger == nil {
logger = slog.Default()
}
w := &Worker{
cfg: cfg,
store: st,
objects: objects,
pipeline: pipeline.New(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
logger: logger,
}
w.faultOnce.Store(true)
return w
}
// StartLeaseSweeper recovers a worker that died after claiming a job. Without
// this, the database would show "processing" forever and no message would be
// left to redeliver.
func (w *Worker) StartLeaseSweeper(ctx context.Context) {
ticker := time.NewTicker(w.cfg.LeaseSweep)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
count, err := w.store.RecoverExpiredLeases(ctx)
if err != nil {
w.logger.Error("lease sweep failed", "err", err)
} else if count > 0 {
w.logger.Warn("recovered expired job leases", "count", count)
}
}
}
}
// Handle is the broker callback. Returning nil means "durable effect or
// deliberate no-op; ack". ErrRequeue means the database itself was unavailable.
func (w *Worker) Handle(ctx context.Context, message model.Job) error {
claim, err := w.store.ClaimJob(ctx, message.JobID, w.cfg.JobLease)
if err != nil {
if common.IsCode(err, common.ErrorCodeNotFound) {
w.logger.Warn("job no longer exists; acking", "job_id", message.JobID)
return nil
}
w.logger.Error("claim job failed", "job_id", message.JobID, "err", err)
return broker.ErrRequeue
}
if claim.Terminal || claim.Busy {
w.logger.Info("job is already finished or owned",
"job_id", message.JobID, "terminal", claim.Terminal, "busy", claim.Busy,
"attempt", claim.Job.Attempt)
return nil
}
if claim.Attachment.ID != message.AttachmentID {
w.logger.Error("message attachment mismatch; acking poison message",
"job_id", message.JobID, "message_attachment", message.AttachmentID,
"database_attachment", claim.Attachment.ID)
return nil
}
log := w.logger.With(
"job_id", claim.Job.JobID,
"attachment_id", claim.Attachment.ID,
"attempt", claim.Job.Attempt,
"image_bytes", claim.Attachment.SizeBytes,
)
if w.cfg.CrashAfterClaim {
// Deterministic fault injection for the "claimed but never finished"
// window. The lease sweeper requeues the durable job after it expires;
// it does not depend on the broker redelivery still being around.
w.logger.Error("fault injection: exiting after claim before processing",
"job_id", claim.Job.JobID, "lease_until", claim.Job.LeaseUntil)
os.Exit(10)
}
log.Info("processing attachment")
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
defer cancel()
if err = w.process(jobCtx, claim, log); err != nil {
if jobCtx.Err() != nil && errors.Is(jobCtx.Err(), context.DeadlineExceeded) {
err = &pipeline.Failure{Permanent: false, Code: "processing_timeout", Err: err}
}
return w.handleFailure(ctx, claim, err, log)
}
log.Info("attachment ready")
if w.cfg.CrashAfterEffect {
// Deterministic fault injection for the "written but not acked" window.
// The unacked delivery is redelivered after the connection dies.
w.logger.Error("fault injection: exiting after durable effect before ack",
"job_id", claim.Job.JobID)
os.Exit(9)
}
return nil
}
func (w *Worker) process(ctx context.Context, claim store.ClaimResult, log *slog.Logger) error {
if mode := w.cfg.WorkerFaultMode; mode != "none" && w.faultOnce.CompareAndSwap(true, false) {
if mode == "transient-once" {
return &pipeline.Failure{Permanent: false, Code: "processing_failed",
Err: errors.New("injected transient failure")}
}
return &pipeline.Failure{Permanent: true, Code: "invalid_image",
Err: errors.New("injected permanent failure")}
}
att := claim.Attachment
if att.StagingKey == "" {
return &pipeline.Failure{Permanent: true, Code: "staging_missing",
Err: errors.New("attachment has no staging key")}
}
reader, size, err := w.objects.Get(ctx, att.StagingKey)
if err != nil {
if errors.Is(err, objectstore.ErrNotFound) {
return &pipeline.Failure{Permanent: true, Code: "staging_missing", Err: err}
}
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
defer reader.Close()
if size != att.SizeBytes {
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt",
Err: fmt.Errorf("staged size %d != metadata %d", size, att.SizeBytes)}
}
if err = os.MkdirAll(w.cfg.WorkTempDir, 0o700); err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
file, err := os.CreateTemp(w.cfg.WorkTempDir, "job-*.part")
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
path := file.Name()
defer func() {
_ = file.Close()
_ = os.Remove(path)
}()
hasher := sha256.New()
// The same 10 MiB contract limit protects the worker from a corrupt row
// pointing at an unexpectedly large object.
written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, w.cfg.MaxUploadBytes+1), make([]byte, 32*1024))
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
if written > w.cfg.MaxUploadBytes {
return &pipeline.Failure{Permanent: true, Code: "invalid_image", Err: errors.New("staged object too large")}
}
if written != att.SizeBytes || hex.EncodeToString(hasher.Sum(nil)) != att.SHA256 {
return &pipeline.Failure{Permanent: true, Code: "staging_corrupt", Err: errors.New("staged checksum mismatch")}
}
encoded, err := w.pipeline.Process(ctx, file)
if err != nil {
return err
}
// Original bytes are copied unchanged from staging to the final key.
if _, err = file.Seek(0, 0); err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if err = w.objects.Put(ctx, att.ObjectKey, file, att.SizeBytes, att.ContentType); err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
variants := make([]model.Variant, 0, len(encoded))
for _, item := range encoded {
key := store.VariantObjectKey(att.ObjectKey, item.Name)
if err = w.objects.Put(ctx, key, bytes.NewReader(item.Bytes), int64(len(item.Bytes)), item.ContentType); err != nil {
return &pipeline.Failure{Permanent: false, Code: "storage_unavailable", Err: err}
}
variants = append(variants, model.Variant{
Name: item.Name,
ContentType: item.ContentType,
SizeBytes: int64(len(item.Bytes)),
Width: item.Width,
Height: item.Height,
SHA256: item.SHA256,
})
}
committed, err := w.store.CompleteJob(ctx, claim.Job.JobID, claim.Job.Attempt, variants)
if err != nil {
return &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if !committed {
log.Warn("ready snapshot was superseded; leaving newer state alone")
return nil
}
// Staging is removed only after ready commits. A crash before this delete
// leaves an object the reconciler will clean after retention.
if err = w.objects.Delete(ctx, att.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
log.Warn("staging delete failed; reconciler will retry", "err", err)
return nil
}
if err = w.store.ClearStagingKey(ctx, att.ID); err != nil {
log.Warn("clear staging key failed; reconciler will retry", "err", err)
}
return nil
}
func (w *Worker) handleFailure(ctx context.Context, claim store.ClaimResult, err error, log *slog.Logger) error {
var failure *pipeline.Failure
if !errors.As(err, &failure) {
failure = &pipeline.Failure{Permanent: false, Code: "processing_failed", Err: err}
}
if failure.Permanent {
log.Error("permanent job failure", "code", failure.Code, "err", failure.Err)
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code); failErr != nil {
log.Error("record permanent failure failed; requeueing", "err", failErr)
return broker.ErrRequeue
}
return nil
}
if claim.Job.Attempt >= claim.Job.MaxAttempts {
log.Error("retry budget exhausted", "code", failure.Code, "err", failure.Err)
if failErr := w.store.FailJob(ctx, claim.Job.JobID, claim.Job.Attempt, "retry_exhausted"); failErr != nil {
log.Error("record exhausted retry failed; requeueing", "err", failErr)
return broker.ErrRequeue
}
return nil
}
delay := w.backoff(claim.Job.Attempt)
log.Warn("transient job failure; scheduling retry",
"code", failure.Code, "err", failure.Err, "retry_in", delay)
scheduled, retryErr := w.store.ScheduleRetry(ctx, claim.Job.JobID, claim.Job.Attempt, failure.Code, delay)
if retryErr != nil {
log.Error("schedule retry failed; requeueing", "err", retryErr)
return broker.ErrRequeue
}
if !scheduled {
// This attempt no longer owns the row (for example a lease sweep or a
// replay changed it). Requeue rather than ack a job whose future work
// was not durably recorded.
log.Warn("retry scheduling lost ownership; requeueing")
return broker.ErrRequeue
}
return nil
}
// backoff is exponential with +/-50% jitter, capped by JOB_RETRY_MAX_SECONDS.
// Jitter prevents many retries from hitting a recovered dependency together.
func (w *Worker) backoff(attempt int) time.Duration {
delay := w.cfg.RetryBase
for i := 1; i < attempt && delay < w.cfg.RetryMax; i++ {
delay *= 2
}
if delay > w.cfg.RetryMax {
delay = w.cfg.RetryMax
}
jitter := 0.5 + rand.Float64() // 0.5 .. 1.5
return time.Duration(float64(delay) * jitter)
}