feat(lab5): durable async attachment pipeline through grade 4

- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
Mikhail Verkovykh
2026-11-15 00:30:36 +03:00
parent 28d68dc30b
commit 9e43faa536
48 changed files with 4077 additions and 333 deletions
+81 -51
View File
@@ -31,35 +31,31 @@ type Handler struct {
validator *Validator
maxUploadBytes int64
drainLimit int64
maxPendingJobs int
jobMaxAttempts int
uploadTimeout time.Duration
downloadTimeout time.Duration
tempDir string
}
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, maxPendingJobs, jobMaxAttempts int, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
return &Handler{
store: st,
objects: client,
validator: validator,
maxUploadBytes: maxUploadBytes,
drainLimit: drainLimit,
maxPendingJobs: maxPendingJobs,
jobMaxAttempts: jobMaxAttempts,
uploadTimeout: uploadTimeout,
downloadTimeout: downloadTimeout,
tempDir: tempDir,
}
}
// Upload stores raw PNG/JPEG bytes. The order is deliberate:
// 1. authorize before reading the body;
// 2. stream the body to a bounded temporary file while hashing;
// 3. validate pixels and decode;
// 4. commit a pending metadata row;
// 5. PUT the object;
// 6. mark the row ready.
//
// A 201 is therefore returned only after both the object and its metadata are
// durable. Every failure between 4 and 6 has a compensation path, and the
// reconciler covers crashes that prevent compensation from running.
// Upload stages the original, then commits the attachment snapshot and its
// outbox job together. The 202 is therefore a promise about durable bytes plus
// durable intent, not a promise that resizing already happened.
func (h *Handler) Upload(c fiber.Ctx) error {
userID := common.MustUserID(c)
@@ -88,6 +84,19 @@ func (h *Handler) Upload(c fiber.Ctx) error {
return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
}
// Backpressure: refuse new work before reading its bytes. The check is
// approximate across API replicas but bounds staging growth under overload.
if h.maxPendingJobs > 0 {
pending, countErr := h.store.CountPendingJobs(c.Context())
if countErr != nil {
return countErr
}
if pending >= h.maxPendingJobs {
c.Set(fiber.HeaderRetryAfter, "5")
return common.Unavailable("processing queue is full")
}
}
// Reject an obviously oversized request before allocating a temp file or
// decoding anything. Chunked requests are caught by the streaming counter
// below.
@@ -151,6 +160,7 @@ func (h *Handler) Upload(c fiber.Ctx) error {
}
attachmentID := uuid.New()
jobID := uuid.New()
attachment := model.Attachment{
ID: attachmentID,
ChatID: chatID,
@@ -159,55 +169,51 @@ func (h *Handler) Upload(c fiber.Ctx) error {
ContentType: info.ContentType,
SizeBytes: written,
SHA256: hex.EncodeToString(hasher.Sum(nil)),
Status: "pending",
Status: "queued",
Variants: []model.Variant{},
ErrorCode: nil,
CreatedAt: common.NowUTC(),
UpdatedAt: common.NowUTC(),
// AGENT: the key is server-generated from UUIDs and never contains the
// client filename. That keeps traversal and collisions out of the
// object store even before application authorization is considered.
ObjectKey: objectKey(chatID, attachmentID),
}
if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil {
return err
// AGENT: keys are server-generated from UUIDs, never the client
// filename. staging/ is the durable handoff prefix; the worker copies
// the same bytes to the final attachments/ key unchanged.
ObjectKey: objectKey(chatID, attachmentID),
StagingKey: stagingKey(chatID, attachmentID),
PipelineVersion: 1,
}
if _, err = file.Seek(0, 0); err != nil {
h.compensate(attachment.ID, attachment.ObjectKey)
return common.BadRequest("cannot read upload body")
}
if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil {
slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
if err = h.objects.Put(ctx, attachment.StagingKey, file, written, attachment.ContentType); err != nil {
slog.Error("stage attachment object", "err", err, "attachment_id", attachment.ID)
return common.Unavailable("object storage unavailable")
}
updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID)
if err != nil {
slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
return common.Unavailable("attachment metadata unavailable")
job := model.JobRecord{
Job: model.Job{
JobID: jobID,
AttachmentID: attachmentID,
PipelineVersion: attachment.PipelineVersion,
CreatedAt: attachment.CreatedAt,
},
MaxAttempts: h.jobMaxAttempts,
}
if err = h.store.CreateQueuedAttachment(ctx, attachment, job); err != nil {
h.compensateStaging(attachment.StagingKey)
return err
}
attachment.Status = "ready"
attachment.UpdatedAt = common.RFCFormTime(updatedAt)
return c.Status(fiber.StatusCreated).JSON(attachment)
return c.Status(fiber.StatusAccepted).JSON(attachment)
}
// compensate is best effort by design. If either delete fails, the pending row
// or the object remains visible to the reconciler, which retries after the
// configured grace period.
func (h *Handler) compensate(attachmentID uuid.UUID, key string) {
// compensateStaging is best effort. If it fails, the reconciler removes an
// object with no metadata row after the grace period.
func (h *Handler) compensateStaging(key string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID)
}
if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil {
slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID)
slog.Error("compensate: delete staging object", "err", err)
}
}
@@ -223,10 +229,9 @@ func (h *Handler) Get(c fiber.Ctx) error {
if err != nil {
return err
}
if attachment.Status != "ready" {
return common.Conflict("attachment is not ready")
}
// queued/processing snapshots are part of lab5: the client polls this
// endpoint until status becomes ready or failed.
return c.JSON(attachment)
}
@@ -242,7 +247,9 @@ func (h *Handler) Download(c fiber.Ctx) error {
if variant == "" {
variant = "original"
}
if variant != "original" {
switch variant {
case "original", "thumbnail", "crop", "optimized":
default:
return common.BadRequest("unsupported variant")
}
@@ -254,9 +261,28 @@ func (h *Handler) Download(c fiber.Ctx) error {
return common.Conflict("attachment is not ready")
}
key := attachment.ObjectKey
contentType := attachment.ContentType
sizeBytes := attachment.SizeBytes
if variant != "original" {
found := false
for _, candidate := range attachment.Variants {
if candidate.Name == variant {
key = store.VariantObjectKey(attachment.ObjectKey, candidate.Name)
contentType = candidate.ContentType
sizeBytes = candidate.SizeBytes
found = true
break
}
}
if !found {
return common.NotFound("attachment variant not found")
}
}
ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout)
body, size, err := h.objects.Get(ctx, attachment.ObjectKey)
body, size, err := h.objects.Get(ctx, key)
if err != nil {
cancel()
if errors.Is(err, objectstore.ErrNotFound) {
@@ -266,16 +292,16 @@ func (h *Handler) Download(c fiber.Ctx) error {
slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID)
return common.Unavailable("object storage unavailable")
}
if size != attachment.SizeBytes {
if size != sizeBytes {
_ = body.Close()
cancel()
slog.Error("attachment object size mismatch",
"attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size)
"attachment_id", attachment.ID, "metadata_size", sizeBytes, "object_size", size)
return common.Unavailable("attachment content is inconsistent")
}
c.Set(fiber.HeaderContentType, attachment.ContentType)
c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10))
c.Set(fiber.HeaderContentType, contentType)
c.Set(fiber.HeaderContentLength, strconv.FormatInt(sizeBytes, 10))
c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
c.Set(fiber.HeaderCacheControl, "private, no-store")
c.Set("X-Content-Type-Options", "nosniff")
@@ -307,6 +333,10 @@ func objectKey(chatID, attachmentID uuid.UUID) string {
return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID)
}
func stagingKey(chatID, attachmentID uuid.UUID) string {
return fmt.Sprintf("staging/%s/%s", chatID, attachmentID)
}
// CleanupStaleUploads removes temp files abandoned by a previous crash. A
// running request removes its own file in a defer; this only handles the
// process-died-mid-upload case.