forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -31,35 +31,31 @@ type Handler struct {
|
||||
validator *Validator
|
||||
maxUploadBytes int64
|
||||
drainLimit int64
|
||||
maxPendingJobs int
|
||||
jobMaxAttempts int
|
||||
uploadTimeout time.Duration
|
||||
downloadTimeout time.Duration
|
||||
tempDir string
|
||||
}
|
||||
|
||||
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
|
||||
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, maxPendingJobs, jobMaxAttempts int, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
|
||||
return &Handler{
|
||||
store: st,
|
||||
objects: client,
|
||||
validator: validator,
|
||||
maxUploadBytes: maxUploadBytes,
|
||||
drainLimit: drainLimit,
|
||||
maxPendingJobs: maxPendingJobs,
|
||||
jobMaxAttempts: jobMaxAttempts,
|
||||
uploadTimeout: uploadTimeout,
|
||||
downloadTimeout: downloadTimeout,
|
||||
tempDir: tempDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Upload stores raw PNG/JPEG bytes. The order is deliberate:
|
||||
// 1. authorize before reading the body;
|
||||
// 2. stream the body to a bounded temporary file while hashing;
|
||||
// 3. validate pixels and decode;
|
||||
// 4. commit a pending metadata row;
|
||||
// 5. PUT the object;
|
||||
// 6. mark the row ready.
|
||||
//
|
||||
// A 201 is therefore returned only after both the object and its metadata are
|
||||
// durable. Every failure between 4 and 6 has a compensation path, and the
|
||||
// reconciler covers crashes that prevent compensation from running.
|
||||
// Upload stages the original, then commits the attachment snapshot and its
|
||||
// outbox job together. The 202 is therefore a promise about durable bytes plus
|
||||
// durable intent, not a promise that resizing already happened.
|
||||
func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
userID := common.MustUserID(c)
|
||||
|
||||
@@ -88,6 +84,19 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
|
||||
}
|
||||
|
||||
// Backpressure: refuse new work before reading its bytes. The check is
|
||||
// approximate across API replicas but bounds staging growth under overload.
|
||||
if h.maxPendingJobs > 0 {
|
||||
pending, countErr := h.store.CountPendingJobs(c.Context())
|
||||
if countErr != nil {
|
||||
return countErr
|
||||
}
|
||||
if pending >= h.maxPendingJobs {
|
||||
c.Set(fiber.HeaderRetryAfter, "5")
|
||||
return common.Unavailable("processing queue is full")
|
||||
}
|
||||
}
|
||||
|
||||
// Reject an obviously oversized request before allocating a temp file or
|
||||
// decoding anything. Chunked requests are caught by the streaming counter
|
||||
// below.
|
||||
@@ -151,6 +160,7 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
}
|
||||
|
||||
attachmentID := uuid.New()
|
||||
jobID := uuid.New()
|
||||
attachment := model.Attachment{
|
||||
ID: attachmentID,
|
||||
ChatID: chatID,
|
||||
@@ -159,55 +169,51 @@ func (h *Handler) Upload(c fiber.Ctx) error {
|
||||
ContentType: info.ContentType,
|
||||
SizeBytes: written,
|
||||
SHA256: hex.EncodeToString(hasher.Sum(nil)),
|
||||
Status: "pending",
|
||||
Status: "queued",
|
||||
Variants: []model.Variant{},
|
||||
ErrorCode: nil,
|
||||
CreatedAt: common.NowUTC(),
|
||||
UpdatedAt: common.NowUTC(),
|
||||
// AGENT: the key is server-generated from UUIDs and never contains the
|
||||
// client filename. That keeps traversal and collisions out of the
|
||||
// object store even before application authorization is considered.
|
||||
ObjectKey: objectKey(chatID, attachmentID),
|
||||
}
|
||||
|
||||
if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil {
|
||||
return err
|
||||
// AGENT: keys are server-generated from UUIDs, never the client
|
||||
// filename. staging/ is the durable handoff prefix; the worker copies
|
||||
// the same bytes to the final attachments/ key unchanged.
|
||||
ObjectKey: objectKey(chatID, attachmentID),
|
||||
StagingKey: stagingKey(chatID, attachmentID),
|
||||
PipelineVersion: 1,
|
||||
}
|
||||
|
||||
if _, err = file.Seek(0, 0); err != nil {
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
return common.BadRequest("cannot read upload body")
|
||||
}
|
||||
if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil {
|
||||
slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
if err = h.objects.Put(ctx, attachment.StagingKey, file, written, attachment.ContentType); err != nil {
|
||||
slog.Error("stage attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
return common.Unavailable("object storage unavailable")
|
||||
}
|
||||
|
||||
updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID)
|
||||
if err != nil {
|
||||
slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID)
|
||||
h.compensate(attachment.ID, attachment.ObjectKey)
|
||||
return common.Unavailable("attachment metadata unavailable")
|
||||
job := model.JobRecord{
|
||||
Job: model.Job{
|
||||
JobID: jobID,
|
||||
AttachmentID: attachmentID,
|
||||
PipelineVersion: attachment.PipelineVersion,
|
||||
CreatedAt: attachment.CreatedAt,
|
||||
},
|
||||
MaxAttempts: h.jobMaxAttempts,
|
||||
}
|
||||
if err = h.store.CreateQueuedAttachment(ctx, attachment, job); err != nil {
|
||||
h.compensateStaging(attachment.StagingKey)
|
||||
return err
|
||||
}
|
||||
|
||||
attachment.Status = "ready"
|
||||
attachment.UpdatedAt = common.RFCFormTime(updatedAt)
|
||||
return c.Status(fiber.StatusCreated).JSON(attachment)
|
||||
return c.Status(fiber.StatusAccepted).JSON(attachment)
|
||||
}
|
||||
|
||||
// compensate is best effort by design. If either delete fails, the pending row
|
||||
// or the object remains visible to the reconciler, which retries after the
|
||||
// configured grace period.
|
||||
func (h *Handler) compensate(attachmentID uuid.UUID, key string) {
|
||||
// compensateStaging is best effort. If it fails, the reconciler removes an
|
||||
// object with no metadata row after the grace period.
|
||||
func (h *Handler) compensateStaging(key string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
||||
slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID)
|
||||
}
|
||||
if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil {
|
||||
slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID)
|
||||
slog.Error("compensate: delete staging object", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,10 +229,9 @@ func (h *Handler) Get(c fiber.Ctx) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if attachment.Status != "ready" {
|
||||
return common.Conflict("attachment is not ready")
|
||||
}
|
||||
|
||||
// queued/processing snapshots are part of lab5: the client polls this
|
||||
// endpoint until status becomes ready or failed.
|
||||
return c.JSON(attachment)
|
||||
}
|
||||
|
||||
@@ -242,7 +247,9 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
if variant == "" {
|
||||
variant = "original"
|
||||
}
|
||||
if variant != "original" {
|
||||
switch variant {
|
||||
case "original", "thumbnail", "crop", "optimized":
|
||||
default:
|
||||
return common.BadRequest("unsupported variant")
|
||||
}
|
||||
|
||||
@@ -254,9 +261,28 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
return common.Conflict("attachment is not ready")
|
||||
}
|
||||
|
||||
key := attachment.ObjectKey
|
||||
contentType := attachment.ContentType
|
||||
sizeBytes := attachment.SizeBytes
|
||||
if variant != "original" {
|
||||
found := false
|
||||
for _, candidate := range attachment.Variants {
|
||||
if candidate.Name == variant {
|
||||
key = store.VariantObjectKey(attachment.ObjectKey, candidate.Name)
|
||||
contentType = candidate.ContentType
|
||||
sizeBytes = candidate.SizeBytes
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return common.NotFound("attachment variant not found")
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout)
|
||||
|
||||
body, size, err := h.objects.Get(ctx, attachment.ObjectKey)
|
||||
body, size, err := h.objects.Get(ctx, key)
|
||||
if err != nil {
|
||||
cancel()
|
||||
if errors.Is(err, objectstore.ErrNotFound) {
|
||||
@@ -266,16 +292,16 @@ func (h *Handler) Download(c fiber.Ctx) error {
|
||||
slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID)
|
||||
return common.Unavailable("object storage unavailable")
|
||||
}
|
||||
if size != attachment.SizeBytes {
|
||||
if size != sizeBytes {
|
||||
_ = body.Close()
|
||||
cancel()
|
||||
slog.Error("attachment object size mismatch",
|
||||
"attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size)
|
||||
"attachment_id", attachment.ID, "metadata_size", sizeBytes, "object_size", size)
|
||||
return common.Unavailable("attachment content is inconsistent")
|
||||
}
|
||||
|
||||
c.Set(fiber.HeaderContentType, attachment.ContentType)
|
||||
c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10))
|
||||
c.Set(fiber.HeaderContentType, contentType)
|
||||
c.Set(fiber.HeaderContentLength, strconv.FormatInt(sizeBytes, 10))
|
||||
c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
|
||||
c.Set(fiber.HeaderCacheControl, "private, no-store")
|
||||
c.Set("X-Content-Type-Options", "nosniff")
|
||||
@@ -307,6 +333,10 @@ func objectKey(chatID, attachmentID uuid.UUID) string {
|
||||
return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID)
|
||||
}
|
||||
|
||||
func stagingKey(chatID, attachmentID uuid.UUID) string {
|
||||
return fmt.Sprintf("staging/%s/%s", chatID, attachmentID)
|
||||
}
|
||||
|
||||
// CleanupStaleUploads removes temp files abandoned by a previous crash. A
|
||||
// running request removes its own file in a defer; this only handles the
|
||||
// process-died-mid-upload case.
|
||||
|
||||
Reference in New Issue
Block a user