feat(attachments): accept uploads through staging and transactional outbox

This commit is contained in:
kk10
2026-11-14 19:52:51 +03:00
parent 4dbbd65bcf
commit 8b0ab5440d
10 changed files with 70 additions and 262 deletions
+3 -9
View File
@@ -96,22 +96,19 @@ func printReport(report reconcile.Report) {
report.ListPages,
)
fmt.Printf(
"found orphan_objects=%d missing_attachments=%d stale_uploads=%d corrupt_objects=%d\n",
"found orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
len(report.OrphanObjects),
len(report.MissingAttachments),
len(report.StaleUploads),
len(report.CorruptObjects),
)
fmt.Printf(
"skipped fresh=%d referenced=%d inactive=%d\n",
"skipped fresh=%d inactive=%d\n",
report.SkippedFresh,
report.SkippedReferenced,
report.SkippedInactive,
)
fmt.Printf(
"changes deleted_objects=%d deleted_rows=%d marked_failed=%d\n",
"changes deleted_objects=%d marked_failed=%d\n",
report.DeletedObjects,
report.DeletedRows,
report.MarkedFailed,
)
@@ -124,9 +121,6 @@ func printReport(report reconcile.Report) {
for _, id := range report.MissingAttachments {
fmt.Printf("missing_attachment %s\n", id)
}
for _, id := range report.StaleUploads {
fmt.Printf("stale_upload %s\n", id)
}
for _, message := range report.Errors {
fmt.Printf("error %s\n", message)
}
+4 -21
View File
@@ -19,8 +19,8 @@ SELECT
sqlc.arg(chat_id)::uuid,
sqlc.arg(owner_id)::uuid,
sqlc.arg(s3_key)::text,
sqlc.narg(staging_key)::text,
sqlc.narg(job_id)::uuid,
sqlc.arg(staging_key)::text,
sqlc.arg(job_id)::uuid,
sqlc.arg(filename)::text,
sqlc.arg(content_type)::attachment_content_type,
sqlc.arg(size_bytes)::int,
@@ -68,22 +68,9 @@ FROM message_attachments
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
ORDER BY message_id, position;
-- name: ListStaleAttachmentUploads :many
SELECT id, s3_key FROM attachments
WHERE status = 'queued' AND created_at < sqlc.arg(cutoff)::timestamptz;
-- name: ListAttachmentsForReconcile :many
SELECT
a.id,
a.s3_key,
a.status,
a.size_bytes,
a.created_at,
EXISTS (
SELECT 1 FROM message_attachments ma
WHERE ma.attachment_id = a.id
) AS referenced
FROM attachments a;
SELECT id, s3_key, status, size_bytes
FROM attachments;
-- name: LockAttachmentJob :one
SELECT * FROM attachments
@@ -182,7 +169,3 @@ SET status = 'failed',
WHERE id = sqlc.arg(id)::uuid
AND status = 'processing'
AND lease_until <= now();
-- name: DeleteQueuedAttachment :execrows
DELETE FROM attachments
WHERE id = sqlc.arg(id)::uuid AND status = 'queued';
+2 -11
View File
@@ -99,16 +99,7 @@ func Run() error {
backgroundWG.Go(func() {
runIdempotencyCleanup(backgroundCtx, log, queries, cfg.Server.IdempotencyTTL, idempotencyCleanupInterval)
})
backgroundWG.Go(func() {
runAttachmentCleanup(
backgroundCtx,
log,
queries,
s3Client,
attachmentOrphanGracePeriod,
attachmentCleanupInterval,
)
})
defer func() {
stopBackground()
backgroundWG.Wait()
@@ -118,7 +109,7 @@ func Run() error {
userService := service.NewUserService(queries)
chatService := service.NewChatService(pool, queries)
messageService := service.NewMessageService(pool, queries)
attachmentService := service.NewAttachmentService(queries, s3Client, cfg.S3.MaxUploadBytes, cfg.S3.MaxImagePixels)
attachmentService := service.NewAttachmentService(pool, queries, s3Client, cfg.S3.MaxUploadBytes, cfg.S3.MaxImagePixels)
router := NewRouter(
log,
-77
View File
@@ -4,18 +4,11 @@ import (
"context"
"log/slog"
"time"
"github.com/google/uuid"
"mtusi2026-messenger/internal/db"
)
const (
sessionCleanupInterval = time.Hour
idempotencyCleanupInterval = time.Hour
attachmentCleanupInterval = 10 * time.Minute
attachmentOrphanGracePeriod = time.Hour
)
type sessionCleaner interface {
@@ -26,15 +19,6 @@ type idempotencyCleaner interface {
DeleteExpiredIdempotencyKeys(ctx context.Context, cutoff time.Time) (int64, error)
}
type staleUploadLister interface {
ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]db.ListStaleAttachmentUploadsRow, error)
DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error)
}
type objectDeleter interface {
DeleteAdmin(ctx context.Context, key string) error
}
func runSessionCleanup(ctx context.Context, log *slog.Logger, q sessionCleaner, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
@@ -86,64 +70,3 @@ func runIdempotencyCleanup(
}
}
}
func runAttachmentCleanup(
ctx context.Context,
log *slog.Logger,
q staleUploadLister,
store objectDeleter,
grace time.Duration,
interval time.Duration,
) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
cleanupStaleUploads(ctx, log, q, store, grace)
}
}
}
func cleanupStaleUploads(
ctx context.Context,
log *slog.Logger,
q staleUploadLister,
store objectDeleter,
grace time.Duration,
) {
cutoff := time.Now().Add(-grace)
rows, err := q.ListStaleAttachmentUploads(ctx, cutoff)
if err != nil {
if ctx.Err() == nil {
log.Error("list stale attachment uploads", "error", err)
}
return
}
deleted := 0
for _, row := range rows {
removed, err := q.DeleteQueuedAttachment(ctx, row.ID)
if err != nil {
log.Error("delete stale upload row", "id", row.ID, "error", err)
continue
}
if removed == 0 {
continue
}
if err := store.DeleteAdmin(ctx, row.S3Key); err != nil {
log.Error("delete stale upload object", "key", row.S3Key, "error", err)
continue
}
deleted++
}
if deleted > 0 {
log.Info("stale attachment uploads deleted", "count", deleted)
}
}
+8 -64
View File
@@ -144,8 +144,8 @@ type CreateAttachmentParams struct {
ChatID uuid.UUID
OwnerID uuid.UUID
S3Key string
StagingKey *string
JobID *uuid.UUID
StagingKey string
JobID uuid.UUID
Filename string
ContentType AttachmentContentType
SizeBytes int
@@ -219,19 +219,6 @@ func (q *Queries) CreateMessageAttachment(ctx context.Context, arg CreateMessage
return err
}
const deleteQueuedAttachment = `-- name: DeleteQueuedAttachment :execrows
DELETE FROM attachments
WHERE id = $1::uuid AND status = 'queued'
`
func (q *Queries) DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error) {
result, err := q.db.Exec(ctx, deleteQueuedAttachment, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const failAttachmentJob = `-- name: FailAttachmentJob :execrows
UPDATE attachments
SET status = 'failed',
@@ -342,26 +329,15 @@ func (q *Queries) ListAttachmentsByIDs(ctx context.Context, arg ListAttachmentsB
}
const listAttachmentsForReconcile = `-- name: ListAttachmentsForReconcile :many
SELECT
a.id,
a.s3_key,
a.status,
a.size_bytes,
a.created_at,
EXISTS (
SELECT 1 FROM message_attachments ma
WHERE ma.attachment_id = a.id
) AS referenced
FROM attachments a
SELECT id, s3_key, status, size_bytes
FROM attachments
`
type ListAttachmentsForReconcileRow struct {
ID uuid.UUID
S3Key string
Status AttachmentStatus
SizeBytes int
CreatedAt time.Time
Referenced bool
ID uuid.UUID
S3Key string
Status AttachmentStatus
SizeBytes int
}
func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttachmentsForReconcileRow, error) {
@@ -378,8 +354,6 @@ func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttach
&i.S3Key,
&i.Status,
&i.SizeBytes,
&i.CreatedAt,
&i.Referenced,
); err != nil {
return nil, err
}
@@ -450,36 +424,6 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag
return items, nil
}
const listStaleAttachmentUploads = `-- name: ListStaleAttachmentUploads :many
SELECT id, s3_key FROM attachments
WHERE status = 'queued' AND created_at < $1::timestamptz
`
type ListStaleAttachmentUploadsRow struct {
ID uuid.UUID
S3Key string
}
func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]ListStaleAttachmentUploadsRow, error) {
rows, err := q.db.Query(ctx, listStaleAttachmentUploads, cutoff)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListStaleAttachmentUploadsRow{}
for rows.Next() {
var i ListStaleAttachmentUploadsRow
if err := rows.Scan(&i.ID, &i.S3Key); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockAttachmentJob = `-- name: LockAttachmentJob :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
WHERE id = $1::uuid
+1 -1
View File
@@ -77,7 +77,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
return
}
httpx.WriteJSON(w, http.StatusCreated, dto.ToAttachmentResponse(att))
httpx.WriteJSON(w, http.StatusAccepted, dto.ToAttachmentResponse(att))
}
func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
+6 -55
View File
@@ -32,13 +32,10 @@ type Report struct {
ListPages int `json:"list_pages"`
OrphanObjects []string `json:"orphan_objects"`
MissingAttachments []uuid.UUID `json:"missing_attachments"`
StaleUploads []uuid.UUID `json:"stale_uploads"`
CorruptObjects []string `json:"corrupt_objects"`
SkippedFresh int `json:"skipped_fresh"`
SkippedReferenced int `json:"skipped_referenced"`
SkippedInactive int `json:"skipped_inactive"`
DeletedObjects int `json:"deleted_objects"`
DeletedRows int `json:"deleted_rows"`
MarkedFailed int `json:"marked_failed"`
DryRun bool `json:"dry_run"`
Errors []string `json:"errors"`
@@ -52,7 +49,6 @@ type ObjectClient interface {
type AttachmentQueries interface {
ListAttachmentsForReconcile(ctx context.Context) ([]db.ListAttachmentsForReconcileRow, error)
DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error)
MarkAttachmentFailedIfReady(ctx context.Context, arg db.MarkAttachmentFailedIfReadyParams) (int64, error)
}
@@ -64,13 +60,11 @@ type Reconciler struct {
}
type plan struct {
orphans []s3store.ObjectInfo
staleUploads []db.ListAttachmentsForReconcileRow
missing []db.ListAttachmentsForReconcileRow
corrupt []s3store.ObjectInfo
skippedFresh int
skippedReferenced int
skippedInactive int
orphans []s3store.ObjectInfo
missing []db.ListAttachmentsForReconcileRow
corrupt []s3store.ObjectInfo
skippedFresh int
skippedInactive int
}
func New(store ObjectClient, queries AttachmentQueries, log *slog.Logger) *Reconciler {
@@ -93,7 +87,6 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
report := Report{
OrphanObjects: []string{},
MissingAttachments: []uuid.UUID{},
StaleUploads: []uuid.UUID{},
CorruptObjects: []string{},
DryRun: opts.DryRun,
Errors: []string{},
@@ -121,15 +114,11 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
for _, obj := range p.corrupt {
report.CorruptObjects = append(report.CorruptObjects, obj.Key)
}
for _, row := range p.staleUploads {
report.StaleUploads = append(report.StaleUploads, row.ID)
}
for _, row := range p.missing {
report.MissingAttachments = append(report.MissingAttachments, row.ID)
}
report.SkippedFresh = p.skippedFresh
report.SkippedReferenced = p.skippedReferenced
report.SkippedInactive = p.skippedInactive
if !opts.DryRun {
@@ -146,10 +135,8 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
"list_pages", report.ListPages,
"orphan_objects", len(report.OrphanObjects),
"missing_attachments", len(report.MissingAttachments),
"stale_uploads", len(report.StaleUploads),
"corrupt_objects", len(report.CorruptObjects),
"deleted_objects", report.DeletedObjects,
"deleted_rows", report.DeletedRows,
"marked_failed", report.MarkedFailed,
"errors", len(report.Errors),
)
@@ -189,15 +176,7 @@ func buildPlan(
switch row.Status {
case db.AttachmentStatusQueued:
if now.Sub(row.CreatedAt) < grace {
p.skippedFresh++
continue
}
if row.Referenced {
p.skippedReferenced++
continue
}
p.staleUploads = append(p.staleUploads, row)
p.skippedInactive++
case db.AttachmentStatusReady:
if !exists {
p.missing = append(p.missing, row)
@@ -227,9 +206,6 @@ func buildPlan(
sort.Slice(p.corrupt, func(i, j int) bool {
return p.corrupt[i].Key < p.corrupt[j].Key
})
sort.Slice(p.staleUploads, func(i, j int) bool {
return p.staleUploads[i].ID.String() < p.staleUploads[j].ID.String()
})
sort.Slice(p.missing, func(i, j int) bool {
return p.missing[i].ID.String() < p.missing[j].ID.String()
})
@@ -240,31 +216,6 @@ func buildPlan(
func (r *Reconciler) apply(ctx context.Context, opts Options, p plan, report *Report) {
deletes := 0
for _, row := range p.staleUploads {
if deletes >= opts.MaxDeletes {
report.Errors = append(report.Errors, "max deletes reached")
break
}
removed, err := r.q.DeleteQueuedAttachment(ctx, row.ID)
if err != nil {
report.Errors = append(report.Errors, fmt.Sprintf("delete row %s: %v", row.ID, err))
continue
}
if removed == 0 {
continue
}
deletes++
report.DeletedRows++
if err := r.store.DeleteAdmin(ctx, row.S3Key); err != nil {
report.Errors = append(report.Errors, fmt.Sprintf("delete object %s: %v", row.S3Key, err))
continue
}
report.DeletedObjects++
}
for _, obj := range p.orphans {
if deletes >= opts.MaxDeletes {
report.Errors = append(report.Errors, "max deletes reached")
+37 -7
View File
@@ -14,6 +14,8 @@ import (
"os"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"mtusi2026-messenger/internal/db"
"mtusi2026-messenger/internal/model"
@@ -31,13 +33,15 @@ var (
type attachmentQueries interface {
CreateAttachment(ctx context.Context, arg db.CreateAttachmentParams) (db.Attachment, error)
CreateOutboxEntry(ctx context.Context, arg db.CreateOutboxEntryParams) (db.Outbox, error)
GetAttachmentByID(ctx context.Context, id uuid.UUID) (db.Attachment, error)
IsChatMember(ctx context.Context, arg db.IsChatMemberParams) (bool, error)
MarkAttachmentReady(ctx context.Context, id uuid.UUID) (db.Attachment, error)
}
type AttachmentService struct {
pool txBeginner
q attachmentQueries
withTx func(pgx.Tx) attachmentQueries
s3 *s3store.Client
maxBytes int
maxPixels int
@@ -45,12 +49,17 @@ type AttachmentService struct {
}
func NewAttachmentService(
q attachmentQueries,
pool *pgxpool.Pool,
q *db.Queries,
s3 *s3store.Client,
maxBytes, maxPixels int,
) *AttachmentService {
return &AttachmentService{
q: q,
pool: pool,
q: q,
withTx: func(tx pgx.Tx) attachmentQueries {
return q.WithTx(tx)
},
s3: s3,
maxBytes: maxBytes,
maxPixels: maxPixels,
@@ -107,19 +116,37 @@ func (s *AttachmentService) Upload(
if err != nil {
return model.Attachment{}, err
}
jobID, err := uuid.NewV7()
if err != nil {
return model.Attachment{}, err
}
hash := hex.EncodeToString(hasher.Sum(nil))
key := "chats/" + chatID.String() + "/attachments/" + id.String()
stagingKey := "staging/" + id.String()
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return model.Attachment{}, err
}
_, err = s.q.CreateAttachment(ctx, db.CreateAttachmentParams{
if err := s.s3.Put(ctx, stagingKey, tmp, size, string(detectedType)); err != nil {
return model.Attachment{}, err
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return model.Attachment{}, err
}
defer tx.Rollback(ctx)
q := s.withTx(tx)
row, err := q.CreateAttachment(ctx, db.CreateAttachmentParams{
ID: id,
ChatID: chatID,
OwnerID: userID,
S3Key: key,
StagingKey: stagingKey,
JobID: jobID,
Filename: filename,
ContentType: db.AttachmentContentType(detectedType),
SizeBytes: int(size),
@@ -135,12 +162,15 @@ func (s *AttachmentService) Upload(
return model.Attachment{}, err
}
if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil {
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
AttachmentID: id,
Attempt: 1,
AvailableAt: row.CreatedAt,
}); err != nil {
return model.Attachment{}, err
}
row, err := s.q.MarkAttachmentReady(ctx, id)
if err != nil {
if err := tx.Commit(ctx); err != nil {
return model.Attachment{}, err
}
+1 -1
View File
@@ -77,7 +77,7 @@ func (s *MessageService) Create(
return model.Message{}, ErrAttachmentNotFound
}
for _, status := range statuses {
if status != db.AttachmentStatusReady {
if status == db.AttachmentStatusFailed {
return model.Message{}, ErrAttachmentNotReady
}
}
+8 -16
View File
@@ -6,7 +6,6 @@ import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"os"
"os/exec"
@@ -62,7 +61,7 @@ func runReconcile(t *testing.T, dryRun bool, grace string) reconcile.Report {
if err := json.Unmarshal(output, &report); err != nil {
t.Fatalf("decode reconcile report: %v\n%s", err, output)
}
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d stale=%d skipped_fresh=%d skipped_referenced=%d deleted_objects=%d deleted_rows=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), len(report.StaleUploads), report.SkippedFresh, report.SkippedReferenced, report.DeletedObjects, report.DeletedRows, report.MarkedFailed, report.Errors)
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d skipped_fresh=%d skipped_inactive=%d deleted_objects=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), report.SkippedFresh, report.SkippedInactive, report.DeletedObjects, report.MarkedFailed, report.Errors)
if len(report.Errors) != 0 {
t.Fatal("reconciler reported errors")
}
@@ -86,7 +85,7 @@ func TestReconcileRepair(t *testing.T) {
if !slices.Contains(dry.OrphanObjects, originalKey) || !slices.Contains(dry.MissingAttachments, att.ID) {
t.Fatal("dry-run did not detect both injected inconsistencies")
}
if dry.DeletedObjects != 0 || dry.DeletedRows != 0 || dry.MarkedFailed != 0 {
if dry.DeletedObjects != 0 || dry.MarkedFailed != 0 {
t.Fatal("dry-run changed data")
}
var status, code string
@@ -110,7 +109,7 @@ func TestReconcileRepair(t *testing.T) {
if slices.Contains(again.OrphanObjects, originalKey) || slices.Contains(again.MissingAttachments, att.ID) {
t.Fatal("fixture still requires repair on repeat")
}
if again.DeletedObjects != 0 || again.DeletedRows != 0 || again.MarkedFailed != 0 {
if again.DeletedObjects != 0 || again.MarkedFailed != 0 {
t.Fatal("repeat repair changed data; run on an idle test stand")
}
t.Log("PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes")
@@ -125,7 +124,7 @@ func TestReconcileSafety(t *testing.T) {
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{used.ID})
requireStatus(t, resp, http.StatusCreated)
resp.Body.Close()
// Exercise both grace-period protection and protection of referenced queued uploads.
// Queued attachments must be preserved regardless of age or message references.
reconcileSQL(t, conn, "UPDATE attachments SET status='queued' WHERE id=$1", fresh.ID)
reconcileSQL(t, conn, "UPDATE attachments SET status='queued', created_at=now()-interval '2 hours' WHERE id=$1", used.ID)
t.Cleanup(func() {
@@ -135,8 +134,8 @@ func TestReconcileSafety(t *testing.T) {
})
for _, dryRun := range []bool{true, false} {
report := runReconcile(t, dryRun, "1h")
if report.SkippedFresh < 1 || report.SkippedReferenced < 1 {
t.Fatal("fresh/referenced protection was not exercised")
if report.SkippedInactive < 2 {
t.Fatal("queued attachment protection was not exercised")
}
for _, att := range []uuid.UUID{fresh.ID, used.ID} {
var status string
@@ -147,18 +146,11 @@ func TestReconcileSafety(t *testing.T) {
t.Fatalf("protected fixture changed status: %s", status)
}
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.String()+"/content", user.Token, nil)
requireStatus(t, resp, http.StatusOK)
data, err := io.ReadAll(resp.Body)
requireStatus(t, resp, http.StatusConflict)
resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, makePNG(t, 32, 32)) {
t.Fatal("protected original changed")
}
}
}
t.Log("PASS: fresh and old referenced uploads preserved, originals unchanged")
t.Log("PASS: queued uploads preserved regardless of age or references; content remains unavailable")
}
func TestReconcileCost(t *testing.T) {