feat(attachments): accept uploads through staging and transactional outbox
This commit is contained in:
@@ -96,22 +96,19 @@ func printReport(report reconcile.Report) {
|
||||
report.ListPages,
|
||||
)
|
||||
fmt.Printf(
|
||||
"found orphan_objects=%d missing_attachments=%d stale_uploads=%d corrupt_objects=%d\n",
|
||||
"found orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
|
||||
len(report.OrphanObjects),
|
||||
len(report.MissingAttachments),
|
||||
len(report.StaleUploads),
|
||||
len(report.CorruptObjects),
|
||||
)
|
||||
fmt.Printf(
|
||||
"skipped fresh=%d referenced=%d inactive=%d\n",
|
||||
"skipped fresh=%d inactive=%d\n",
|
||||
report.SkippedFresh,
|
||||
report.SkippedReferenced,
|
||||
report.SkippedInactive,
|
||||
)
|
||||
fmt.Printf(
|
||||
"changes deleted_objects=%d deleted_rows=%d marked_failed=%d\n",
|
||||
"changes deleted_objects=%d marked_failed=%d\n",
|
||||
report.DeletedObjects,
|
||||
report.DeletedRows,
|
||||
report.MarkedFailed,
|
||||
)
|
||||
|
||||
@@ -124,9 +121,6 @@ func printReport(report reconcile.Report) {
|
||||
for _, id := range report.MissingAttachments {
|
||||
fmt.Printf("missing_attachment %s\n", id)
|
||||
}
|
||||
for _, id := range report.StaleUploads {
|
||||
fmt.Printf("stale_upload %s\n", id)
|
||||
}
|
||||
for _, message := range report.Errors {
|
||||
fmt.Printf("error %s\n", message)
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ SELECT
|
||||
sqlc.arg(chat_id)::uuid,
|
||||
sqlc.arg(owner_id)::uuid,
|
||||
sqlc.arg(s3_key)::text,
|
||||
sqlc.narg(staging_key)::text,
|
||||
sqlc.narg(job_id)::uuid,
|
||||
sqlc.arg(staging_key)::text,
|
||||
sqlc.arg(job_id)::uuid,
|
||||
sqlc.arg(filename)::text,
|
||||
sqlc.arg(content_type)::attachment_content_type,
|
||||
sqlc.arg(size_bytes)::int,
|
||||
@@ -68,22 +68,9 @@ FROM message_attachments
|
||||
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
|
||||
ORDER BY message_id, position;
|
||||
|
||||
-- name: ListStaleAttachmentUploads :many
|
||||
SELECT id, s3_key FROM attachments
|
||||
WHERE status = 'queued' AND created_at < sqlc.arg(cutoff)::timestamptz;
|
||||
|
||||
-- name: ListAttachmentsForReconcile :many
|
||||
SELECT
|
||||
a.id,
|
||||
a.s3_key,
|
||||
a.status,
|
||||
a.size_bytes,
|
||||
a.created_at,
|
||||
EXISTS (
|
||||
SELECT 1 FROM message_attachments ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
) AS referenced
|
||||
FROM attachments a;
|
||||
SELECT id, s3_key, status, size_bytes
|
||||
FROM attachments;
|
||||
|
||||
-- name: LockAttachmentJob :one
|
||||
SELECT * FROM attachments
|
||||
@@ -182,7 +169,3 @@ SET status = 'failed',
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND lease_until <= now();
|
||||
|
||||
-- name: DeleteQueuedAttachment :execrows
|
||||
DELETE FROM attachments
|
||||
WHERE id = sqlc.arg(id)::uuid AND status = 'queued';
|
||||
|
||||
+2
-11
@@ -99,16 +99,7 @@ func Run() error {
|
||||
backgroundWG.Go(func() {
|
||||
runIdempotencyCleanup(backgroundCtx, log, queries, cfg.Server.IdempotencyTTL, idempotencyCleanupInterval)
|
||||
})
|
||||
backgroundWG.Go(func() {
|
||||
runAttachmentCleanup(
|
||||
backgroundCtx,
|
||||
log,
|
||||
queries,
|
||||
s3Client,
|
||||
attachmentOrphanGracePeriod,
|
||||
attachmentCleanupInterval,
|
||||
)
|
||||
})
|
||||
|
||||
defer func() {
|
||||
stopBackground()
|
||||
backgroundWG.Wait()
|
||||
@@ -118,7 +109,7 @@ func Run() error {
|
||||
userService := service.NewUserService(queries)
|
||||
chatService := service.NewChatService(pool, queries)
|
||||
messageService := service.NewMessageService(pool, queries)
|
||||
attachmentService := service.NewAttachmentService(queries, s3Client, cfg.S3.MaxUploadBytes, cfg.S3.MaxImagePixels)
|
||||
attachmentService := service.NewAttachmentService(pool, queries, s3Client, cfg.S3.MaxUploadBytes, cfg.S3.MaxImagePixels)
|
||||
|
||||
router := NewRouter(
|
||||
log,
|
||||
|
||||
@@ -4,18 +4,11 @@ import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionCleanupInterval = time.Hour
|
||||
idempotencyCleanupInterval = time.Hour
|
||||
|
||||
attachmentCleanupInterval = 10 * time.Minute
|
||||
attachmentOrphanGracePeriod = time.Hour
|
||||
)
|
||||
|
||||
type sessionCleaner interface {
|
||||
@@ -26,15 +19,6 @@ type idempotencyCleaner interface {
|
||||
DeleteExpiredIdempotencyKeys(ctx context.Context, cutoff time.Time) (int64, error)
|
||||
}
|
||||
|
||||
type staleUploadLister interface {
|
||||
ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]db.ListStaleAttachmentUploadsRow, error)
|
||||
DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error)
|
||||
}
|
||||
|
||||
type objectDeleter interface {
|
||||
DeleteAdmin(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
func runSessionCleanup(ctx context.Context, log *slog.Logger, q sessionCleaner, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
@@ -86,64 +70,3 @@ func runIdempotencyCleanup(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runAttachmentCleanup(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
q staleUploadLister,
|
||||
store objectDeleter,
|
||||
grace time.Duration,
|
||||
interval time.Duration,
|
||||
) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
cleanupStaleUploads(ctx, log, q, store, grace)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupStaleUploads(
|
||||
ctx context.Context,
|
||||
log *slog.Logger,
|
||||
q staleUploadLister,
|
||||
store objectDeleter,
|
||||
grace time.Duration,
|
||||
) {
|
||||
cutoff := time.Now().Add(-grace)
|
||||
|
||||
rows, err := q.ListStaleAttachmentUploads(ctx, cutoff)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Error("list stale attachment uploads", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
|
||||
for _, row := range rows {
|
||||
removed, err := q.DeleteQueuedAttachment(ctx, row.ID)
|
||||
if err != nil {
|
||||
log.Error("delete stale upload row", "id", row.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
if removed == 0 {
|
||||
continue
|
||||
}
|
||||
if err := store.DeleteAdmin(ctx, row.S3Key); err != nil {
|
||||
log.Error("delete stale upload object", "key", row.S3Key, "error", err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
|
||||
if deleted > 0 {
|
||||
log.Info("stale attachment uploads deleted", "count", deleted)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,8 +144,8 @@ type CreateAttachmentParams struct {
|
||||
ChatID uuid.UUID
|
||||
OwnerID uuid.UUID
|
||||
S3Key string
|
||||
StagingKey *string
|
||||
JobID *uuid.UUID
|
||||
StagingKey string
|
||||
JobID uuid.UUID
|
||||
Filename string
|
||||
ContentType AttachmentContentType
|
||||
SizeBytes int
|
||||
@@ -219,19 +219,6 @@ func (q *Queries) CreateMessageAttachment(ctx context.Context, arg CreateMessage
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteQueuedAttachment = `-- name: DeleteQueuedAttachment :execrows
|
||||
DELETE FROM attachments
|
||||
WHERE id = $1::uuid AND status = 'queued'
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, deleteQueuedAttachment, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const failAttachmentJob = `-- name: FailAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
@@ -342,26 +329,15 @@ func (q *Queries) ListAttachmentsByIDs(ctx context.Context, arg ListAttachmentsB
|
||||
}
|
||||
|
||||
const listAttachmentsForReconcile = `-- name: ListAttachmentsForReconcile :many
|
||||
SELECT
|
||||
a.id,
|
||||
a.s3_key,
|
||||
a.status,
|
||||
a.size_bytes,
|
||||
a.created_at,
|
||||
EXISTS (
|
||||
SELECT 1 FROM message_attachments ma
|
||||
WHERE ma.attachment_id = a.id
|
||||
) AS referenced
|
||||
FROM attachments a
|
||||
SELECT id, s3_key, status, size_bytes
|
||||
FROM attachments
|
||||
`
|
||||
|
||||
type ListAttachmentsForReconcileRow struct {
|
||||
ID uuid.UUID
|
||||
S3Key string
|
||||
Status AttachmentStatus
|
||||
SizeBytes int
|
||||
CreatedAt time.Time
|
||||
Referenced bool
|
||||
ID uuid.UUID
|
||||
S3Key string
|
||||
Status AttachmentStatus
|
||||
SizeBytes int
|
||||
}
|
||||
|
||||
func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttachmentsForReconcileRow, error) {
|
||||
@@ -378,8 +354,6 @@ func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttach
|
||||
&i.S3Key,
|
||||
&i.Status,
|
||||
&i.SizeBytes,
|
||||
&i.CreatedAt,
|
||||
&i.Referenced,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -450,36 +424,6 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listStaleAttachmentUploads = `-- name: ListStaleAttachmentUploads :many
|
||||
SELECT id, s3_key FROM attachments
|
||||
WHERE status = 'queued' AND created_at < $1::timestamptz
|
||||
`
|
||||
|
||||
type ListStaleAttachmentUploadsRow struct {
|
||||
ID uuid.UUID
|
||||
S3Key string
|
||||
}
|
||||
|
||||
func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Time) ([]ListStaleAttachmentUploadsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listStaleAttachmentUploads, cutoff)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []ListStaleAttachmentUploadsRow{}
|
||||
for rows.Next() {
|
||||
var i ListStaleAttachmentUploadsRow
|
||||
if err := rows.Scan(&i.ID, &i.S3Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockAttachmentJob = `-- name: LockAttachmentJob :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
|
||||
@@ -77,7 +77,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
httpx.WriteJSON(w, http.StatusCreated, dto.ToAttachmentResponse(att))
|
||||
httpx.WriteJSON(w, http.StatusAccepted, dto.ToAttachmentResponse(att))
|
||||
}
|
||||
|
||||
func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -32,13 +32,10 @@ type Report struct {
|
||||
ListPages int `json:"list_pages"`
|
||||
OrphanObjects []string `json:"orphan_objects"`
|
||||
MissingAttachments []uuid.UUID `json:"missing_attachments"`
|
||||
StaleUploads []uuid.UUID `json:"stale_uploads"`
|
||||
CorruptObjects []string `json:"corrupt_objects"`
|
||||
SkippedFresh int `json:"skipped_fresh"`
|
||||
SkippedReferenced int `json:"skipped_referenced"`
|
||||
SkippedInactive int `json:"skipped_inactive"`
|
||||
DeletedObjects int `json:"deleted_objects"`
|
||||
DeletedRows int `json:"deleted_rows"`
|
||||
MarkedFailed int `json:"marked_failed"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Errors []string `json:"errors"`
|
||||
@@ -52,7 +49,6 @@ type ObjectClient interface {
|
||||
|
||||
type AttachmentQueries interface {
|
||||
ListAttachmentsForReconcile(ctx context.Context) ([]db.ListAttachmentsForReconcileRow, error)
|
||||
DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int64, error)
|
||||
MarkAttachmentFailedIfReady(ctx context.Context, arg db.MarkAttachmentFailedIfReadyParams) (int64, error)
|
||||
}
|
||||
|
||||
@@ -64,13 +60,11 @@ type Reconciler struct {
|
||||
}
|
||||
|
||||
type plan struct {
|
||||
orphans []s3store.ObjectInfo
|
||||
staleUploads []db.ListAttachmentsForReconcileRow
|
||||
missing []db.ListAttachmentsForReconcileRow
|
||||
corrupt []s3store.ObjectInfo
|
||||
skippedFresh int
|
||||
skippedReferenced int
|
||||
skippedInactive int
|
||||
orphans []s3store.ObjectInfo
|
||||
missing []db.ListAttachmentsForReconcileRow
|
||||
corrupt []s3store.ObjectInfo
|
||||
skippedFresh int
|
||||
skippedInactive int
|
||||
}
|
||||
|
||||
func New(store ObjectClient, queries AttachmentQueries, log *slog.Logger) *Reconciler {
|
||||
@@ -93,7 +87,6 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
report := Report{
|
||||
OrphanObjects: []string{},
|
||||
MissingAttachments: []uuid.UUID{},
|
||||
StaleUploads: []uuid.UUID{},
|
||||
CorruptObjects: []string{},
|
||||
DryRun: opts.DryRun,
|
||||
Errors: []string{},
|
||||
@@ -121,15 +114,11 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
for _, obj := range p.corrupt {
|
||||
report.CorruptObjects = append(report.CorruptObjects, obj.Key)
|
||||
}
|
||||
for _, row := range p.staleUploads {
|
||||
report.StaleUploads = append(report.StaleUploads, row.ID)
|
||||
}
|
||||
for _, row := range p.missing {
|
||||
report.MissingAttachments = append(report.MissingAttachments, row.ID)
|
||||
}
|
||||
|
||||
report.SkippedFresh = p.skippedFresh
|
||||
report.SkippedReferenced = p.skippedReferenced
|
||||
report.SkippedInactive = p.skippedInactive
|
||||
|
||||
if !opts.DryRun {
|
||||
@@ -146,10 +135,8 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
"list_pages", report.ListPages,
|
||||
"orphan_objects", len(report.OrphanObjects),
|
||||
"missing_attachments", len(report.MissingAttachments),
|
||||
"stale_uploads", len(report.StaleUploads),
|
||||
"corrupt_objects", len(report.CorruptObjects),
|
||||
"deleted_objects", report.DeletedObjects,
|
||||
"deleted_rows", report.DeletedRows,
|
||||
"marked_failed", report.MarkedFailed,
|
||||
"errors", len(report.Errors),
|
||||
)
|
||||
@@ -189,15 +176,7 @@ func buildPlan(
|
||||
|
||||
switch row.Status {
|
||||
case db.AttachmentStatusQueued:
|
||||
if now.Sub(row.CreatedAt) < grace {
|
||||
p.skippedFresh++
|
||||
continue
|
||||
}
|
||||
if row.Referenced {
|
||||
p.skippedReferenced++
|
||||
continue
|
||||
}
|
||||
p.staleUploads = append(p.staleUploads, row)
|
||||
p.skippedInactive++
|
||||
case db.AttachmentStatusReady:
|
||||
if !exists {
|
||||
p.missing = append(p.missing, row)
|
||||
@@ -227,9 +206,6 @@ func buildPlan(
|
||||
sort.Slice(p.corrupt, func(i, j int) bool {
|
||||
return p.corrupt[i].Key < p.corrupt[j].Key
|
||||
})
|
||||
sort.Slice(p.staleUploads, func(i, j int) bool {
|
||||
return p.staleUploads[i].ID.String() < p.staleUploads[j].ID.String()
|
||||
})
|
||||
sort.Slice(p.missing, func(i, j int) bool {
|
||||
return p.missing[i].ID.String() < p.missing[j].ID.String()
|
||||
})
|
||||
@@ -240,31 +216,6 @@ func buildPlan(
|
||||
func (r *Reconciler) apply(ctx context.Context, opts Options, p plan, report *Report) {
|
||||
deletes := 0
|
||||
|
||||
for _, row := range p.staleUploads {
|
||||
if deletes >= opts.MaxDeletes {
|
||||
report.Errors = append(report.Errors, "max deletes reached")
|
||||
break
|
||||
}
|
||||
|
||||
removed, err := r.q.DeleteQueuedAttachment(ctx, row.ID)
|
||||
if err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("delete row %s: %v", row.ID, err))
|
||||
continue
|
||||
}
|
||||
if removed == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
deletes++
|
||||
report.DeletedRows++
|
||||
|
||||
if err := r.store.DeleteAdmin(ctx, row.S3Key); err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("delete object %s: %v", row.S3Key, err))
|
||||
continue
|
||||
}
|
||||
report.DeletedObjects++
|
||||
}
|
||||
|
||||
for _, obj := range p.orphans {
|
||||
if deletes >= opts.MaxDeletes {
|
||||
report.Errors = append(report.Errors, "max deletes reached")
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/model"
|
||||
@@ -31,13 +33,15 @@ var (
|
||||
|
||||
type attachmentQueries interface {
|
||||
CreateAttachment(ctx context.Context, arg db.CreateAttachmentParams) (db.Attachment, error)
|
||||
CreateOutboxEntry(ctx context.Context, arg db.CreateOutboxEntryParams) (db.Outbox, error)
|
||||
GetAttachmentByID(ctx context.Context, id uuid.UUID) (db.Attachment, error)
|
||||
IsChatMember(ctx context.Context, arg db.IsChatMemberParams) (bool, error)
|
||||
MarkAttachmentReady(ctx context.Context, id uuid.UUID) (db.Attachment, error)
|
||||
}
|
||||
|
||||
type AttachmentService struct {
|
||||
pool txBeginner
|
||||
q attachmentQueries
|
||||
withTx func(pgx.Tx) attachmentQueries
|
||||
s3 *s3store.Client
|
||||
maxBytes int
|
||||
maxPixels int
|
||||
@@ -45,12 +49,17 @@ type AttachmentService struct {
|
||||
}
|
||||
|
||||
func NewAttachmentService(
|
||||
q attachmentQueries,
|
||||
pool *pgxpool.Pool,
|
||||
q *db.Queries,
|
||||
s3 *s3store.Client,
|
||||
maxBytes, maxPixels int,
|
||||
) *AttachmentService {
|
||||
return &AttachmentService{
|
||||
q: q,
|
||||
pool: pool,
|
||||
q: q,
|
||||
withTx: func(tx pgx.Tx) attachmentQueries {
|
||||
return q.WithTx(tx)
|
||||
},
|
||||
s3: s3,
|
||||
maxBytes: maxBytes,
|
||||
maxPixels: maxPixels,
|
||||
@@ -107,19 +116,37 @@ func (s *AttachmentService) Upload(
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
jobID, err := uuid.NewV7()
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
key := "chats/" + chatID.String() + "/attachments/" + id.String()
|
||||
stagingKey := "staging/" + id.String()
|
||||
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
_, err = s.q.CreateAttachment(ctx, db.CreateAttachmentParams{
|
||||
if err := s.s3.Put(ctx, stagingKey, tmp, size, string(detectedType)); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
q := s.withTx(tx)
|
||||
|
||||
row, err := q.CreateAttachment(ctx, db.CreateAttachmentParams{
|
||||
ID: id,
|
||||
ChatID: chatID,
|
||||
OwnerID: userID,
|
||||
S3Key: key,
|
||||
StagingKey: stagingKey,
|
||||
JobID: jobID,
|
||||
Filename: filename,
|
||||
ContentType: db.AttachmentContentType(detectedType),
|
||||
SizeBytes: int(size),
|
||||
@@ -135,12 +162,15 @@ func (s *AttachmentService) Upload(
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
if err := s.s3.Put(ctx, key, tmp, size, string(detectedType)); err != nil {
|
||||
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: id,
|
||||
Attempt: 1,
|
||||
AvailableAt: row.CreatedAt,
|
||||
}); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
row, err := s.q.MarkAttachmentReady(ctx, id)
|
||||
if err != nil {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ func (s *MessageService) Create(
|
||||
return model.Message{}, ErrAttachmentNotFound
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status != db.AttachmentStatusReady {
|
||||
if status == db.AttachmentStatusFailed {
|
||||
return model.Message{}, ErrAttachmentNotReady
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -62,7 +61,7 @@ func runReconcile(t *testing.T, dryRun bool, grace string) reconcile.Report {
|
||||
if err := json.Unmarshal(output, &report); err != nil {
|
||||
t.Fatalf("decode reconcile report: %v\n%s", err, output)
|
||||
}
|
||||
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d stale=%d skipped_fresh=%d skipped_referenced=%d deleted_objects=%d deleted_rows=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), len(report.StaleUploads), report.SkippedFresh, report.SkippedReferenced, report.DeletedObjects, report.DeletedRows, report.MarkedFailed, report.Errors)
|
||||
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d skipped_fresh=%d skipped_inactive=%d deleted_objects=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), report.SkippedFresh, report.SkippedInactive, report.DeletedObjects, report.MarkedFailed, report.Errors)
|
||||
if len(report.Errors) != 0 {
|
||||
t.Fatal("reconciler reported errors")
|
||||
}
|
||||
@@ -86,7 +85,7 @@ func TestReconcileRepair(t *testing.T) {
|
||||
if !slices.Contains(dry.OrphanObjects, originalKey) || !slices.Contains(dry.MissingAttachments, att.ID) {
|
||||
t.Fatal("dry-run did not detect both injected inconsistencies")
|
||||
}
|
||||
if dry.DeletedObjects != 0 || dry.DeletedRows != 0 || dry.MarkedFailed != 0 {
|
||||
if dry.DeletedObjects != 0 || dry.MarkedFailed != 0 {
|
||||
t.Fatal("dry-run changed data")
|
||||
}
|
||||
var status, code string
|
||||
@@ -110,7 +109,7 @@ func TestReconcileRepair(t *testing.T) {
|
||||
if slices.Contains(again.OrphanObjects, originalKey) || slices.Contains(again.MissingAttachments, att.ID) {
|
||||
t.Fatal("fixture still requires repair on repeat")
|
||||
}
|
||||
if again.DeletedObjects != 0 || again.DeletedRows != 0 || again.MarkedFailed != 0 {
|
||||
if again.DeletedObjects != 0 || again.MarkedFailed != 0 {
|
||||
t.Fatal("repeat repair changed data; run on an idle test stand")
|
||||
}
|
||||
t.Log("PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes")
|
||||
@@ -125,7 +124,7 @@ func TestReconcileSafety(t *testing.T) {
|
||||
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{used.ID})
|
||||
requireStatus(t, resp, http.StatusCreated)
|
||||
resp.Body.Close()
|
||||
// Exercise both grace-period protection and protection of referenced queued uploads.
|
||||
// Queued attachments must be preserved regardless of age or message references.
|
||||
reconcileSQL(t, conn, "UPDATE attachments SET status='queued' WHERE id=$1", fresh.ID)
|
||||
reconcileSQL(t, conn, "UPDATE attachments SET status='queued', created_at=now()-interval '2 hours' WHERE id=$1", used.ID)
|
||||
t.Cleanup(func() {
|
||||
@@ -135,8 +134,8 @@ func TestReconcileSafety(t *testing.T) {
|
||||
})
|
||||
for _, dryRun := range []bool{true, false} {
|
||||
report := runReconcile(t, dryRun, "1h")
|
||||
if report.SkippedFresh < 1 || report.SkippedReferenced < 1 {
|
||||
t.Fatal("fresh/referenced protection was not exercised")
|
||||
if report.SkippedInactive < 2 {
|
||||
t.Fatal("queued attachment protection was not exercised")
|
||||
}
|
||||
for _, att := range []uuid.UUID{fresh.ID, used.ID} {
|
||||
var status string
|
||||
@@ -147,18 +146,11 @@ func TestReconcileSafety(t *testing.T) {
|
||||
t.Fatalf("protected fixture changed status: %s", status)
|
||||
}
|
||||
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.String()+"/content", user.Token, nil)
|
||||
requireStatus(t, resp, http.StatusOK)
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
requireStatus(t, resp, http.StatusConflict)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, makePNG(t, 32, 32)) {
|
||||
t.Fatal("protected original changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Log("PASS: fresh and old referenced uploads preserved, originals unchanged")
|
||||
t.Log("PASS: queued uploads preserved regardless of age or references; content remains unavailable")
|
||||
}
|
||||
|
||||
func TestReconcileCost(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user