feat:level4

This commit is contained in:
Оля Мороз
2026-11-16 14:41:07 +03:00
parent 0a361367c7
commit 90f407b174
16 changed files with 1270 additions and 77 deletions
+70 -11
View File
@@ -419,6 +419,19 @@ func safeFilename(name string) (string, bool) {
return name, true
}
func errorCodeFor(err error) string {
switch {
case errors.Is(err, domain.ErrTooLarge):
return "payload_too_large"
case errors.Is(err, domain.ErrUnsupported):
return "unsupported_media_type"
case errors.Is(err, domain.ErrNotImage):
return "invalid_image"
default:
return "storage_error"
}
}
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
@@ -442,17 +455,18 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
return
}
objectKey := "attachments/" + chatID + "/" + uuid.NewString()
pendingID, err := h.Store.CreateAttachmentPending(r.Context(), chatID, me.ID, objectKey, filename)
stagingKey := domain.StagingPrefix + chatID + "/" + uuid.NewString()
pendingID, err := h.Store.CreateAttachmentPending(
r.Context(), chatID, me.ID, stagingKey, filename)
if err != nil {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
res, err := h.S3.PutValidatedImage(r.Context(), objectKey, r.Body)
res, err := h.S3.PutValidatedImage(r.Context(), stagingKey, r.Body)
if err != nil {
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, err.Error())
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "upload_failed")
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, errorCodeFor(err))
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "upload_failed")
switch {
case errors.Is(err, domain.ErrTooLarge):
writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large")
@@ -466,21 +480,24 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
return
}
att, err := h.Store.FinalizeAttachment(r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256, res.Width, res.Height)
// Атомарно: attachment queued + outbox-строка. 202 только после COMMIT.
att, err := h.Store.EnqueueAttachment(
r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256,
res.Width, res.Height, domain.PipelineVersion)
if err != nil {
// S3 PUT прошёл, COMMIT — нет: объект в GC.
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "finalize_failed")
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "enqueue_failed")
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
writeJSON(w, http.StatusCreated, att)
writeJSON(w, http.StatusAccepted, att)
})(w, r)
}
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
me, _ := userFrom(r)
att, err := h.Store.GetAttachmentForUser(r.Context(), r.PathValue("attachment_id"), me.ID)
id := r.PathValue("attachment_id")
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
switch {
case err == nil:
writeJSON(w, http.StatusOK, att)
@@ -505,7 +522,27 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
key, err := h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
switch att.Status {
case domain.AttachmentReady:
// ok
case domain.AttachmentFailed:
writeError(w, r, http.StatusConflict, "conflict", "attachment failed")
return
default:
writeError(w, r, http.StatusConflict, "conflict", "attachment not ready")
return
}
variant := r.URL.Query().Get("variant")
if variant == "" {
variant = "original"
}
var key string
if variant == "original" {
key, err = h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
} else {
key, _, err = h.Store.GetVariantKey(r.Context(), att.ID, att.PipelineVersion, variant)
}
if err != nil {
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
return
@@ -535,3 +572,25 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
}
})(w, r)
}
func (h *Handler) ReplayAttachment(w http.ResponseWriter, r *http.Request) {
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
// Только owner или участник чата; упрощённо — участник чата.
me, _ := userFrom(r)
id := r.PathValue("attachment_id")
if _, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID); err != nil {
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
return
}
err := h.Store.ReplayFailed(r.Context(), id)
if errors.Is(err, domain.ErrConflict) {
writeError(w, r, http.StatusConflict, "conflict", "not in failed state")
return
}
if err != nil {
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
return
}
w.WriteHeader(http.StatusAccepted)
})(w, r)
}
+1
View File
@@ -18,6 +18,7 @@ func NewRouter(h *Handler, instanceID string) http.Handler {
mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment)
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment)
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment)
mux.HandleFunc("POST /api/v1/attachments/{attachment_id}/replay", h.ReplayAttachment)
// Порядок: instanceID снаружи, requestID внутри.
return withInstanceID(instanceID)(withRequestID(mux))
+12
View File
@@ -0,0 +1,12 @@
package domain
const (
AttachmentQueued = "queued"
AttachmentProcessing = "processing"
AttachmentReady = "ready"
AttachmentFailed = "failed"
)
const PipelineVersion = 1
var RequiredVariants = []string{"original", "thumbnail", "crop", "optimized"}
+450 -43
View File
@@ -3,6 +3,7 @@ package domain
import (
"context"
"errors"
"math/rand"
"time"
"github.com/google/uuid"
@@ -11,57 +12,25 @@ import (
// CreateAttachmentPending резервирует строку ДО записи в S3.
// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится.
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, objectKey, filename string) (string, error) {
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, stagingKey, filename string) (string, error) {
id := uuid.NewString()
_, err := s.pool.Exec(ctx,
`INSERT INTO attachments (id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, width, height, status)
VALUES ($1,$2,$3,$4,$5,'image/png',1,repeat('0',64),1,1,'pending')`,
id, chatID, ownerID, objectKey, filename)
`INSERT INTO attachments (id, chat_id, owner_id, object_key, staging_key, filename,
content_type, size_bytes, sha256, width, height,
pipeline_version, status)
VALUES ($1,$2,$3,$4,$4,$5,'application/octet-stream',0,repeat('0',64),0,0,1,'queued')`,
id, chatID, ownerID, stagingKey, filename)
if err != nil {
return "", classifyDBError(err)
}
return id, nil
}
// FinalizeAttachment атомарно переводит pending→ready с реальными метаданными.
func (s *Store) FinalizeAttachment(ctx context.Context, id, contentType string, size int64, sha string, w, h int) (*Attachment, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var a Attachment
var errCode *string
err = tx.QueryRow(ctx,
`UPDATE attachments
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
status='ready', error_code=NULL, updated_at=now()
WHERE id=$1 AND status='pending'
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at`,
id, contentType, size, sha, w, h,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
a.ErrorCode = errCode
a.Variants = []Variant{}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
return &a, nil
}
// MarkAttachmentFailed помечает pending как failed (компенсация).
func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachments SET status='failed', error_code=$2, updated_at=now()
WHERE id=$1 AND status='pending'`, id, code)
WHERE id=$1 AND status='queued'`, id, code)
return classifyDBError(err)
}
@@ -79,10 +48,10 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
var errCode *string
err := s.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.status, a.error_code, a.created_at, a.updated_at
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id
WHERE a.id=$1 AND m.user_id=$2 AND a.status='ready'`, id, userID,
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
if errors.Is(err, pgx.ErrNoRows) {
@@ -93,6 +62,27 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
}
a.ErrorCode = errCode
a.Variants = []Variant{}
rows, err := s.pool.Query(ctx,
`SELECT variant, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2
ORDER BY variant`,
a.ID, a.PipelineVersion)
if err != nil {
return nil, classifyDBError(err)
}
defer rows.Close()
for rows.Next() {
var v Variant
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes,
&v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
return nil, classifyDBError(err)
}
a.Variants = append(a.Variants, v)
}
if err := rows.Err(); err != nil {
return nil, classifyDBError(err)
}
return &a, nil
}
@@ -164,8 +154,10 @@ func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Du
// 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет.
prow, err := s.pool.Query(ctx,
`SELECT id, object_key FROM attachments
WHERE status='pending' AND created_at < now() - $1::interval LIMIT 100`,
`SELECT a.id, a.staging_key FROM attachments a
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id
WHERE a.status='queued' AND j.job_id IS NULL
AND a.created_at < now() - $1::interval LIMIT 100`,
grace.String())
if err != nil {
return classifyDBError(err)
@@ -187,3 +179,418 @@ func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Du
}
return nil
}
// EnqueueAttachment: атомарно фиксируем метаданные приёма и кладём outbox-строку.
// 202 отвечаем только после успешного COMMIT этой транзакции.
func (s *Store) EnqueueAttachment(
ctx context.Context, attachmentID, contentType string,
size int64, sha string, w, h int, pipelineVersion int,
) (*Attachment, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var a Attachment
err = tx.QueryRow(ctx, `
UPDATE attachments
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
pipeline_version=$7, status='queued', updated_at=now()
WHERE id=$1 AND status='queued'
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at`,
attachmentID, contentType, size, sha, w, h, pipelineVersion,
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes,
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_jobs
(job_id, attachment_id, pipeline_version, status, next_attempt_at)
VALUES ($1,$2,$3,'queued', now())`,
uuid.NewString(), attachmentID, pipelineVersion,
); err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
a.Variants = []Variant{}
return &a, nil
}
// ClaimJob — берём одну доступную задачу под lease.
// Работает и для queued, и для протухшего processing (reaper тем же запросом).
func (s *Store) ClaimJob(ctx context.Context, workerID string, lease time.Duration) (*Job, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var j Job
err = tx.QueryRow(ctx, `
WITH picked AS (
SELECT job_id FROM attachment_jobs
WHERE status IN ('queued','processing')
AND next_attempt_at <= now()
AND (status='queued' OR lease_expires_at < now())
ORDER BY next_attempt_at
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE attachment_jobs aj
SET status='processing',
lease_owner=$1,
lease_expires_at = now() + $2::interval,
updated_at = now()
FROM picked
WHERE aj.job_id = picked.job_id
RETURNING aj.job_id, aj.attachment_id, aj.pipeline_version,
aj.attempt, aj.status, aj.next_attempt_at, aj.last_error_code`,
workerID, lease.String(),
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.Status,
&j.NextAttemptAt, &j.LastError)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
return &j, nil
}
// MarkReady: атомарно пишем все варианты (идемпотентно по ключу) и завершаем job.
// Конкурентная попытка не перезапишет существующий вариант — ON CONFLICT DO NOTHING.
func (s *Store) MarkReady(
ctx context.Context, jobID, attachmentID string, pipelineVersion int,
finalOriginalKey string, variants []Variant,
) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
// Если job уже done — no-op (redelivery после ack).
var status string
err = tx.QueryRow(ctx,
`SELECT status FROM attachment_jobs WHERE job_id=$1 FOR UPDATE`, jobID,
).Scan(&status)
if errors.Is(err, pgx.ErrNoRows) {
return ErrNotFound
}
if err != nil {
return classifyDBError(err)
}
if status == "done" {
return tx.Commit(ctx)
}
if status == "failed" {
return ErrConflict // нельзя откатывать failed→ready; см. replay
}
// Записать оригинал в final prefix.
if _, err := tx.Exec(ctx,
`UPDATE attachments
SET object_key=$2, status='ready', ready_at=now(), updated_at=now()
WHERE id=$1 AND pipeline_version=$3`,
attachmentID, finalOriginalKey, pipelineVersion,
); err != nil {
return classifyDBError(err)
}
for _, v := range variants {
if _, err := tx.Exec(ctx, `
INSERT INTO attachment_variants
(attachment_id, pipeline_version, variant, object_key,
content_type, size_bytes, width, height, sha256)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`,
attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name,
v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256,
); err != nil {
return classifyDBError(err)
}
}
if _, err := tx.Exec(ctx,
`UPDATE attachment_jobs
SET status='done', lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID,
); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
// RetryJob: временная ошибка — вернуть в очередь с backoff+jitter.
// exhausted=true — если попытки кончились.
func (s *Store) RetryJob(
ctx context.Context, jobID string, errorCode string,
maxAttempts int, baseBackoff time.Duration,
) (exhausted bool, err error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return false, classifyDBError(err)
}
defer tx.Rollback(ctx)
var attempt int
var attachmentID string
err = tx.QueryRow(ctx,
`SELECT attempt, attachment_id FROM attachment_jobs
WHERE job_id=$1 FOR UPDATE`, jobID,
).Scan(&attempt, &attachmentID)
if err != nil {
return false, classifyDBError(err)
}
if attempt >= maxAttempts {
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status='failed', last_error_code=$2,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID, errorCode); err != nil {
return false, classifyDBError(err)
}
if _, err := tx.Exec(ctx, `
UPDATE attachments SET status='failed', failed_at=now(),
error_code=$2, updated_at=now()
WHERE id=$1`, attachmentID, errorCode); err != nil {
return false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
return true, nil
}
// backoff: base * 2^(attempt-1) + jitter
backoff := baseBackoff * time.Duration(1<<(attempt-1))
jitter := time.Duration(rand.Int63n(int64(backoff / 4)))
next := time.Now().Add(backoff + jitter)
if _, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET attempt = attempt + 1,
status = 'queued',
last_error_code=$2,
next_attempt_at=$3,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1`, jobID, errorCode, next); err != nil {
return false, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return false, classifyDBError(err)
}
return false, nil
}
// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox.
func (s *Store) RequeueStuck(ctx context.Context) (int64, error) {
tag, err := s.pool.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
next_attempt_at = now(), updated_at = now()
WHERE status='processing' AND lease_expires_at < now()`)
if err != nil {
return 0, classifyDBError(err)
}
return tag.RowsAffected(), nil
}
// ReplayFailed — операторский replay (уровень 4, CLI/endpoint).
func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
tag, err := tx.Exec(ctx, `
UPDATE attachment_jobs
SET status='queued', attempt=1, next_attempt_at=now(),
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
if err != nil {
return classifyDBError(err)
}
if tag.RowsAffected() == 0 {
return ErrConflict
}
if _, err := tx.Exec(ctx,
`UPDATE attachments SET status='queued', error_code=NULL, failed_at=NULL,
updated_at=now()
WHERE id=$1 AND status='failed'`, attachmentID); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
// ClaimJobByID — берёт конкретную job под lease. Нужен для redelivery:
// если worker упал после publish, повторно доставленное сообщение должно
// либо взять ту же работу (lease протух), либо быть no-op (status=done).
func (s *Store) ClaimJobByID(ctx context.Context, jobID, workerID string, lease time.Duration) (*Job, error) {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
if err != nil {
return nil, classifyDBError(err)
}
defer tx.Rollback(ctx)
var j Job
err = tx.QueryRow(ctx, `
UPDATE attachment_jobs
SET status='processing',
lease_owner=$2,
lease_expires_at = now() + $3::interval,
updated_at = now()
WHERE job_id=$1
AND status IN ('queued','processing')
AND (status='queued' OR lease_expires_at < now())
RETURNING job_id, attachment_id, pipeline_version, attempt,
status, next_attempt_at, last_error_code`,
jobID, workerID, lease.String(),
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt,
&j.Status, &j.NextAttemptAt, &j.LastError)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, classifyDBError(err)
}
if err := tx.Commit(ctx); err != nil {
return nil, classifyDBError(err)
}
return &j, nil
}
// GetStagingKey — доверенные метаданные для worker’а (не из сообщения брокера).
func (s *Store) GetStagingKey(ctx context.Context, attachmentID string) (string, string, error) {
var key, ct string
err := s.pool.QueryRow(ctx,
`SELECT staging_key, content_type FROM attachments WHERE id=$1`, attachmentID,
).Scan(&key, &ct)
if errors.Is(err, pgx.ErrNoRows) {
return "", "", ErrNotFound
}
if err != nil {
return "", "", classifyDBError(err)
}
return key, ct, nil
}
// VariantsExist — no-op-проверка на повторную доставку уже готовой работы.
func (s *Store) VariantsExist(ctx context.Context, attachmentID string, pv int) (bool, error) {
var n int
err := s.pool.QueryRow(ctx,
`SELECT count(*) FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2`,
attachmentID, pv,
).Scan(&n)
if err != nil {
return false, classifyDBError(err)
}
return n >= len(RequiredVariants), nil
}
// MarkFailedPermanent — терминальный failed без retry (постоянная ошибка).
func (s *Store) MarkFailedPermanent(ctx context.Context, jobID, attachmentID, code string) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return classifyDBError(err)
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx,
`UPDATE attachment_jobs SET status='failed', last_error_code=$2,
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
WHERE job_id=$1 AND status='processing'`, jobID, code); err != nil {
return classifyDBError(err)
}
if _, err := tx.Exec(ctx,
`UPDATE attachments SET status='failed', error_code=$2,
failed_at=now(), updated_at=now()
WHERE id=$1`, attachmentID, code); err != nil {
return classifyDBError(err)
}
return tx.Commit(ctx)
}
type UnpublishedJob struct {
JobID string
AttachmentID string
PipelineVersion int
Attempt int
}
// SelectUnpublishedJobs — выборка outbox’ом. Транзакция не нужна:
// published_at ставим отдельным UPDATE после XADD; повторная публикация
// допустима (worker идемпотентен).
func (s *Store) SelectUnpublishedJobs(ctx context.Context, limit int) ([]UnpublishedJob, error) {
rows, err := s.pool.Query(ctx, `
SELECT job_id, attachment_id, pipeline_version, attempt
FROM attachment_jobs
WHERE published_at IS NULL AND status='queued'
AND next_attempt_at <= now()
ORDER BY next_attempt_at
LIMIT $1`, limit)
if err != nil {
return nil, classifyDBError(err)
}
defer rows.Close()
var out []UnpublishedJob
for rows.Next() {
var u UnpublishedJob
if err := rows.Scan(&u.JobID, &u.AttachmentID, &u.PipelineVersion, &u.Attempt); err != nil {
return nil, classifyDBError(err)
}
out = append(out, u)
}
return out, rows.Err()
}
// MarkPublished — фиксируем факт XADD (publisher confirm).
func (s *Store) MarkPublished(ctx context.Context, jobID string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachment_jobs SET published_at=now(),
publish_attempts=publish_attempts+1, updated_at=now()
WHERE job_id=$1`, jobID)
return classifyDBError(err)
}
// BumpPublishAttempt — XADD упал; фиксируем причину, повторим в следующем цикле.
func (s *Store) BumpPublishAttempt(ctx context.Context, jobID, code string) error {
_, err := s.pool.Exec(ctx,
`UPDATE attachment_jobs SET publish_attempts=publish_attempts+1,
last_error_code=$2, updated_at=now()
WHERE job_id=$1`, jobID, code)
return classifyDBError(err)
}
func (s *Store) GetVariantKey(ctx context.Context, attachmentID string, pv int, variant string) (string, string, error) {
var key, ct string
err := s.pool.QueryRow(ctx,
`SELECT object_key, content_type FROM attachment_variants
WHERE attachment_id=$1 AND pipeline_version=$2 AND variant=$3`,
attachmentID, pv, variant).Scan(&key, &ct)
if errors.Is(err, pgx.ErrNoRows) {
return "", "", ErrNotFound
}
if err != nil {
return "", "", classifyDBError(err)
}
return key, ct, nil
}
+80
View File
@@ -0,0 +1,80 @@
package domain
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
const (
StreamName = "attachment.jobs"
ConsumerGroup = "workers"
)
type JobMessage struct {
SchemaVersion int `json:"schema_version"`
JobID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt string `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
}
type Broker struct{ rdb *redis.Client }
func NewBroker(ctx context.Context, url string) (*Broker, error) {
opt, err := redis.ParseURL(url)
if err != nil {
return nil, err
}
rdb := redis.NewClient(opt)
if err := rdb.Ping(ctx).Err(); err != nil {
return nil, err
}
// Группа создаётся идемпотентно.
err = rdb.XGroupCreateMkStream(ctx, StreamName, ConsumerGroup, "0").Err()
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
return nil, err
}
return &Broker{rdb: rdb}, nil
}
func (b *Broker) Publish(ctx context.Context, m JobMessage) (string, error) {
buf, err := json.Marshal(m)
if err != nil {
return "", err
}
return b.rdb.XAdd(ctx, &redis.XAddArgs{
Stream: StreamName,
Values: map[string]any{"payload": buf},
}).Result()
}
// Consume — блокирующее чтение из consumer group; ack делает вызывающий.
func (b *Broker) Consume(ctx context.Context, consumer string) ([]redis.XMessage, error) {
res, err := b.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: ConsumerGroup, Consumer: consumer,
Streams: []string{StreamName, ">"},
Count: 1, Block: 5 * time.Second,
}).Result()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err != nil {
return nil, err
}
if len(res) == 0 {
return nil, nil
}
return res[0].Messages, nil
}
func (b *Broker) Ack(ctx context.Context, id string) error {
return b.rdb.XAck(ctx, StreamName, ConsumerGroup, id).Err()
}
+13
View File
@@ -0,0 +1,13 @@
package domain
import "time"
type Job struct {
JobID string
AttachmentID string
PipelineVersion int
Attempt int
Status string
NextAttemptAt time.Time
LastError *string
}
+14 -12
View File
@@ -62,21 +62,23 @@ type Variant struct {
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
ObjectKey string `json:"-"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
PipelineVersion int `json:"-"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
var (
+45
View File
@@ -158,6 +158,51 @@ func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body
}, nil
}
const (
StagingPrefix = "staging/"
FinalPrefix = "final/"
)
// MoveToFinal: копирование staging→final и удаление staging.
// Используем CopyObject, чтобы не тянуть байты в память. Если MinIO не умеет
// server-side copy — используйте compose? MinIO умеет S3 CopyObject.
func (s *S3Store) MoveToFinal(ctx context.Context, stagingKey, finalKey string) error {
src := minio.CopySrcOptions{Bucket: s.bucket, Object: stagingKey}
dst := minio.CopyDestOptions{Bucket: s.bucket, Object: finalKey}
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
return err
}
return s.DeleteObject(ctx, stagingKey)
}
func (s *S3Store) PutBytes(ctx context.Context, key, contentType string, data []byte) (string, error) {
h := sha256.Sum256(data)
_, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType})
if err != nil {
return "", err
}
return hex.EncodeToString(h[:]), nil
}
// FetchStagingForProcessing — читает staging в память с жёстким лимитом.
// maxBytes гарантирует, что worker не съест RAM на «злом» файле.
func (s *S3Store) FetchStagingForProcessing(ctx context.Context, key string) ([]byte, string, error) {
rc, ct, size, err := s.GetObject(ctx, key)
if err != nil {
return nil, "", err
}
defer rc.Close()
if size > s.maxBytes {
return nil, "", ErrTooLarge
}
buf, err := io.ReadAll(rc)
if err != nil {
return nil, "", err
}
return buf, ct, nil
}
func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) {
// Общий дедлайн на все попытки — 6 секунд.
overallCtx, overallCancel := context.WithTimeout(ctx, 6*time.Second)