forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
251 lines
7.5 KiB
Go
251 lines
7.5 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"time"
|
|
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// CreateQueuedAttachment is the durability boundary of the upload endpoint.
|
|
// The attachment snapshot and the outbox job commit together, so there is no
|
|
// observable state in which the API promised work but PostgreSQL lost the
|
|
// intent. Publication to RabbitMQ happens later and may be retried safely.
|
|
func (s *Store) CreateQueuedAttachment(ctx context.Context, att model.Attachment, job model.JobRecord) error {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
hash, err := hex.DecodeString(att.SHA256)
|
|
if err != nil {
|
|
return common.BadRequest("invalid sha256")
|
|
}
|
|
if att.PipelineVersion < 1 {
|
|
att.PipelineVersion = 1
|
|
}
|
|
variants, err := json.Marshal([]model.Variant{})
|
|
if err != nil {
|
|
return common.Internal()
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return dbError(err)
|
|
}
|
|
defer rollback(tx)
|
|
|
|
_, err = tx.Exec(ctx, `
|
|
INSERT INTO attachments
|
|
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
|
object_key, staging_key, pipeline_version, variants, status, error_code,
|
|
created_at, updated_at)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued', NULL, $12, $12)`,
|
|
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes,
|
|
hash, att.ObjectKey, att.StagingKey, att.PipelineVersion, variants, time.Time(att.CreatedAt))
|
|
if err != nil {
|
|
return dbError(err)
|
|
}
|
|
|
|
_, err = tx.Exec(ctx, `
|
|
INSERT INTO attachment_jobs
|
|
(job_id, attachment_id, type, pipeline_version, attempt, max_attempts,
|
|
status, error_code, available_at, published_at, lease_until,
|
|
processing_started_at, finished_at, replay_count, created_at, updated_at)
|
|
VALUES ($1, $2, $3, $4, 0, $5, 'queued', NULL, $6, NULL, NULL,
|
|
NULL, NULL, 0, $6, $6)`,
|
|
job.JobID, att.ID, model.JobTypeAttachmentProcessV1, att.PipelineVersion,
|
|
job.MaxAttempts, time.Time(att.CreatedAt))
|
|
if err != nil {
|
|
return dbError(err)
|
|
}
|
|
|
|
if err = tx.Commit(ctx); err != nil {
|
|
return dbError(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CountPendingJobs is an admission control signal, not a queue size proof.
|
|
// Two API replicas may observe slightly stale counts, but the durable cap is
|
|
// still useful: it stops an overload from growing staging and PostgreSQL
|
|
// without limit. Rejections are 503 because the client may retry later.
|
|
func (s *Store) CountPendingJobs(ctx context.Context) (int, error) {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
var count int
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT count(*) FROM attachment_jobs
|
|
WHERE status IN ('queued', 'processing')`).Scan(&count)
|
|
if err != nil {
|
|
return 0, dbError(err)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+`
|
|
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2
|
|
WHERE a.id = $1`, attachmentID, userID))
|
|
if err != nil {
|
|
return model.Attachment{}, err
|
|
}
|
|
return att, nil
|
|
}
|
|
|
|
// GetAttachment is the worker's trusted read. The worker never accepts bucket
|
|
// keys or filenames from the broker message.
|
|
func (s *Store) GetAttachment(ctx context.Context, attachmentID uuid.UUID) (model.Attachment, error) {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
att, err := scanAttachment(s.pool.QueryRow(ctx, attachmentSelect+` WHERE a.id = $1`, attachmentID))
|
|
if err != nil {
|
|
return model.Attachment{}, err
|
|
}
|
|
return att, nil
|
|
}
|
|
|
|
const attachmentSelect = `
|
|
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
|
a.size_bytes, a.sha256, a.status, a.error_code,
|
|
a.created_at, a.updated_at, a.object_key, COALESCE(a.staging_key, ''),
|
|
a.pipeline_version, a.variants
|
|
FROM attachments a`
|
|
|
|
func scanAttachment(row rowScanner) (model.Attachment, error) {
|
|
var (
|
|
att model.Attachment
|
|
hash []byte
|
|
errCode *string
|
|
createdAt time.Time
|
|
updatedAt time.Time
|
|
variants []byte
|
|
)
|
|
|
|
if err := row.Scan(
|
|
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
|
|
&att.SizeBytes, &hash, &att.Status, &errCode,
|
|
&createdAt, &updatedAt, &att.ObjectKey, &att.StagingKey,
|
|
&att.PipelineVersion, &variants,
|
|
); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return model.Attachment{}, common.NotFound("attachment not found")
|
|
}
|
|
return model.Attachment{}, dbError(err)
|
|
}
|
|
|
|
att.SHA256 = hex.EncodeToString(hash)
|
|
att.ErrorCode = errCode
|
|
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
|
|
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
|
|
att.Variants = []model.Variant{}
|
|
if len(variants) > 0 {
|
|
if err := json.Unmarshal(variants, &att.Variants); err != nil {
|
|
return model.Attachment{}, dbError(err)
|
|
}
|
|
}
|
|
if att.Variants == nil {
|
|
att.Variants = []model.Variant{}
|
|
}
|
|
return att, nil
|
|
}
|
|
|
|
// ListStaleStagingAttachments returns terminal attachments whose staging copy
|
|
// may be removed. The grace period is applied to updated_at so a just-finished
|
|
// job is never cleaned while its commit is still settling.
|
|
func (s *Store) ListStaleStagingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
rows, err := s.pool.Query(ctx, attachmentSelect+`
|
|
WHERE a.status IN ('ready', 'failed')
|
|
AND a.updated_at < $1
|
|
AND a.staging_key IS NOT NULL
|
|
ORDER BY a.updated_at
|
|
LIMIT $2`, cutoff, limit)
|
|
if err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
var result []model.Attachment
|
|
for rows.Next() {
|
|
att, scanErr := scanAttachment(rows)
|
|
if scanErr != nil {
|
|
return nil, scanErr
|
|
}
|
|
result = append(result, att)
|
|
}
|
|
if err = rows.Err(); err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Store) ClearStagingKey(ctx context.Context, attachmentID uuid.UUID) error {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
_, err := s.pool.Exec(ctx,
|
|
`UPDATE attachments SET staging_key = NULL WHERE id = $1`, attachmentID)
|
|
return dbError(err)
|
|
}
|
|
|
|
// ListStorageKeys returns every object the database still owns. It includes
|
|
// variant keys derived from the original key; the reconciler must never delete
|
|
// a thumbnail that has no separate row.
|
|
func (s *Store) ListStorageKeys(ctx context.Context) (map[string]struct{}, error) {
|
|
ctx, cancel := s.withTimeout(ctx)
|
|
defer cancel()
|
|
|
|
rows, err := s.pool.Query(ctx,
|
|
`SELECT object_key, COALESCE(staging_key, ''), variants FROM attachments`)
|
|
if err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
keys := make(map[string]struct{})
|
|
for rows.Next() {
|
|
var (
|
|
objectKey string
|
|
staging string
|
|
variants []byte
|
|
)
|
|
if err = rows.Scan(&objectKey, &staging, &variants); err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
keys[objectKey] = struct{}{}
|
|
if staging != "" {
|
|
keys[staging] = struct{}{}
|
|
}
|
|
var list []model.Variant
|
|
if err = json.Unmarshal(variants, &list); err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
for _, variant := range list {
|
|
keys[VariantObjectKey(objectKey, variant.Name)] = struct{}{}
|
|
}
|
|
}
|
|
if err = rows.Err(); err != nil {
|
|
return nil, dbError(err)
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
// VariantObjectKey is the one place that names variant objects. Keeping it
|
|
// deterministic makes a retried job overwrite the same keys instead of
|
|
// creating a second variant set.
|
|
func VariantObjectKey(originalKey, variant string) string {
|
|
return originalKey + "/" + variant
|
|
}
|