diff --git a/db/migrations/00004_attachment_pipeline.sql b/db/migrations/00004_attachment_pipeline.sql new file mode 100644 index 00000000..6ed6ec50 --- /dev/null +++ b/db/migrations/00004_attachment_pipeline.sql @@ -0,0 +1,54 @@ +-- +goose Up +CREATE TYPE attachment_variant_name AS ENUM ('thumbnail', 'crop', 'optimized'); + +ALTER TABLE attachments + ADD COLUMN staging_key text UNIQUE, + ADD COLUMN job_id uuid UNIQUE, + ADD COLUMN pipeline_version int NOT NULL DEFAULT 1 CHECK (pipeline_version > 0), + ADD COLUMN attempts int NOT NULL DEFAULT 0 CHECK (attempts >= 0), + ADD COLUMN width int CHECK (width > 0), + ADD COLUMN height int CHECK (height > 0), + ADD COLUMN lease_until timestamptz; + +CREATE INDEX idx_attachments_expired_lease ON attachments (lease_until) WHERE status = 'processing'; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version int NOT NULL CHECK (pipeline_version > 0), + variant attachment_variant_name NOT NULL, + s3_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + size_bytes int NOT NULL CHECK (size_bytes > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width int NOT NULL CHECK (width > 0), + height int NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id, pipeline_version, variant) +); + +CREATE TABLE outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachments (job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version int NOT NULL CHECK (pipeline_version > 0), + attempt int NOT NULL CHECK (attempt > 0), + created_at timestamptz NOT NULL DEFAULT now(), + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + UNIQUE (job_id, attempt) +); + +CREATE INDEX idx_outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; + +-- +goose Down +DROP TABLE outbox; +DROP TABLE attachment_variants; +DROP TYPE attachment_variant_name; +DROP INDEX idx_attachments_expired_lease; +ALTER TABLE attachments + DROP COLUMN height, + DROP COLUMN width, + DROP COLUMN lease_until, + DROP COLUMN attempts, + DROP COLUMN pipeline_version, + DROP COLUMN job_id, + DROP COLUMN staging_key; diff --git a/db/queries/attachment_variants.sql b/db/queries/attachment_variants.sql new file mode 100644 index 00000000..6b38542f --- /dev/null +++ b/db/queries/attachment_variants.sql @@ -0,0 +1,28 @@ +-- name: CreateAttachmentVariant :exec +INSERT INTO attachment_variants ( + attachment_id, + pipeline_version, + variant, + s3_key, + content_type, + size_bytes, + width, + height, + sha256 +) VALUES ( + sqlc.arg(attachment_id), + sqlc.arg(pipeline_version), + sqlc.arg(variant), + sqlc.arg(s3_key), + sqlc.arg(content_type), + sqlc.arg(size_bytes), + sqlc.arg(width), + sqlc.arg(height), + sqlc.arg(sha256) +); + +-- name: ListAttachmentVariants :many +SELECT * FROM attachment_variants +WHERE attachment_id = sqlc.arg(attachment_id)::uuid + AND pipeline_version = sqlc.arg(pipeline_version)::int +ORDER BY variant; diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql index 9f6ce774..9eecedbf 100644 --- a/db/queries/attachments.sql +++ b/db/queries/attachments.sql @@ -4,10 +4,14 @@ INSERT INTO attachments ( chat_id, owner_id, s3_key, + staging_key, + job_id, filename, content_type, size_bytes, sha256, + width, + height, error_code ) SELECT @@ -15,10 +19,14 @@ SELECT sqlc.arg(chat_id)::uuid, sqlc.arg(owner_id)::uuid, sqlc.arg(s3_key)::text, + sqlc.narg(staging_key)::text, + sqlc.narg(job_id)::uuid, sqlc.arg(filename)::text, sqlc.arg(content_type)::attachment_content_type, sqlc.arg(size_bytes)::int, sqlc.arg(sha256)::text, + sqlc.arg(width)::int, + sqlc.arg(height)::int, NULLIF(sqlc.arg(error_code)::text, '') WHERE EXISTS ( SELECT 1 FROM chat_members @@ -77,6 +85,30 @@ SELECT ) AS referenced FROM attachments a; +-- name: LockAttachmentJob :one +SELECT * FROM attachments +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND job_id = sqlc.arg(job_id)::uuid + AND pipeline_version = sqlc.arg(pipeline_version)::int + AND attempts = sqlc.arg(attempt)::int + AND lease_until > now() +FOR UPDATE; + +-- name: LockExpiredAttachmentJobs :many +SELECT * FROM attachments +WHERE status = 'processing' + AND lease_until <= now() +ORDER BY lease_until, id +LIMIT sqlc.arg(batch_size)::int +FOR UPDATE SKIP LOCKED; + +-- name: MarkAttachmentReady :one +UPDATE attachments +SET status = 'ready', updated_at = now() +WHERE id = sqlc.arg(id)::uuid +RETURNING *; + -- name: MarkAttachmentFailedIfReady :execrows UPDATE attachments SET status = 'failed', @@ -84,12 +116,73 @@ SET status = 'failed', updated_at = now() WHERE id = sqlc.arg(id)::uuid AND status = 'ready'; --- name: MarkAttachmentReady :one +-- name: ClaimAttachmentJob :one UPDATE attachments -SET status = 'ready', updated_at = now() -WHERE id = $1::uuid +SET status = 'processing', + attempts = attempts + 1, + lease_until = sqlc.arg(lease_until)::timestamptz, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND job_id = sqlc.arg(job_id)::uuid + AND pipeline_version = sqlc.arg(pipeline_version)::int + AND status = 'queued' + AND attempts + 1 = sqlc.arg(attempt)::int RETURNING *; +-- name: CompleteAttachmentJob :execrows +UPDATE attachments +SET status = 'ready', + error_code = NULL, + lease_until = NULL, + width = sqlc.arg(width)::int, + height = sqlc.arg(height)::int, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND attempts = sqlc.arg(attempt)::int + AND lease_until > now(); + +-- name: RetryAttachmentJob :execrows +UPDATE attachments +SET status = 'queued', + error_code = sqlc.arg(error_code)::text, + lease_until = NULL, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND attempts = sqlc.arg(attempt)::int + AND lease_until > now(); + +-- name: FailAttachmentJob :execrows +UPDATE attachments +SET status = 'failed', + error_code = sqlc.arg(error_code)::text, + lease_until = NULL, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND attempts = sqlc.arg(attempt)::int + AND lease_until > now(); + +-- name: RecoverExpiredAttachmentJob :execrows +UPDATE attachments +SET status = 'queued', + lease_until = NULL, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND lease_until <= now(); + +-- name: FailExpiredAttachmentJob :execrows +UPDATE attachments +SET status = 'failed', + error_code = 'retry_exhausted', + lease_until = NULL, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND status = 'processing' + AND lease_until <= now(); + -- name: DeleteQueuedAttachment :execrows DELETE FROM attachments WHERE id = sqlc.arg(id)::uuid AND status = 'queued'; diff --git a/db/queries/outbox.sql b/db/queries/outbox.sql new file mode 100644 index 00000000..0aecc5bd --- /dev/null +++ b/db/queries/outbox.sql @@ -0,0 +1,32 @@ +-- name: CreateOutboxEntry :one +INSERT INTO outbox ( + job_id, + attachment_id, + pipeline_version, + attempt, + available_at +) +SELECT + job_id, + id, + pipeline_version, + sqlc.arg(attempt)::int, + sqlc.arg(available_at)::timestamptz +FROM attachments +WHERE id = sqlc.arg(attachment_id)::uuid + AND job_id IS NOT NULL +RETURNING *; + +-- name: LockPendingOutboxEntries :many +SELECT * FROM outbox +WHERE published_at IS NULL + AND available_at <= now() +ORDER BY available_at, id +LIMIT sqlc.arg(batch_size)::int +FOR UPDATE SKIP LOCKED; + +-- name: MarkOutboxPublished :execrows +UPDATE outbox +SET published_at = now() +WHERE id = sqlc.arg(id)::bigint + AND published_at IS NULL; diff --git a/internal/db/attachment_variants.sql.go b/internal/db/attachment_variants.sql.go new file mode 100644 index 00000000..f6270f00 --- /dev/null +++ b/internal/db/attachment_variants.sql.go @@ -0,0 +1,105 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachment_variants.sql + +package db + +import ( + "context" + + "github.com/google/uuid" +) + +const createAttachmentVariant = `-- name: CreateAttachmentVariant :exec +INSERT INTO attachment_variants ( + attachment_id, + pipeline_version, + variant, + s3_key, + content_type, + size_bytes, + width, + height, + sha256 +) VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9 +) +` + +type CreateAttachmentVariantParams struct { + AttachmentID uuid.UUID + PipelineVersion int + Variant AttachmentVariantName + S3Key string + ContentType string + SizeBytes int + Width int + Height int + Sha256 string +} + +func (q *Queries) CreateAttachmentVariant(ctx context.Context, arg CreateAttachmentVariantParams) error { + _, err := q.db.Exec(ctx, createAttachmentVariant, + arg.AttachmentID, + arg.PipelineVersion, + arg.Variant, + arg.S3Key, + arg.ContentType, + arg.SizeBytes, + arg.Width, + arg.Height, + arg.Sha256, + ) + return err +} + +const listAttachmentVariants = `-- name: ListAttachmentVariants :many +SELECT attachment_id, pipeline_version, variant, s3_key, content_type, size_bytes, sha256, width, height FROM attachment_variants +WHERE attachment_id = $1::uuid + AND pipeline_version = $2::int +ORDER BY variant +` + +type ListAttachmentVariantsParams struct { + AttachmentID uuid.UUID + PipelineVersion int +} + +func (q *Queries) ListAttachmentVariants(ctx context.Context, arg ListAttachmentVariantsParams) ([]AttachmentVariant, error) { + rows, err := q.db.Query(ctx, listAttachmentVariants, arg.AttachmentID, arg.PipelineVersion) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AttachmentVariant{} + for rows.Next() { + var i AttachmentVariant + if err := rows.Scan( + &i.AttachmentID, + &i.PipelineVersion, + &i.Variant, + &i.S3Key, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Width, + &i.Height, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/db/attachments.sql.go b/internal/db/attachments.sql.go index 8103dd3a..00a21832 100644 --- a/internal/db/attachments.sql.go +++ b/internal/db/attachments.sql.go @@ -12,16 +12,109 @@ import ( "github.com/google/uuid" ) +const claimAttachmentJob = `-- name: ClaimAttachmentJob :one +UPDATE attachments +SET status = 'processing', + attempts = attempts + 1, + lease_until = $1::timestamptz, + updated_at = now() +WHERE id = $2::uuid + AND job_id = $3::uuid + AND pipeline_version = $4::int + AND status = 'queued' + AND attempts + 1 = $5::int +RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until +` + +type ClaimAttachmentJobParams struct { + LeaseUntil time.Time + ID uuid.UUID + JobID uuid.UUID + PipelineVersion int + Attempt int +} + +func (q *Queries) ClaimAttachmentJob(ctx context.Context, arg ClaimAttachmentJobParams) (Attachment, error) { + row := q.db.QueryRow(ctx, claimAttachmentJob, + arg.LeaseUntil, + arg.ID, + arg.JobID, + arg.PipelineVersion, + arg.Attempt, + ) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, + ) + return i, err +} + +const completeAttachmentJob = `-- name: CompleteAttachmentJob :execrows +UPDATE attachments +SET status = 'ready', + error_code = NULL, + lease_until = NULL, + width = $1::int, + height = $2::int, + updated_at = now() +WHERE id = $3::uuid + AND status = 'processing' + AND attempts = $4::int + AND lease_until > now() +` + +type CompleteAttachmentJobParams struct { + Width int + Height int + ID uuid.UUID + Attempt int +} + +func (q *Queries) CompleteAttachmentJob(ctx context.Context, arg CompleteAttachmentJobParams) (int64, error) { + result, err := q.db.Exec(ctx, completeAttachmentJob, + arg.Width, + arg.Height, + arg.ID, + arg.Attempt, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const createAttachment = `-- name: CreateAttachment :one INSERT INTO attachments ( id, chat_id, owner_id, s3_key, + staging_key, + job_id, filename, content_type, size_bytes, sha256, + width, + height, error_code ) SELECT @@ -30,16 +123,20 @@ SELECT $3::uuid, $4::text, $5::text, - $6::attachment_content_type, - $7::int, - $8::text, - NULLIF($9::text, '') + $6::uuid, + $7::text, + $8::attachment_content_type, + $9::int, + $10::text, + $11::int, + $12::int, + NULLIF($13::text, '') WHERE EXISTS ( SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 ) -RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at +RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until ` type CreateAttachmentParams struct { @@ -47,10 +144,14 @@ type CreateAttachmentParams struct { ChatID uuid.UUID OwnerID uuid.UUID S3Key string + StagingKey *string + JobID *uuid.UUID Filename string ContentType AttachmentContentType SizeBytes int Sha256 string + Width int + Height int ErrorCode string } @@ -60,10 +161,14 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara arg.ChatID, arg.OwnerID, arg.S3Key, + arg.StagingKey, + arg.JobID, arg.Filename, arg.ContentType, arg.SizeBytes, arg.Sha256, + arg.Width, + arg.Height, arg.ErrorCode, ) var i Attachment @@ -80,6 +185,13 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara &i.ErrorCode, &i.CreatedAt, &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, ) return i, err } @@ -120,8 +232,53 @@ func (q *Queries) DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int return result.RowsAffected(), nil } +const failAttachmentJob = `-- name: FailAttachmentJob :execrows +UPDATE attachments +SET status = 'failed', + error_code = $1::text, + lease_until = NULL, + updated_at = now() +WHERE id = $2::uuid + AND status = 'processing' + AND attempts = $3::int + AND lease_until > now() +` + +type FailAttachmentJobParams struct { + ErrorCode string + ID uuid.UUID + Attempt int +} + +func (q *Queries) FailAttachmentJob(ctx context.Context, arg FailAttachmentJobParams) (int64, error) { + result, err := q.db.Exec(ctx, failAttachmentJob, arg.ErrorCode, arg.ID, arg.Attempt) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const failExpiredAttachmentJob = `-- name: FailExpiredAttachmentJob :execrows +UPDATE attachments +SET status = 'failed', + error_code = 'retry_exhausted', + lease_until = NULL, + updated_at = now() +WHERE id = $1::uuid + AND status = 'processing' + AND lease_until <= now() +` + +func (q *Queries) FailExpiredAttachmentJob(ctx context.Context, id uuid.UUID) (int64, error) { + result, err := q.db.Exec(ctx, failExpiredAttachmentJob, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const getAttachmentByID = `-- name: GetAttachmentByID :one -SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at FROM attachments +SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments WHERE id = $1::uuid ` @@ -141,6 +298,13 @@ func (q *Queries) GetAttachmentByID(ctx context.Context, dollar_1 uuid.UUID) (At &i.ErrorCode, &i.CreatedAt, &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, ) return i, err } @@ -316,6 +480,105 @@ func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Ti return items, nil } +const lockAttachmentJob = `-- name: LockAttachmentJob :one +SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments +WHERE id = $1::uuid + AND status = 'processing' + AND job_id = $2::uuid + AND pipeline_version = $3::int + AND attempts = $4::int + AND lease_until > now() +FOR UPDATE +` + +type LockAttachmentJobParams struct { + ID uuid.UUID + JobID uuid.UUID + PipelineVersion int + Attempt int +} + +func (q *Queries) LockAttachmentJob(ctx context.Context, arg LockAttachmentJobParams) (Attachment, error) { + row := q.db.QueryRow(ctx, lockAttachmentJob, + arg.ID, + arg.JobID, + arg.PipelineVersion, + arg.Attempt, + ) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, + ) + return i, err +} + +const lockExpiredAttachmentJobs = `-- name: LockExpiredAttachmentJobs :many +SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments +WHERE status = 'processing' + AND lease_until <= now() +ORDER BY lease_until, id +LIMIT $1::int +FOR UPDATE SKIP LOCKED +` + +func (q *Queries) LockExpiredAttachmentJobs(ctx context.Context, batchSize int) ([]Attachment, error) { + rows, err := q.db.Query(ctx, lockExpiredAttachmentJobs, batchSize) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Attachment{} + for rows.Next() { + var i Attachment + if err := rows.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const markAttachmentFailedIfReady = `-- name: MarkAttachmentFailedIfReady :execrows UPDATE attachments SET status = 'failed', @@ -341,11 +604,11 @@ const markAttachmentReady = `-- name: MarkAttachmentReady :one UPDATE attachments SET status = 'ready', updated_at = now() WHERE id = $1::uuid -RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at +RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until ` -func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) (Attachment, error) { - row := q.db.QueryRow(ctx, markAttachmentReady, dollar_1) +func (q *Queries) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, markAttachmentReady, id) var i Attachment err := row.Scan( &i.ID, @@ -360,6 +623,57 @@ func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) ( &i.ErrorCode, &i.CreatedAt, &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.Width, + &i.Height, + &i.LeaseUntil, ) return i, err } + +const recoverExpiredAttachmentJob = `-- name: RecoverExpiredAttachmentJob :execrows +UPDATE attachments +SET status = 'queued', + lease_until = NULL, + updated_at = now() +WHERE id = $1::uuid + AND status = 'processing' + AND lease_until <= now() +` + +func (q *Queries) RecoverExpiredAttachmentJob(ctx context.Context, id uuid.UUID) (int64, error) { + result, err := q.db.Exec(ctx, recoverExpiredAttachmentJob, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const retryAttachmentJob = `-- name: RetryAttachmentJob :execrows +UPDATE attachments +SET status = 'queued', + error_code = $1::text, + lease_until = NULL, + updated_at = now() +WHERE id = $2::uuid + AND status = 'processing' + AND attempts = $3::int + AND lease_until > now() +` + +type RetryAttachmentJobParams struct { + ErrorCode string + ID uuid.UUID + Attempt int +} + +func (q *Queries) RetryAttachmentJob(ctx context.Context, arg RetryAttachmentJobParams) (int64, error) { + result, err := q.db.Exec(ctx, retryAttachmentJob, arg.ErrorCode, arg.ID, arg.Attempt) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/db/models.go b/internal/db/models.go index 5d69de15..98dbdaf3 100644 --- a/internal/db/models.go +++ b/internal/db/models.go @@ -98,19 +98,81 @@ func (ns NullAttachmentStatus) Value() (driver.Value, error) { return string(ns.AttachmentStatus), nil } +type AttachmentVariantName string + +const ( + AttachmentVariantNameThumbnail AttachmentVariantName = "thumbnail" + AttachmentVariantNameCrop AttachmentVariantName = "crop" + AttachmentVariantNameOptimized AttachmentVariantName = "optimized" +) + +func (e *AttachmentVariantName) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = AttachmentVariantName(s) + case string: + *e = AttachmentVariantName(s) + default: + return fmt.Errorf("unsupported scan type for AttachmentVariantName: %T", src) + } + return nil +} + +type NullAttachmentVariantName struct { + AttachmentVariantName AttachmentVariantName + Valid bool // Valid is true if AttachmentVariantName is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullAttachmentVariantName) Scan(value interface{}) error { + if value == nil { + ns.AttachmentVariantName, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.AttachmentVariantName.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullAttachmentVariantName) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.AttachmentVariantName), nil +} + type Attachment struct { - ID uuid.UUID - ChatID uuid.UUID - OwnerID uuid.UUID - S3Key string - Filename string - ContentType AttachmentContentType - SizeBytes int - Sha256 string - Status AttachmentStatus - ErrorCode *string - CreatedAt time.Time - UpdatedAt time.Time + ID uuid.UUID + ChatID uuid.UUID + OwnerID uuid.UUID + S3Key string + Filename string + ContentType AttachmentContentType + SizeBytes int + Sha256 string + Status AttachmentStatus + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time + StagingKey *string + JobID *uuid.UUID + PipelineVersion int + Attempts int + Width *int32 + Height *int32 + LeaseUntil *time.Time +} + +type AttachmentVariant struct { + AttachmentID uuid.UUID + PipelineVersion int + Variant AttachmentVariantName + S3Key string + ContentType string + SizeBytes int + Sha256 string + Width int + Height int } type Chat struct { @@ -148,6 +210,17 @@ type MessageAttachment struct { Position int } +type Outbox struct { + ID int64 + JobID uuid.UUID + AttachmentID uuid.UUID + PipelineVersion int + Attempt int + CreatedAt time.Time + AvailableAt time.Time + PublishedAt *time.Time +} + type Session struct { TokenHash string UserID uuid.UUID diff --git a/internal/db/outbox.sql.go b/internal/db/outbox.sql.go new file mode 100644 index 00000000..f02c4f28 --- /dev/null +++ b/internal/db/outbox.sql.go @@ -0,0 +1,108 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: outbox.sql + +package db + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const createOutboxEntry = `-- name: CreateOutboxEntry :one +INSERT INTO outbox ( + job_id, + attachment_id, + pipeline_version, + attempt, + available_at +) +SELECT + job_id, + id, + pipeline_version, + $1::int, + $2::timestamptz +FROM attachments +WHERE id = $3::uuid + AND job_id IS NOT NULL +RETURNING id, job_id, attachment_id, pipeline_version, attempt, created_at, available_at, published_at +` + +type CreateOutboxEntryParams struct { + Attempt int + AvailableAt time.Time + AttachmentID uuid.UUID +} + +func (q *Queries) CreateOutboxEntry(ctx context.Context, arg CreateOutboxEntryParams) (Outbox, error) { + row := q.db.QueryRow(ctx, createOutboxEntry, arg.Attempt, arg.AvailableAt, arg.AttachmentID) + var i Outbox + err := row.Scan( + &i.ID, + &i.JobID, + &i.AttachmentID, + &i.PipelineVersion, + &i.Attempt, + &i.CreatedAt, + &i.AvailableAt, + &i.PublishedAt, + ) + return i, err +} + +const lockPendingOutboxEntries = `-- name: LockPendingOutboxEntries :many +SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, available_at, published_at FROM outbox +WHERE published_at IS NULL + AND available_at <= now() +ORDER BY available_at, id +LIMIT $1::int +FOR UPDATE SKIP LOCKED +` + +func (q *Queries) LockPendingOutboxEntries(ctx context.Context, batchSize int) ([]Outbox, error) { + rows, err := q.db.Query(ctx, lockPendingOutboxEntries, batchSize) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Outbox{} + for rows.Next() { + var i Outbox + if err := rows.Scan( + &i.ID, + &i.JobID, + &i.AttachmentID, + &i.PipelineVersion, + &i.Attempt, + &i.CreatedAt, + &i.AvailableAt, + &i.PublishedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markOutboxPublished = `-- name: MarkOutboxPublished :execrows +UPDATE outbox +SET published_at = now() +WHERE id = $1::bigint + AND published_at IS NULL +` + +func (q *Queries) MarkOutboxPublished(ctx context.Context, id int64) (int64, error) { + result, err := q.db.Exec(ctx, markOutboxPublished, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/service/attachment.go b/internal/service/attachment.go index 63112c94..e1e0c395 100644 --- a/internal/service/attachment.go +++ b/internal/service/attachment.go @@ -98,7 +98,7 @@ func (s *AttachmentService) Upload( if _, err := tmp.Seek(0, io.SeekStart); err != nil { return model.Attachment{}, err } - detectedType, err := s.checkImage(ctx, tmp, contentType) + detectedType, width, height, err := s.checkImage(ctx, tmp, contentType) if err != nil { return model.Attachment{}, err } @@ -124,6 +124,8 @@ func (s *AttachmentService) Upload( ContentType: db.AttachmentContentType(detectedType), SizeBytes: int(size), Sha256: hash, + Width: width, + Height: height, ErrorCode: "", }) if postgres.IsNotFound(err) { @@ -222,52 +224,52 @@ func (s *AttachmentService) GetContent( return data, model.ContentType(row.ContentType), nil } -func (s *AttachmentService) checkImage(ctx context.Context, r io.ReadSeeker, contentType string) (model.ContentType, error) { +func (s *AttachmentService) checkImage(ctx context.Context, r io.ReadSeeker, contentType string) (model.ContentType, int, int, error) { cfg, format, err := image.DecodeConfig(r) if err != nil { - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } if cfg.Width <= 0 || cfg.Height <= 0 { - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } if cfg.Width > s.maxPixels/cfg.Height { - return "", ErrInvalidSize + return "", 0, 0, ErrInvalidSize } var detectedType model.ContentType switch format { case "png": if contentType != string(model.ContentTypePNG) { - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } detectedType = model.ContentTypePNG case "jpeg": if contentType != string(model.ContentTypeJPEG) { - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } detectedType = model.ContentTypeJPEG default: - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } select { case s.decodeSlot <- struct{}{}: defer func() { <-s.decodeSlot }() case <-ctx.Done(): - return "", ctx.Err() + return "", 0, 0, ctx.Err() } if err := ctx.Err(); err != nil { - return "", err + return "", 0, 0, err } if _, err := r.Seek(0, io.SeekStart); err != nil { - return "", err + return "", 0, 0, err } if _, _, err := image.Decode(r); err != nil { - return "", ErrInvalidImage + return "", 0, 0, ErrInvalidImage } - return detectedType, nil + return detectedType, cfg.Width, cfg.Height, nil } func (s *AttachmentService) ensureMember(ctx context.Context, userID, chatID uuid.UUID) error {