feat(attachments): add pipeline and outbox schema and queries
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
-- +goose Up
|
||||
CREATE TYPE attachment_variant_name AS ENUM ('thumbnail', 'crop', 'optimized');
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text UNIQUE,
|
||||
ADD COLUMN job_id uuid UNIQUE,
|
||||
ADD COLUMN pipeline_version int NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
|
||||
ADD COLUMN attempts int NOT NULL DEFAULT 0 CHECK (attempts >= 0),
|
||||
ADD COLUMN width int CHECK (width > 0),
|
||||
ADD COLUMN height int CHECK (height > 0),
|
||||
ADD COLUMN lease_until timestamptz;
|
||||
|
||||
CREATE INDEX idx_attachments_expired_lease ON attachments (lease_until) WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version int NOT NULL CHECK (pipeline_version > 0),
|
||||
variant attachment_variant_name NOT NULL,
|
||||
s3_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')),
|
||||
size_bytes int NOT NULL CHECK (size_bytes > 0),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width int NOT NULL CHECK (width > 0),
|
||||
height int NOT NULL CHECK (height > 0),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachments (job_id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version int NOT NULL CHECK (pipeline_version > 0),
|
||||
attempt int NOT NULL CHECK (attempt > 0),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
UNIQUE (job_id, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE outbox;
|
||||
DROP TABLE attachment_variants;
|
||||
DROP TYPE attachment_variant_name;
|
||||
DROP INDEX idx_attachments_expired_lease;
|
||||
ALTER TABLE attachments
|
||||
DROP COLUMN height,
|
||||
DROP COLUMN width,
|
||||
DROP COLUMN lease_until,
|
||||
DROP COLUMN attempts,
|
||||
DROP COLUMN pipeline_version,
|
||||
DROP COLUMN job_id,
|
||||
DROP COLUMN staging_key;
|
||||
@@ -0,0 +1,28 @@
|
||||
-- name: CreateAttachmentVariant :exec
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
variant,
|
||||
s3_key,
|
||||
content_type,
|
||||
size_bytes,
|
||||
width,
|
||||
height,
|
||||
sha256
|
||||
) VALUES (
|
||||
sqlc.arg(attachment_id),
|
||||
sqlc.arg(pipeline_version),
|
||||
sqlc.arg(variant),
|
||||
sqlc.arg(s3_key),
|
||||
sqlc.arg(content_type),
|
||||
sqlc.arg(size_bytes),
|
||||
sqlc.arg(width),
|
||||
sqlc.arg(height),
|
||||
sqlc.arg(sha256)
|
||||
);
|
||||
|
||||
-- name: ListAttachmentVariants :many
|
||||
SELECT * FROM attachment_variants
|
||||
WHERE attachment_id = sqlc.arg(attachment_id)::uuid
|
||||
AND pipeline_version = sqlc.arg(pipeline_version)::int
|
||||
ORDER BY variant;
|
||||
@@ -4,10 +4,14 @@ INSERT INTO attachments (
|
||||
chat_id,
|
||||
owner_id,
|
||||
s3_key,
|
||||
staging_key,
|
||||
job_id,
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
sha256,
|
||||
width,
|
||||
height,
|
||||
error_code
|
||||
)
|
||||
SELECT
|
||||
@@ -15,10 +19,14 @@ SELECT
|
||||
sqlc.arg(chat_id)::uuid,
|
||||
sqlc.arg(owner_id)::uuid,
|
||||
sqlc.arg(s3_key)::text,
|
||||
sqlc.narg(staging_key)::text,
|
||||
sqlc.narg(job_id)::uuid,
|
||||
sqlc.arg(filename)::text,
|
||||
sqlc.arg(content_type)::attachment_content_type,
|
||||
sqlc.arg(size_bytes)::int,
|
||||
sqlc.arg(sha256)::text,
|
||||
sqlc.arg(width)::int,
|
||||
sqlc.arg(height)::int,
|
||||
NULLIF(sqlc.arg(error_code)::text, '')
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members
|
||||
@@ -77,6 +85,30 @@ SELECT
|
||||
) AS referenced
|
||||
FROM attachments a;
|
||||
|
||||
-- name: LockAttachmentJob :one
|
||||
SELECT * FROM attachments
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND job_id = sqlc.arg(job_id)::uuid
|
||||
AND pipeline_version = sqlc.arg(pipeline_version)::int
|
||||
AND attempts = sqlc.arg(attempt)::int
|
||||
AND lease_until > now()
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: LockExpiredAttachmentJobs :many
|
||||
SELECT * FROM attachments
|
||||
WHERE status = 'processing'
|
||||
AND lease_until <= now()
|
||||
ORDER BY lease_until, id
|
||||
LIMIT sqlc.arg(batch_size)::int
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
-- name: MarkAttachmentReady :one
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
RETURNING *;
|
||||
|
||||
-- name: MarkAttachmentFailedIfReady :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
@@ -84,12 +116,73 @@ SET status = 'failed',
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid AND status = 'ready';
|
||||
|
||||
-- name: MarkAttachmentReady :one
|
||||
-- name: ClaimAttachmentJob :one
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
SET status = 'processing',
|
||||
attempts = attempts + 1,
|
||||
lease_until = sqlc.arg(lease_until)::timestamptz,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND job_id = sqlc.arg(job_id)::uuid
|
||||
AND pipeline_version = sqlc.arg(pipeline_version)::int
|
||||
AND status = 'queued'
|
||||
AND attempts + 1 = sqlc.arg(attempt)::int
|
||||
RETURNING *;
|
||||
|
||||
-- name: CompleteAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
width = sqlc.arg(width)::int,
|
||||
height = sqlc.arg(height)::int,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = sqlc.arg(attempt)::int
|
||||
AND lease_until > now();
|
||||
|
||||
-- name: RetryAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
error_code = sqlc.arg(error_code)::text,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = sqlc.arg(attempt)::int
|
||||
AND lease_until > now();
|
||||
|
||||
-- name: FailAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = sqlc.arg(error_code)::text,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = sqlc.arg(attempt)::int
|
||||
AND lease_until > now();
|
||||
|
||||
-- name: RecoverExpiredAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND lease_until <= now();
|
||||
|
||||
-- name: FailExpiredAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = 'retry_exhausted',
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
AND lease_until <= now();
|
||||
|
||||
-- name: DeleteQueuedAttachment :execrows
|
||||
DELETE FROM attachments
|
||||
WHERE id = sqlc.arg(id)::uuid AND status = 'queued';
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
-- name: CreateOutboxEntry :one
|
||||
INSERT INTO outbox (
|
||||
job_id,
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
attempt,
|
||||
available_at
|
||||
)
|
||||
SELECT
|
||||
job_id,
|
||||
id,
|
||||
pipeline_version,
|
||||
sqlc.arg(attempt)::int,
|
||||
sqlc.arg(available_at)::timestamptz
|
||||
FROM attachments
|
||||
WHERE id = sqlc.arg(attachment_id)::uuid
|
||||
AND job_id IS NOT NULL
|
||||
RETURNING *;
|
||||
|
||||
-- name: LockPendingOutboxEntries :many
|
||||
SELECT * FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
AND available_at <= now()
|
||||
ORDER BY available_at, id
|
||||
LIMIT sqlc.arg(batch_size)::int
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
-- name: MarkOutboxPublished :execrows
|
||||
UPDATE outbox
|
||||
SET published_at = now()
|
||||
WHERE id = sqlc.arg(id)::bigint
|
||||
AND published_at IS NULL;
|
||||
@@ -0,0 +1,105 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: attachment_variants.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const createAttachmentVariant = `-- name: CreateAttachmentVariant :exec
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
variant,
|
||||
s3_key,
|
||||
content_type,
|
||||
size_bytes,
|
||||
width,
|
||||
height,
|
||||
sha256
|
||||
) VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4,
|
||||
$5,
|
||||
$6,
|
||||
$7,
|
||||
$8,
|
||||
$9
|
||||
)
|
||||
`
|
||||
|
||||
type CreateAttachmentVariantParams struct {
|
||||
AttachmentID uuid.UUID
|
||||
PipelineVersion int
|
||||
Variant AttachmentVariantName
|
||||
S3Key string
|
||||
ContentType string
|
||||
SizeBytes int
|
||||
Width int
|
||||
Height int
|
||||
Sha256 string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateAttachmentVariant(ctx context.Context, arg CreateAttachmentVariantParams) error {
|
||||
_, err := q.db.Exec(ctx, createAttachmentVariant,
|
||||
arg.AttachmentID,
|
||||
arg.PipelineVersion,
|
||||
arg.Variant,
|
||||
arg.S3Key,
|
||||
arg.ContentType,
|
||||
arg.SizeBytes,
|
||||
arg.Width,
|
||||
arg.Height,
|
||||
arg.Sha256,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const listAttachmentVariants = `-- name: ListAttachmentVariants :many
|
||||
SELECT attachment_id, pipeline_version, variant, s3_key, content_type, size_bytes, sha256, width, height FROM attachment_variants
|
||||
WHERE attachment_id = $1::uuid
|
||||
AND pipeline_version = $2::int
|
||||
ORDER BY variant
|
||||
`
|
||||
|
||||
type ListAttachmentVariantsParams struct {
|
||||
AttachmentID uuid.UUID
|
||||
PipelineVersion int
|
||||
}
|
||||
|
||||
func (q *Queries) ListAttachmentVariants(ctx context.Context, arg ListAttachmentVariantsParams) ([]AttachmentVariant, error) {
|
||||
rows, err := q.db.Query(ctx, listAttachmentVariants, arg.AttachmentID, arg.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []AttachmentVariant{}
|
||||
for rows.Next() {
|
||||
var i AttachmentVariant
|
||||
if err := rows.Scan(
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.Variant,
|
||||
&i.S3Key,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -12,16 +12,109 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const claimAttachmentJob = `-- name: ClaimAttachmentJob :one
|
||||
UPDATE attachments
|
||||
SET status = 'processing',
|
||||
attempts = attempts + 1,
|
||||
lease_until = $1::timestamptz,
|
||||
updated_at = now()
|
||||
WHERE id = $2::uuid
|
||||
AND job_id = $3::uuid
|
||||
AND pipeline_version = $4::int
|
||||
AND status = 'queued'
|
||||
AND attempts + 1 = $5::int
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until
|
||||
`
|
||||
|
||||
type ClaimAttachmentJobParams struct {
|
||||
LeaseUntil time.Time
|
||||
ID uuid.UUID
|
||||
JobID uuid.UUID
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) ClaimAttachmentJob(ctx context.Context, arg ClaimAttachmentJobParams) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, claimAttachmentJob,
|
||||
arg.LeaseUntil,
|
||||
arg.ID,
|
||||
arg.JobID,
|
||||
arg.PipelineVersion,
|
||||
arg.Attempt,
|
||||
)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const completeAttachmentJob = `-- name: CompleteAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
width = $1::int,
|
||||
height = $2::int,
|
||||
updated_at = now()
|
||||
WHERE id = $3::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = $4::int
|
||||
AND lease_until > now()
|
||||
`
|
||||
|
||||
type CompleteAttachmentJobParams struct {
|
||||
Width int
|
||||
Height int
|
||||
ID uuid.UUID
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) CompleteAttachmentJob(ctx context.Context, arg CompleteAttachmentJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, completeAttachmentJob,
|
||||
arg.Width,
|
||||
arg.Height,
|
||||
arg.ID,
|
||||
arg.Attempt,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const createAttachment = `-- name: CreateAttachment :one
|
||||
INSERT INTO attachments (
|
||||
id,
|
||||
chat_id,
|
||||
owner_id,
|
||||
s3_key,
|
||||
staging_key,
|
||||
job_id,
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
sha256,
|
||||
width,
|
||||
height,
|
||||
error_code
|
||||
)
|
||||
SELECT
|
||||
@@ -30,16 +123,20 @@ SELECT
|
||||
$3::uuid,
|
||||
$4::text,
|
||||
$5::text,
|
||||
$6::attachment_content_type,
|
||||
$7::int,
|
||||
$8::text,
|
||||
NULLIF($9::text, '')
|
||||
$6::uuid,
|
||||
$7::text,
|
||||
$8::attachment_content_type,
|
||||
$9::int,
|
||||
$10::text,
|
||||
$11::int,
|
||||
$12::int,
|
||||
NULLIF($13::text, '')
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members
|
||||
WHERE chat_id = $2
|
||||
AND user_id = $3
|
||||
)
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until
|
||||
`
|
||||
|
||||
type CreateAttachmentParams struct {
|
||||
@@ -47,10 +144,14 @@ type CreateAttachmentParams struct {
|
||||
ChatID uuid.UUID
|
||||
OwnerID uuid.UUID
|
||||
S3Key string
|
||||
StagingKey *string
|
||||
JobID *uuid.UUID
|
||||
Filename string
|
||||
ContentType AttachmentContentType
|
||||
SizeBytes int
|
||||
Sha256 string
|
||||
Width int
|
||||
Height int
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
@@ -60,10 +161,14 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara
|
||||
arg.ChatID,
|
||||
arg.OwnerID,
|
||||
arg.S3Key,
|
||||
arg.StagingKey,
|
||||
arg.JobID,
|
||||
arg.Filename,
|
||||
arg.ContentType,
|
||||
arg.SizeBytes,
|
||||
arg.Sha256,
|
||||
arg.Width,
|
||||
arg.Height,
|
||||
arg.ErrorCode,
|
||||
)
|
||||
var i Attachment
|
||||
@@ -80,6 +185,13 @@ func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentPara
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -120,8 +232,53 @@ func (q *Queries) DeleteQueuedAttachment(ctx context.Context, id uuid.UUID) (int
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const failAttachmentJob = `-- name: FailAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = $1::text,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $2::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = $3::int
|
||||
AND lease_until > now()
|
||||
`
|
||||
|
||||
type FailAttachmentJobParams struct {
|
||||
ErrorCode string
|
||||
ID uuid.UUID
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) FailAttachmentJob(ctx context.Context, arg FailAttachmentJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, failAttachmentJob, arg.ErrorCode, arg.ID, arg.Attempt)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const failExpiredAttachmentJob = `-- name: FailExpiredAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = 'retry_exhausted',
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND lease_until <= now()
|
||||
`
|
||||
|
||||
func (q *Queries) FailExpiredAttachmentJob(ctx context.Context, id uuid.UUID) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, failExpiredAttachmentJob, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const getAttachmentByID = `-- name: GetAttachmentByID :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at FROM attachments
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
`
|
||||
|
||||
@@ -141,6 +298,13 @@ func (q *Queries) GetAttachmentByID(ctx context.Context, dollar_1 uuid.UUID) (At
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -316,6 +480,105 @@ func (q *Queries) ListStaleAttachmentUploads(ctx context.Context, cutoff time.Ti
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockAttachmentJob = `-- name: LockAttachmentJob :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND job_id = $2::uuid
|
||||
AND pipeline_version = $3::int
|
||||
AND attempts = $4::int
|
||||
AND lease_until > now()
|
||||
FOR UPDATE
|
||||
`
|
||||
|
||||
type LockAttachmentJobParams struct {
|
||||
ID uuid.UUID
|
||||
JobID uuid.UUID
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) LockAttachmentJob(ctx context.Context, arg LockAttachmentJobParams) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, lockAttachmentJob,
|
||||
arg.ID,
|
||||
arg.JobID,
|
||||
arg.PipelineVersion,
|
||||
arg.Attempt,
|
||||
)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockExpiredAttachmentJobs = `-- name: LockExpiredAttachmentJobs :many
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE status = 'processing'
|
||||
AND lease_until <= now()
|
||||
ORDER BY lease_until, id
|
||||
LIMIT $1::int
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`
|
||||
|
||||
func (q *Queries) LockExpiredAttachmentJobs(ctx context.Context, batchSize int) ([]Attachment, error) {
|
||||
rows, err := q.db.Query(ctx, lockExpiredAttachmentJobs, batchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Attachment{}
|
||||
for rows.Next() {
|
||||
var i Attachment
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markAttachmentFailedIfReady = `-- name: MarkAttachmentFailedIfReady :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
@@ -341,11 +604,11 @@ const markAttachmentReady = `-- name: MarkAttachmentReady :one
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until
|
||||
`
|
||||
|
||||
func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, markAttachmentReady, dollar_1)
|
||||
func (q *Queries) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, markAttachmentReady, id)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
@@ -360,6 +623,57 @@ func (q *Queries) MarkAttachmentReady(ctx context.Context, dollar_1 uuid.UUID) (
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const recoverExpiredAttachmentJob = `-- name: RecoverExpiredAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND lease_until <= now()
|
||||
`
|
||||
|
||||
func (q *Queries) RecoverExpiredAttachmentJob(ctx context.Context, id uuid.UUID) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, recoverExpiredAttachmentJob, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const retryAttachmentJob = `-- name: RetryAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
error_code = $1::text,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $2::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = $3::int
|
||||
AND lease_until > now()
|
||||
`
|
||||
|
||||
type RetryAttachmentJobParams struct {
|
||||
ErrorCode string
|
||||
ID uuid.UUID
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) RetryAttachmentJob(ctx context.Context, arg RetryAttachmentJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, retryAttachmentJob, arg.ErrorCode, arg.ID, arg.Attempt)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
+85
-12
@@ -98,19 +98,81 @@ func (ns NullAttachmentStatus) Value() (driver.Value, error) {
|
||||
return string(ns.AttachmentStatus), nil
|
||||
}
|
||||
|
||||
type AttachmentVariantName string
|
||||
|
||||
const (
|
||||
AttachmentVariantNameThumbnail AttachmentVariantName = "thumbnail"
|
||||
AttachmentVariantNameCrop AttachmentVariantName = "crop"
|
||||
AttachmentVariantNameOptimized AttachmentVariantName = "optimized"
|
||||
)
|
||||
|
||||
func (e *AttachmentVariantName) Scan(src interface{}) error {
|
||||
switch s := src.(type) {
|
||||
case []byte:
|
||||
*e = AttachmentVariantName(s)
|
||||
case string:
|
||||
*e = AttachmentVariantName(s)
|
||||
default:
|
||||
return fmt.Errorf("unsupported scan type for AttachmentVariantName: %T", src)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type NullAttachmentVariantName struct {
|
||||
AttachmentVariantName AttachmentVariantName
|
||||
Valid bool // Valid is true if AttachmentVariantName is not NULL
|
||||
}
|
||||
|
||||
// Scan implements the Scanner interface.
|
||||
func (ns *NullAttachmentVariantName) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
ns.AttachmentVariantName, ns.Valid = "", false
|
||||
return nil
|
||||
}
|
||||
ns.Valid = true
|
||||
return ns.AttachmentVariantName.Scan(value)
|
||||
}
|
||||
|
||||
// Value implements the driver Valuer interface.
|
||||
func (ns NullAttachmentVariantName) Value() (driver.Value, error) {
|
||||
if !ns.Valid {
|
||||
return nil, nil
|
||||
}
|
||||
return string(ns.AttachmentVariantName), nil
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
OwnerID uuid.UUID
|
||||
S3Key string
|
||||
Filename string
|
||||
ContentType AttachmentContentType
|
||||
SizeBytes int
|
||||
Sha256 string
|
||||
Status AttachmentStatus
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
OwnerID uuid.UUID
|
||||
S3Key string
|
||||
Filename string
|
||||
ContentType AttachmentContentType
|
||||
SizeBytes int
|
||||
Sha256 string
|
||||
Status AttachmentStatus
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
StagingKey *string
|
||||
JobID *uuid.UUID
|
||||
PipelineVersion int
|
||||
Attempts int
|
||||
Width *int32
|
||||
Height *int32
|
||||
LeaseUntil *time.Time
|
||||
}
|
||||
|
||||
type AttachmentVariant struct {
|
||||
AttachmentID uuid.UUID
|
||||
PipelineVersion int
|
||||
Variant AttachmentVariantName
|
||||
S3Key string
|
||||
ContentType string
|
||||
SizeBytes int
|
||||
Sha256 string
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
@@ -148,6 +210,17 @@ type MessageAttachment struct {
|
||||
Position int
|
||||
}
|
||||
|
||||
type Outbox struct {
|
||||
ID int64
|
||||
JobID uuid.UUID
|
||||
AttachmentID uuid.UUID
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
CreatedAt time.Time
|
||||
AvailableAt time.Time
|
||||
PublishedAt *time.Time
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
TokenHash string
|
||||
UserID uuid.UUID
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: outbox.sql
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const createOutboxEntry = `-- name: CreateOutboxEntry :one
|
||||
INSERT INTO outbox (
|
||||
job_id,
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
attempt,
|
||||
available_at
|
||||
)
|
||||
SELECT
|
||||
job_id,
|
||||
id,
|
||||
pipeline_version,
|
||||
$1::int,
|
||||
$2::timestamptz
|
||||
FROM attachments
|
||||
WHERE id = $3::uuid
|
||||
AND job_id IS NOT NULL
|
||||
RETURNING id, job_id, attachment_id, pipeline_version, attempt, created_at, available_at, published_at
|
||||
`
|
||||
|
||||
type CreateOutboxEntryParams struct {
|
||||
Attempt int
|
||||
AvailableAt time.Time
|
||||
AttachmentID uuid.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) CreateOutboxEntry(ctx context.Context, arg CreateOutboxEntryParams) (Outbox, error) {
|
||||
row := q.db.QueryRow(ctx, createOutboxEntry, arg.Attempt, arg.AvailableAt, arg.AttachmentID)
|
||||
var i Outbox
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.JobID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempt,
|
||||
&i.CreatedAt,
|
||||
&i.AvailableAt,
|
||||
&i.PublishedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockPendingOutboxEntries = `-- name: LockPendingOutboxEntries :many
|
||||
SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, available_at, published_at FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
AND available_at <= now()
|
||||
ORDER BY available_at, id
|
||||
LIMIT $1::int
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`
|
||||
|
||||
func (q *Queries) LockPendingOutboxEntries(ctx context.Context, batchSize int) ([]Outbox, error) {
|
||||
rows, err := q.db.Query(ctx, lockPendingOutboxEntries, batchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Outbox{}
|
||||
for rows.Next() {
|
||||
var i Outbox
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.JobID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempt,
|
||||
&i.CreatedAt,
|
||||
&i.AvailableAt,
|
||||
&i.PublishedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markOutboxPublished = `-- name: MarkOutboxPublished :execrows
|
||||
UPDATE outbox
|
||||
SET published_at = now()
|
||||
WHERE id = $1::bigint
|
||||
AND published_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) MarkOutboxPublished(ctx context.Context, id int64) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, markOutboxPublished, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func (s *AttachmentService) Upload(
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
detectedType, err := s.checkImage(ctx, tmp, contentType)
|
||||
detectedType, width, height, err := s.checkImage(ctx, tmp, contentType)
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
@@ -124,6 +124,8 @@ func (s *AttachmentService) Upload(
|
||||
ContentType: db.AttachmentContentType(detectedType),
|
||||
SizeBytes: int(size),
|
||||
Sha256: hash,
|
||||
Width: width,
|
||||
Height: height,
|
||||
ErrorCode: "",
|
||||
})
|
||||
if postgres.IsNotFound(err) {
|
||||
@@ -222,52 +224,52 @@ func (s *AttachmentService) GetContent(
|
||||
return data, model.ContentType(row.ContentType), nil
|
||||
}
|
||||
|
||||
func (s *AttachmentService) checkImage(ctx context.Context, r io.ReadSeeker, contentType string) (model.ContentType, error) {
|
||||
func (s *AttachmentService) checkImage(ctx context.Context, r io.ReadSeeker, contentType string) (model.ContentType, int, int, error) {
|
||||
cfg, format, err := image.DecodeConfig(r)
|
||||
if err != nil {
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
if cfg.Width > s.maxPixels/cfg.Height {
|
||||
return "", ErrInvalidSize
|
||||
return "", 0, 0, ErrInvalidSize
|
||||
}
|
||||
|
||||
var detectedType model.ContentType
|
||||
switch format {
|
||||
case "png":
|
||||
if contentType != string(model.ContentTypePNG) {
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
detectedType = model.ContentTypePNG
|
||||
case "jpeg":
|
||||
if contentType != string(model.ContentTypeJPEG) {
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
detectedType = model.ContentTypeJPEG
|
||||
default:
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
|
||||
select {
|
||||
case s.decodeSlot <- struct{}{}:
|
||||
defer func() { <-s.decodeSlot }()
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
return "", 0, 0, ctx.Err()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return "", err
|
||||
return "", 0, 0, err
|
||||
}
|
||||
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
||||
return "", err
|
||||
return "", 0, 0, err
|
||||
}
|
||||
if _, _, err := image.Decode(r); err != nil {
|
||||
return "", ErrInvalidImage
|
||||
return "", 0, 0, ErrInvalidImage
|
||||
}
|
||||
|
||||
return detectedType, nil
|
||||
return detectedType, cfg.Width, cfg.Height, nil
|
||||
}
|
||||
|
||||
func (s *AttachmentService) ensureMember(ctx context.Context, userID, chatID uuid.UUID) error {
|
||||
|
||||
Reference in New Issue
Block a user