From f8ca7a72182e940f8c3cff3ace4585e02650a024 Mon Sep 17 00:00:00 2001 From: kk10 Date: Mon, 16 Nov 2026 22:26:35 +0300 Subject: [PATCH] feat(worker): add pipeline version upgrade command --- cmd/worker/main.go | 42 +++++++++++++-- db/migrations/00004_attachment_pipeline.sql | 2 +- db/queries/attachments.sql | 21 ++++++++ internal/db/attachments.sql.go | 59 ++++++++++++++++++++ internal/worker/upgrade.go | 60 +++++++++++++++++++++ internal/worker/worker.go | 2 +- 6 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 internal/worker/upgrade.go diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 60c1cd8..06dcb16 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -24,9 +24,11 @@ import ( ) var ( - errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined") - errInvalidReplayID = errors.New("replay must be an attachment UUID") - errMissingReplayAudit = errors.New("replay requires nonempty operator and reason") + errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined") + errInvalidReplayID = errors.New("replay must be an attachment UUID") + errMissingReplayAudit = errors.New("replay requires nonempty operator and reason") + errConflictingUpgradeFlags = errors.New("upgrade cannot be combined with replay or list-failed") + errInvalidUpgradeID = errors.New("upgrade must be an attachment UUID") ) func main() { @@ -36,6 +38,7 @@ func main() { operator := flag.String("operator", "", "operator requesting replay") reason := flag.String("reason", "", "reason for replay") listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments") + upgrade := flag.String("upgrade", "", "upgrade a v1 queued or processing attachment to v2; requires retained staging") flag.Parse() if *maxPixels > 0 { @@ -56,12 +59,16 @@ func main() { var err error switch { + case *upgrade != "" && (*listFailed || *replay != ""): + err = errConflictingUpgradeFlags case *listFailed && *replay != "": err = errConflictingReplayFlags case *listFailed: err = runListFailed(ctx) case *replay != "": err = runReplay(ctx, *replay, *operator, *reason) + case *upgrade != "": + err = runUpgrade(ctx, *upgrade) default: err = runWorker(ctx) } @@ -133,6 +140,35 @@ func runReplay(ctx context.Context, replay, operator, reason string) error { return nil } +func runUpgrade(ctx context.Context, upgrade string) error { + id, err := uuid.Parse(upgrade) + if err != nil { + return errInvalidUpgradeID + } + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + cfg, pool, err := loadDatabase(ctx) + if err != nil { + return err + } + defer pool.Close() + + log := logger.New(cfg.Logging.Level) + storage, err := s3store.NewAPI(ctx, cfg.S3, log) + if err != nil { + return err + } + + if err := worker.Upgrade(ctx, pool, storage, id); err != nil { + return err + } + + log.Info("attachment pipeline upgrade queued", "attachment_id", id, "pipeline_version", 2) + return nil +} + func runWorker(ctx context.Context) error { cfg, pool, err := loadDatabase(ctx) if err != nil { diff --git a/db/migrations/00004_attachment_pipeline.sql b/db/migrations/00004_attachment_pipeline.sql index 9bd17b9..676f456 100644 --- a/db/migrations/00004_attachment_pipeline.sql +++ b/db/migrations/00004_attachment_pipeline.sql @@ -35,7 +35,7 @@ CREATE TABLE outbox ( created_at timestamptz NOT NULL DEFAULT now(), available_at timestamptz NOT NULL DEFAULT now(), published_at timestamptz, - UNIQUE (job_id, attempt) + UNIQUE (job_id, pipeline_version, attempt) ); CREATE INDEX idx_outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql index 9406050..d0de5aa 100644 --- a/db/queries/attachments.sql +++ b/db/queries/attachments.sql @@ -113,6 +113,14 @@ WHERE id = sqlc.arg(id)::uuid ) FOR UPDATE; +-- name: LockAttachmentForUpgrade :one +SELECT * FROM attachments +WHERE id = sqlc.arg(id)::uuid + AND pipeline_version = 1 + AND status IN ('queued', 'processing') + AND staging_key IS NOT NULL +FOR UPDATE; + -- name: ClearAttachmentStagingKey :exec UPDATE attachments SET staging_key = NULL @@ -160,6 +168,19 @@ WHERE id = sqlc.arg(id)::uuid AND attempts = sqlc.arg(attempt)::int AND lease_until > now(); +-- name: UpgradeAttachmentPipeline :execrows +UPDATE attachments +SET pipeline_version = 2, + status = 'queued', + retry_base = attempts, + error_code = NULL, + lease_until = NULL, + updated_at = now() +WHERE id = sqlc.arg(id)::uuid + AND pipeline_version = 1 + AND status IN ('queued', 'processing') + AND staging_key IS NOT NULL; + -- name: ReplayAttachmentJob :execrows UPDATE attachments SET status = 'queued', diff --git a/internal/db/attachments.sql.go b/internal/db/attachments.sql.go index b367fe2..a0a1272 100644 --- a/internal/db/attachments.sql.go +++ b/internal/db/attachments.sql.go @@ -509,6 +509,43 @@ func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey str return i, err } +const lockAttachmentForUpgrade = `-- name: LockAttachmentForUpgrade :one +SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments +WHERE id = $1::uuid + AND pipeline_version = 1 + AND status IN ('queued', 'processing') + AND staging_key IS NOT NULL +FOR UPDATE +` + +func (q *Queries) LockAttachmentForUpgrade(ctx context.Context, id uuid.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, lockAttachmentForUpgrade, id) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.S3Key, + &i.Filename, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.Status, + &i.ErrorCode, + &i.CreatedAt, + &i.UpdatedAt, + &i.StagingKey, + &i.JobID, + &i.PipelineVersion, + &i.Attempts, + &i.RetryBase, + &i.Width, + &i.Height, + &i.LeaseUntil, + ) + return i, err +} + const lockAttachmentJob = `-- name: LockAttachmentJob :one SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments WHERE id = $1::uuid @@ -738,3 +775,25 @@ func (q *Queries) RetryAttachmentJob(ctx context.Context, arg RetryAttachmentJob } return result.RowsAffected(), nil } + +const upgradeAttachmentPipeline = `-- name: UpgradeAttachmentPipeline :execrows +UPDATE attachments +SET pipeline_version = 2, + status = 'queued', + retry_base = attempts, + error_code = NULL, + lease_until = NULL, + updated_at = now() +WHERE id = $1::uuid + AND pipeline_version = 1 + AND status IN ('queued', 'processing') + AND staging_key IS NOT NULL +` + +func (q *Queries) UpgradeAttachmentPipeline(ctx context.Context, id uuid.UUID) (int64, error) { + result, err := q.db.Exec(ctx, upgradeAttachmentPipeline, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/worker/upgrade.go b/internal/worker/upgrade.go new file mode 100644 index 0000000..ed1dd92 --- /dev/null +++ b/internal/worker/upgrade.go @@ -0,0 +1,60 @@ +package worker + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "mtusi2026-messenger/internal/db" + s3store "mtusi2026-messenger/internal/s3" +) + +var ( + errAttachmentNotUpgradeable = errors.New("attachment is not upgradeable: expected v1 queued or processing with staging") + errUpgradeNotUpdated = errors.New("attachment upgrade did not update one row") +) + +func Upgrade(ctx context.Context, pool txBeginner, storage *s3store.Client, id uuid.UUID) error { + tx, err := pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + + q := db.New(tx) + row, err := q.LockAttachmentForUpgrade(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return errAttachmentNotUpgradeable + } + if err != nil { + return err + } + if row.JobID == nil { + return errMissingPipelineJob + } + + if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil { + return err + } + + n, err := q.UpgradeAttachmentPipeline(ctx, id) + if err != nil { + return err + } + if n != 1 { + return errUpgradeNotUpdated + } + + if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ + AttachmentID: id, + Attempt: row.Attempts + 1, + AvailableAt: time.Now(), + }); err != nil { + return err + } + + return tx.Commit(ctx) +} diff --git a/internal/worker/worker.go b/internal/worker/worker.go index a02cf82..9a7f6f6 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -116,7 +116,7 @@ func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error { job.Type != "attachment.process.v1" || job.JobID == uuid.Nil || job.AttachmentID == uuid.Nil || - job.PipelineVersion != 1 || + (job.PipelineVersion != 1 && job.PipelineVersion != 2) || job.Attempt < 1 || job.CreatedAt.IsZero() { if err := delivery.Reject(false); err != nil {