feat(worker): add pipeline version upgrade command

This commit is contained in:
kk10
2026-11-16 22:26:35 +03:00
parent 5442dfd3bc
commit f8ca7a7218
6 changed files with 181 additions and 5 deletions
+39 -3
View File
@@ -24,9 +24,11 @@ import (
)
var (
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
errInvalidReplayID = errors.New("replay must be an attachment UUID")
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
errInvalidReplayID = errors.New("replay must be an attachment UUID")
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
errConflictingUpgradeFlags = errors.New("upgrade cannot be combined with replay or list-failed")
errInvalidUpgradeID = errors.New("upgrade must be an attachment UUID")
)
func main() {
@@ -36,6 +38,7 @@ func main() {
operator := flag.String("operator", "", "operator requesting replay")
reason := flag.String("reason", "", "reason for replay")
listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments")
upgrade := flag.String("upgrade", "", "upgrade a v1 queued or processing attachment to v2; requires retained staging")
flag.Parse()
if *maxPixels > 0 {
@@ -56,12 +59,16 @@ func main() {
var err error
switch {
case *upgrade != "" && (*listFailed || *replay != ""):
err = errConflictingUpgradeFlags
case *listFailed && *replay != "":
err = errConflictingReplayFlags
case *listFailed:
err = runListFailed(ctx)
case *replay != "":
err = runReplay(ctx, *replay, *operator, *reason)
case *upgrade != "":
err = runUpgrade(ctx, *upgrade)
default:
err = runWorker(ctx)
}
@@ -133,6 +140,35 @@ func runReplay(ctx context.Context, replay, operator, reason string) error {
return nil
}
func runUpgrade(ctx context.Context, upgrade string) error {
id, err := uuid.Parse(upgrade)
if err != nil {
return errInvalidUpgradeID
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
cfg, pool, err := loadDatabase(ctx)
if err != nil {
return err
}
defer pool.Close()
log := logger.New(cfg.Logging.Level)
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
if err != nil {
return err
}
if err := worker.Upgrade(ctx, pool, storage, id); err != nil {
return err
}
log.Info("attachment pipeline upgrade queued", "attachment_id", id, "pipeline_version", 2)
return nil
}
func runWorker(ctx context.Context) error {
cfg, pool, err := loadDatabase(ctx)
if err != nil {
+1 -1
View File
@@ -35,7 +35,7 @@ CREATE TABLE outbox (
created_at timestamptz NOT NULL DEFAULT now(),
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
UNIQUE (job_id, attempt)
UNIQUE (job_id, pipeline_version, attempt)
);
CREATE INDEX idx_outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL;
+21
View File
@@ -113,6 +113,14 @@ WHERE id = sqlc.arg(id)::uuid
)
FOR UPDATE;
-- name: LockAttachmentForUpgrade :one
SELECT * FROM attachments
WHERE id = sqlc.arg(id)::uuid
AND pipeline_version = 1
AND status IN ('queued', 'processing')
AND staging_key IS NOT NULL
FOR UPDATE;
-- name: ClearAttachmentStagingKey :exec
UPDATE attachments
SET staging_key = NULL
@@ -160,6 +168,19 @@ WHERE id = sqlc.arg(id)::uuid
AND attempts = sqlc.arg(attempt)::int
AND lease_until > now();
-- name: UpgradeAttachmentPipeline :execrows
UPDATE attachments
SET pipeline_version = 2,
status = 'queued',
retry_base = attempts,
error_code = NULL,
lease_until = NULL,
updated_at = now()
WHERE id = sqlc.arg(id)::uuid
AND pipeline_version = 1
AND status IN ('queued', 'processing')
AND staging_key IS NOT NULL;
-- name: ReplayAttachmentJob :execrows
UPDATE attachments
SET status = 'queued',
+59
View File
@@ -509,6 +509,43 @@ func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey str
return i, err
}
const lockAttachmentForUpgrade = `-- name: LockAttachmentForUpgrade :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments
WHERE id = $1::uuid
AND pipeline_version = 1
AND status IN ('queued', 'processing')
AND staging_key IS NOT NULL
FOR UPDATE
`
func (q *Queries) LockAttachmentForUpgrade(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, lockAttachmentForUpgrade, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.S3Key,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.ErrorCode,
&i.CreatedAt,
&i.UpdatedAt,
&i.StagingKey,
&i.JobID,
&i.PipelineVersion,
&i.Attempts,
&i.RetryBase,
&i.Width,
&i.Height,
&i.LeaseUntil,
)
return i, err
}
const lockAttachmentJob = `-- name: LockAttachmentJob :one
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments
WHERE id = $1::uuid
@@ -738,3 +775,25 @@ func (q *Queries) RetryAttachmentJob(ctx context.Context, arg RetryAttachmentJob
}
return result.RowsAffected(), nil
}
const upgradeAttachmentPipeline = `-- name: UpgradeAttachmentPipeline :execrows
UPDATE attachments
SET pipeline_version = 2,
status = 'queued',
retry_base = attempts,
error_code = NULL,
lease_until = NULL,
updated_at = now()
WHERE id = $1::uuid
AND pipeline_version = 1
AND status IN ('queued', 'processing')
AND staging_key IS NOT NULL
`
func (q *Queries) UpgradeAttachmentPipeline(ctx context.Context, id uuid.UUID) (int64, error) {
result, err := q.db.Exec(ctx, upgradeAttachmentPipeline, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
+60
View File
@@ -0,0 +1,60 @@
package worker
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"mtusi2026-messenger/internal/db"
s3store "mtusi2026-messenger/internal/s3"
)
var (
errAttachmentNotUpgradeable = errors.New("attachment is not upgradeable: expected v1 queued or processing with staging")
errUpgradeNotUpdated = errors.New("attachment upgrade did not update one row")
)
func Upgrade(ctx context.Context, pool txBeginner, storage *s3store.Client, id uuid.UUID) error {
tx, err := pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := db.New(tx)
row, err := q.LockAttachmentForUpgrade(ctx, id)
if errors.Is(err, pgx.ErrNoRows) {
return errAttachmentNotUpgradeable
}
if err != nil {
return err
}
if row.JobID == nil {
return errMissingPipelineJob
}
if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil {
return err
}
n, err := q.UpgradeAttachmentPipeline(ctx, id)
if err != nil {
return err
}
if n != 1 {
return errUpgradeNotUpdated
}
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
AttachmentID: id,
Attempt: row.Attempts + 1,
AvailableAt: time.Now(),
}); err != nil {
return err
}
return tx.Commit(ctx)
}
+1 -1
View File
@@ -116,7 +116,7 @@ func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error {
job.Type != "attachment.process.v1" ||
job.JobID == uuid.Nil ||
job.AttachmentID == uuid.Nil ||
job.PipelineVersion != 1 ||
(job.PipelineVersion != 1 && job.PipelineVersion != 2) ||
job.Attempt < 1 ||
job.CreatedAt.IsZero() {
if err := delivery.Reject(false); err != nil {