forked from mtusi2026/mtusi2026lab5
feat(worker): add pipeline version upgrade command
This commit is contained in:
+39
-3
@@ -24,9 +24,11 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
|
||||
errInvalidReplayID = errors.New("replay must be an attachment UUID")
|
||||
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
|
||||
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
|
||||
errInvalidReplayID = errors.New("replay must be an attachment UUID")
|
||||
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
|
||||
errConflictingUpgradeFlags = errors.New("upgrade cannot be combined with replay or list-failed")
|
||||
errInvalidUpgradeID = errors.New("upgrade must be an attachment UUID")
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -36,6 +38,7 @@ func main() {
|
||||
operator := flag.String("operator", "", "operator requesting replay")
|
||||
reason := flag.String("reason", "", "reason for replay")
|
||||
listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments")
|
||||
upgrade := flag.String("upgrade", "", "upgrade a v1 queued or processing attachment to v2; requires retained staging")
|
||||
flag.Parse()
|
||||
|
||||
if *maxPixels > 0 {
|
||||
@@ -56,12 +59,16 @@ func main() {
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case *upgrade != "" && (*listFailed || *replay != ""):
|
||||
err = errConflictingUpgradeFlags
|
||||
case *listFailed && *replay != "":
|
||||
err = errConflictingReplayFlags
|
||||
case *listFailed:
|
||||
err = runListFailed(ctx)
|
||||
case *replay != "":
|
||||
err = runReplay(ctx, *replay, *operator, *reason)
|
||||
case *upgrade != "":
|
||||
err = runUpgrade(ctx, *upgrade)
|
||||
default:
|
||||
err = runWorker(ctx)
|
||||
}
|
||||
@@ -133,6 +140,35 @@ func runReplay(ctx context.Context, replay, operator, reason string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func runUpgrade(ctx context.Context, upgrade string) error {
|
||||
id, err := uuid.Parse(upgrade)
|
||||
if err != nil {
|
||||
return errInvalidUpgradeID
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cfg, pool, err := loadDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
log := logger.New(cfg.Logging.Level)
|
||||
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := worker.Upgrade(ctx, pool, storage, id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("attachment pipeline upgrade queued", "attachment_id", id, "pipeline_version", 2)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runWorker(ctx context.Context) error {
|
||||
cfg, pool, err := loadDatabase(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -35,7 +35,7 @@ CREATE TABLE outbox (
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
UNIQUE (job_id, attempt)
|
||||
UNIQUE (job_id, pipeline_version, attempt)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
@@ -113,6 +113,14 @@ WHERE id = sqlc.arg(id)::uuid
|
||||
)
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: LockAttachmentForUpgrade :one
|
||||
SELECT * FROM attachments
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND pipeline_version = 1
|
||||
AND status IN ('queued', 'processing')
|
||||
AND staging_key IS NOT NULL
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: ClearAttachmentStagingKey :exec
|
||||
UPDATE attachments
|
||||
SET staging_key = NULL
|
||||
@@ -160,6 +168,19 @@ WHERE id = sqlc.arg(id)::uuid
|
||||
AND attempts = sqlc.arg(attempt)::int
|
||||
AND lease_until > now();
|
||||
|
||||
-- name: UpgradeAttachmentPipeline :execrows
|
||||
UPDATE attachments
|
||||
SET pipeline_version = 2,
|
||||
status = 'queued',
|
||||
retry_base = attempts,
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND pipeline_version = 1
|
||||
AND status IN ('queued', 'processing')
|
||||
AND staging_key IS NOT NULL;
|
||||
|
||||
-- name: ReplayAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
|
||||
@@ -509,6 +509,43 @@ func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey str
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockAttachmentForUpgrade = `-- name: LockAttachmentForUpgrade :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
AND pipeline_version = 1
|
||||
AND status IN ('queued', 'processing')
|
||||
AND staging_key IS NOT NULL
|
||||
FOR UPDATE
|
||||
`
|
||||
|
||||
func (q *Queries) LockAttachmentForUpgrade(ctx context.Context, id uuid.UUID) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, lockAttachmentForUpgrade, id)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.RetryBase,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockAttachmentJob = `-- name: LockAttachmentJob :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, retry_base, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
@@ -738,3 +775,25 @@ func (q *Queries) RetryAttachmentJob(ctx context.Context, arg RetryAttachmentJob
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const upgradeAttachmentPipeline = `-- name: UpgradeAttachmentPipeline :execrows
|
||||
UPDATE attachments
|
||||
SET pipeline_version = 2,
|
||||
status = 'queued',
|
||||
retry_base = attempts,
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND pipeline_version = 1
|
||||
AND status IN ('queued', 'processing')
|
||||
AND staging_key IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) UpgradeAttachmentPipeline(ctx context.Context, id uuid.UUID) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, upgradeAttachmentPipeline, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
var (
|
||||
errAttachmentNotUpgradeable = errors.New("attachment is not upgradeable: expected v1 queued or processing with staging")
|
||||
errUpgradeNotUpdated = errors.New("attachment upgrade did not update one row")
|
||||
)
|
||||
|
||||
func Upgrade(ctx context.Context, pool txBeginner, storage *s3store.Client, id uuid.UUID) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := db.New(tx)
|
||||
row, err := q.LockAttachmentForUpgrade(ctx, id)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errAttachmentNotUpgradeable
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.JobID == nil {
|
||||
return errMissingPipelineJob
|
||||
}
|
||||
|
||||
if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := q.UpgradeAttachmentPipeline(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return errUpgradeNotUpdated
|
||||
}
|
||||
|
||||
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: id,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now(),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
@@ -116,7 +116,7 @@ func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error {
|
||||
job.Type != "attachment.process.v1" ||
|
||||
job.JobID == uuid.Nil ||
|
||||
job.AttachmentID == uuid.Nil ||
|
||||
job.PipelineVersion != 1 ||
|
||||
(job.PipelineVersion != 1 && job.PipelineVersion != 2) ||
|
||||
job.Attempt < 1 ||
|
||||
job.CreatedAt.IsZero() {
|
||||
if err := delivery.Reject(false); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user