From c8ff31e5ff441820e8fadec05b46953ed7bc96e4 Mon Sep 17 00:00:00 2001 From: TAskMAster339 Date: Thu, 5 Nov 2026 23:46:31 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8=20pipeline=20?= =?UTF-8?q?=D0=B8=20=D0=B7=D0=B0=D1=89=D0=B8=D1=82=D1=83=20=D0=BE=D1=82=20?= =?UTF-8?q?=D1=83=D1=81=D1=82=D0=B0=D1=80=D0=B5=D0=B2=D1=88=D0=B8=D1=85=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=8B=D1=82=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/commands/jobs.go | 17 +++- src/backend/handlers/attachment_http.go | 7 +- src/backend/internal/faults/faults.go | 35 +++++++++ src/backend/repositories/attachment.go | 29 +++---- src/backend/repositories/job_admin.go | 12 +-- src/backend/repositories/jobs.go | 18 +++-- .../000009_pipeline_versions.down.sql | 11 +++ .../000009_pipeline_versions.up.sql | 19 +++++ src/backend/repositories/pipeline_versions.go | 74 ++++++++++++++++++ src/backend/tests/pipeline_versions_test.go | 77 +++++++++++++++++++ src/backend/tests/test_fixture_test.go | 7 ++ src/backend/workers/attachment.go | 8 +- src/backend/workers/attachment_test.go | 41 ++++++++++ src/backend/workers/image_process.go | 19 ++++- src/backend/workers/image_render.go | 16 +++- src/backend/workers/pipeline.go | 7 +- 16 files changed, 350 insertions(+), 47 deletions(-) create mode 100644 src/backend/repositories/migrations/000009_pipeline_versions.down.sql create mode 100644 src/backend/repositories/migrations/000009_pipeline_versions.up.sql create mode 100644 src/backend/repositories/pipeline_versions.go create mode 100644 src/backend/tests/pipeline_versions_test.go diff --git a/src/backend/commands/jobs.go b/src/backend/commands/jobs.go index e102d858..6af97e29 100644 --- a/src/backend/commands/jobs.go +++ b/src/backend/commands/jobs.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "os" + "strconv" "time" "github.com/google/uuid" @@ -14,7 +15,7 @@ import ( func runJobAdmin(args []string) error { if len(args) < 1 { - return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID") + return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON") } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -43,6 +44,18 @@ func runJobAdmin(args []string) error { } var changed bool switch { + case args[0] == "upgrade" && len(args) == 4: + version, e := strconv.Atoi(args[2]) + if e != nil { + return errors.New("invalid pipeline version") + } + changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error { + r, e := objects.Get(ctx, key) + if e == nil { + r.Close() + } + return e + }) case args[0] == "replay" && len(args) == 3: changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error { r, e := objects.Get(ctx, key) @@ -54,7 +67,7 @@ func runJobAdmin(args []string) error { case args[0] == "cleanup-staging" && len(args) == 2: changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete) default: - return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID") + return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON") } if err != nil { return err diff --git a/src/backend/handlers/attachment_http.go b/src/backend/handlers/attachment_http.go index 15c6cc82..f41bb458 100644 --- a/src/backend/handlers/attachment_http.go +++ b/src/backend/handlers/attachment_http.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "io" "log" + "messenger/src/backend/repositories" "messenger/src/backend/services" "net/http" "net/url" @@ -120,11 +121,7 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc found := false for _, raw := range a.Variants { if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] { - extension := ".jpg" - if variants[0] == "optimized" { - extension = ".webp" - } - a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension + a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0]) a.ContentType = v["content_type"].(string) a.SizeBytes = int64(v["size_bytes"].(float64)) found = true diff --git a/src/backend/internal/faults/faults.go b/src/backend/internal/faults/faults.go index 74c7c7ef..b6133e87 100644 --- a/src/backend/internal/faults/faults.go +++ b/src/backend/internal/faults/faults.go @@ -1,9 +1,11 @@ package faults import ( + "context" "log" "os" "path/filepath" + "time" "github.com/google/uuid" ) @@ -27,3 +29,36 @@ func Crash(point, id string) { os.Exit(86) } } + +// Observe records a test-only branch without exposing an API or a lease token. +func Observe(point, id string) { + directory := os.Getenv("PIPELINE_FAULT_DIR") + if directory == "" { + return + } + if _, err := uuid.Parse(id); err != nil { + return + } + _ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644) +} + +// Pause is operator-only and ends on release or context expiry. +func Pause(ctx context.Context, point, id string) { + if !Take(point, id) { + return + } + release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release") + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if _, err := os.Stat(release); err == nil { + _ = os.WriteFile(release+".observed", []byte("released\n"), 0644) + return + } + } + } +} diff --git a/src/backend/repositories/attachment.go b/src/backend/repositories/attachment.go index 50ff6bdb..f0be2cbe 100644 --- a/src/backend/repositories/attachment.go +++ b/src/backend/repositories/attachment.go @@ -8,19 +8,20 @@ import ( ) type Attachment struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - OwnerID string `json:"owner_id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - SizeBytes int64 `json:"size_bytes"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - Variants []any `json:"variants"` - ErrorCode *string `json:"error_code"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - ObjectKey string `json:"-"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` } type AttachmentStore struct{ db *pgxpool.Pool } @@ -37,7 +38,7 @@ func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) ( defer cancel() a := Attachment{Variants: []any{}} var variants []byte - err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode) + err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion) if err == nil { err = json.Unmarshal(variants, &a.Variants) } diff --git a/src/backend/repositories/job_admin.go b/src/backend/repositories/job_admin.go index e2be2e79..e280364c 100644 --- a/src/backend/repositories/job_admin.go +++ b/src/backend/repositories/job_admin.go @@ -18,7 +18,7 @@ type FailedJob struct { } func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) { - rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`) + rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`) if err != nil { return nil, err } @@ -45,12 +45,12 @@ func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exis } defer tx.Rollback(context.Background()) var status, key, attachment string - var deleted bool - err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted) + var deleted, obsolete bool + err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete) if err != nil { return false, err } - if status != "failed" { + if obsolete || status != "failed" { return false, nil } if deleted { @@ -84,7 +84,7 @@ func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, rete defer tx.Rollback(context.Background()) var key, attachment, status string var eligible bool - err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation) + err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation) if errors.Is(err, pgx.ErrNoRows) { - return errors.New("stale job lease") + return ErrStaleJob } if err != nil { return err @@ -121,6 +123,7 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m if a.Status != "processing" { return errors.New("job is not processing") } + a.PipelineVersion = j.PipelineVersion variants, code, workErr := process(a, staging) var next *time.Time var auditCode *string @@ -129,7 +132,10 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m if e != nil { return e } - _, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data) + _, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,published_pipeline_version=$3,error_code=NULL,updated_at=now() WHERE id=$1 AND pipeline_version=$3`, a.ID, data, j.PipelineVersion) + if err == nil { + _, err = tx.Exec(ctx, `INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) VALUES($1,$2,$3,$4)`, a.ID, j.ID, j.PipelineVersion, data) + } } else { reason := "temporary_failure" if errors.Is(workErr, ErrProcessingTimeout) { @@ -202,6 +208,6 @@ func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) er func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, error) { var done bool - err := s.db.QueryRow(ctx, `SELECT a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done) + err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version1) THEN + RAISE EXCEPTION 'cannot downgrade while versioned jobs exist'; + END IF; +END $$; +DROP TABLE attachment_publications,attachment_pipeline_upgrades; +ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_version_key; +ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_attachment_id_key UNIQUE(attachment_id); +ALTER TABLE attachment_jobs DROP COLUMN superseded_at; +ALTER TABLE attachments DROP COLUMN pipeline_version,DROP COLUMN published_pipeline_version; diff --git a/src/backend/repositories/migrations/000009_pipeline_versions.up.sql b/src/backend/repositories/migrations/000009_pipeline_versions.up.sql new file mode 100644 index 00000000..a00af7b8 --- /dev/null +++ b/src/backend/repositories/migrations/000009_pipeline_versions.up.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version BETWEEN 1 AND 2); +ALTER TABLE attachments ADD COLUMN published_pipeline_version integer NOT NULL DEFAULT 0 CHECK(published_pipeline_version BETWEEN 0 AND 2); +UPDATE attachments SET published_pipeline_version=1 WHERE status='ready'; +ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_attachment_id_key; +ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_version_key UNIQUE(attachment_id,pipeline_version); +ALTER TABLE attachment_jobs ADD COLUMN superseded_at timestamptz; +CREATE TABLE attachment_pipeline_upgrades ( + id bigserial PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + from_version integer NOT NULL,to_version integer NOT NULL,reason text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE attachment_publications ( + id bigserial PRIMARY KEY,attachment_id uuid NOT NULL REFERENCES attachments(id), + job_id uuid NOT NULL REFERENCES attachment_jobs(id),pipeline_version integer NOT NULL, + variants jsonb NOT NULL,published_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) + SELECT a.id,j.id,1,a.variants FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id + WHERE a.status='ready'; diff --git a/src/backend/repositories/pipeline_versions.go b/src/backend/repositories/pipeline_versions.go new file mode 100644 index 00000000..e6452950 --- /dev/null +++ b/src/backend/repositories/pipeline_versions.go @@ -0,0 +1,74 @@ +package repositories + +import ( + "context" + "errors" + "fmt" + "github.com/google/uuid" + "strings" + "unicode/utf8" +) + +// VariantObjectKey separates effects of different pipelines, including retries. +func VariantObjectKey(id string, version int, name string) string { + if version < 1 { + version = 1 + } + extension := ".jpg" + if name == "optimized" { + extension = ".webp" + } + return fmt.Sprintf("variants/%s/v%d/%s%s", id, version, name, extension) +} + +// UpgradePipeline is an explicit operator action. It never downgrades a result. +// The row lock serializes this change with publication, replay and cleanup. +func (s *AttachmentStore) UpgradePipeline(ctx context.Context, id string, version int, reason string, exists func(context.Context, string) error) (bool, error) { + reason = strings.TrimSpace(reason) + if reason == "" || utf8.RuneCountInString(reason) > 200 { + return false, errors.New("upgrade reason must contain 1-200 characters") + } + tx, err := s.db.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var current int + var status, original, source string + err = tx.QueryRow(ctx, `SELECT a.pipeline_version,a.status,a.object_key,j.staging_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j,a`, id).Scan(¤t, &status, &original, &source) + if err != nil { + return false, err + } + if version <= current { + return false, nil + } + if version != current+1 || version > 2 { + return false, errors.New("only sequential upgrade to pipeline 2 is supported") + } + + if status == "ready" { + source = original + } + if source != "staging/jobs/"+id && source != "originals/"+id { + return false, errors.New("invalid pipeline source") + } + if err = exists(ctx, source); err != nil { + return false, errors.New("pipeline source unavailable") + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET superseded_at=clock_timestamp(),lease_token=NULL,lease_until=NULL WHERE attachment_id=$1 AND pipeline_version<$2`, id, version); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='superseded' WHERE job_id IN (SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version<$2) AND finished_at IS NULL`, id, version); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,pipeline_version) VALUES($1,$2,$3,$4)`, uuid.NewString(), id, source, version); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET pipeline_version=$2,status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, id, version); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_pipeline_upgrades(attachment_id,from_version,to_version,reason) VALUES($1,$2,$3,$4)`, id, current, version, reason); err != nil { + return false, err + } + return true, tx.Commit(ctx) +} diff --git a/src/backend/tests/pipeline_versions_test.go b/src/backend/tests/pipeline_versions_test.go new file mode 100644 index 00000000..6d9d8a2c --- /dev/null +++ b/src/backend/tests/pipeline_versions_test.go @@ -0,0 +1,77 @@ +package tests + +import ( + "context" + "errors" + "github.com/google/uuid" + "messenger/src/backend/repositories" + "testing" + "time" +) + +func TestPipelineUpgradeFencesOldLeaseAndPublishesV2(t *testing.T) { + f := newTestFixture(t) + ctx := context.Background() + store := repositories.NewAttachmentStore(f.db) + user, _ := createTestUser(t, f.users, "versions", "Versions") + chat := createTestChat(t, f.chats, "Versions", []string{user.ID}) + id, oldID := uuid.NewString(), uuid.NewString() + if err := store.Create(ctx, repositories.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "v.png", ObjectKey: "originals/" + id, ContentType: "image/png", SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil { + t.Fatal(err) + } + if _, err := f.db.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1`, id); err != nil { + t.Fatal(err) + } + if _, err := f.db.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, oldID, id, "staging/jobs/"+id); err != nil { + t.Fatal(err) + } + old := repositories.Job{ID: oldID, AttachmentID: id, PipelineVersion: 1} + token, err := store.StartAttempt(ctx, old, time.Minute, 5) + if err != nil || token == "" { + t.Fatal(err) + } + changed, err := store.UpgradePipeline(ctx, id, 2, "unit test upgrade", func(context.Context, string) error { return nil }) + if err != nil || !changed { + t.Fatal(err) + } + effects := 0 + callback := func(a repositories.Attachment, _ string) ([]repositories.Variant, string, error) { + effects++ + if a.PipelineVersion != 2 { + t.Fatal("old version reached effects") + } + return []repositories.Variant{{Name: "thumbnail"}, {Name: "crop"}, {Name: "optimized"}}, "", nil + } + if err = store.ProcessJob(ctx, old, token, 5, callback); !errors.Is(err, repositories.ErrStaleJob) || effects != 0 { + t.Fatal("old lease not fenced", err) + } + if terminal, err := store.JobTerminal(ctx, oldID); err != nil || !terminal { + t.Fatal("superseded delivery not terminal", err) + } + var newID string + if err = f.db.QueryRow(ctx, `SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&newID); err != nil { + t.Fatal(err) + } + next := repositories.Job{ID: newID, AttachmentID: id, PipelineVersion: 2} + token, err = store.StartAttempt(ctx, next, time.Minute, 5) + if err != nil || token == "" { + t.Fatal(err) + } + if err = store.ProcessJob(ctx, next, token, 5, callback); err != nil || effects != 1 { + t.Fatal(err) + } + published, err := store.GetForMember(ctx, id, user.ID) + if err != nil || published.Status != "ready" || published.PipelineVersion != 2 { + t.Fatal("wrong active publication", err) + } + if changed, err = store.UpgradePipeline(ctx, id, 1, "no downgrade", nil); err != nil || changed { + t.Fatal("version rollback allowed", err) + } + if changed, err = store.ReplayJob(ctx, oldID, "obsolete replay", nil); err != nil || changed { + t.Fatal("old job reopened", err) + } + var count int + if err = f.db.QueryRow(ctx, `SELECT count(*) FROM attachment_publications WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&count); err != nil || count != 1 { + t.Fatal("publication count", count, err) + } +} diff --git a/src/backend/tests/test_fixture_test.go b/src/backend/tests/test_fixture_test.go index 51f3281a..4f0ee292 100644 --- a/src/backend/tests/test_fixture_test.go +++ b/src/backend/tests/test_fixture_test.go @@ -121,6 +121,13 @@ func newTestFixture(t *testing.T) *testFixture { if _, err = db.Exec(ctx, string(ddl)); err != nil { t.Fatal(err) } + ddl, err = os.ReadFile("../repositories/migrations/000009_pipeline_versions.up.sql") + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(ctx, string(ddl)); err != nil { + t.Fatal(err) + } t.Cleanup(func() { _ = redisClient.Close() }) if err := redisClient.Ping(ctx).Err(); err != nil { t.Fatal(err) diff --git a/src/backend/workers/attachment.go b/src/backend/workers/attachment.go index f0a21182..4faf6c24 100644 --- a/src/backend/workers/attachment.go +++ b/src/backend/workers/attachment.go @@ -15,7 +15,7 @@ import ( func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) { - return renderInSubprocess(ctx, a.ID, data) + return renderInSubprocess(ctx, a.ID, a.PipelineVersion, data) }) } @@ -42,11 +42,7 @@ func publishImage(ctx context.Context, objects services.AttachmentObjects, a rep } metadata := []repositories.Variant{} for _, variant := range variants { - extension := ".jpg" - if variant.Metadata.Name == "optimized" { - extension = ".webp" - } - key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension + key := repositories.VariantObjectKey(a.ID, a.PipelineVersion, variant.Metadata.Name) if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil { return nil, "", err } diff --git a/src/backend/workers/attachment_test.go b/src/backend/workers/attachment_test.go index 7b49a57a..a050410c 100644 --- a/src/backend/workers/attachment_test.go +++ b/src/backend/workers/attachment_test.go @@ -158,3 +158,44 @@ func TestOptimizedBoundsAndCenteredCrop(t *testing.T) { t.Fatal("crop did not preserve the green center", pixel) } } + +func TestVersionedRenderingHasDistinctImmutableResults(t *testing.T) { + if _, err := exec.LookPath("cwebp"); err != nil { + t.Skip("cwebp: use Docker builder tests") + } + imageData := image.NewNRGBA(image.Rect(0, 0, 48, 32)) + for y := 0; y < 32; y++ { + for x := 0; x < 48; x++ { + imageData.SetNRGBA(x, y, color.NRGBA{R: uint8(x*17 + y), G: uint8(y*31 + x), B: uint8(x * y * 3), A: 255}) + } + } + var input bytes.Buffer + if err := png.Encode(&input, imageData); err != nil { + t.Fatal(err) + } + original := append([]byte{}, input.Bytes()...) + first, _, err := renderVersion(context.Background(), input.Bytes(), 1) + if err != nil { + t.Fatal(err) + } + second, _, err := renderVersion(context.Background(), input.Bytes(), 2) + if err != nil { + t.Fatal(err) + } + if len(first) != 3 || len(second) != 3 { + t.Fatal("incomplete version") + } + changed := false + for i := range first { + if first[i].Metadata.Width != second[i].Metadata.Width || first[i].Metadata.Height != second[i].Metadata.Height { + t.Fatal("version changed geometry") + } + changed = changed || first[i].Metadata.SHA256 != second[i].Metadata.SHA256 + if repositories.VariantObjectKey("sample", 1, first[i].Metadata.Name) == repositories.VariantObjectKey("sample", 2, second[i].Metadata.Name) { + t.Fatal("versions share an object key") + } + } + if !changed || !bytes.Equal(original, input.Bytes()) { + t.Fatal("versions indistinguishable or source mutated") + } +} diff --git a/src/backend/workers/image_process.go b/src/backend/workers/image_process.go index ce5641c3..7778c9c9 100644 --- a/src/backend/workers/image_process.go +++ b/src/backend/workers/image_process.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "strconv" "time" "messenger/src/backend/internal/faults" @@ -24,7 +25,7 @@ func (b *boundedOutput) Write(data []byte) (int, error) { return b.Buffer.Write(data) } -func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) { +func renderInSubprocess(ctx context.Context, id string, version int, data []byte) ([]encodedVariant, string, error) { seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60) if err != nil { return nil, "", err @@ -41,7 +42,10 @@ func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedV return nil, "", err } defer os.RemoveAll(directory) - command := exec.CommandContext(task, executable, "image-process", id) + if version < 1 { + version = 1 + } + command := exec.CommandContext(task, executable, "image-process", id, strconv.Itoa(version)) // No database, broker or S3 credentials are needed by a decoder. command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")} command.Env = append(command.Env, "TMPDIR="+directory) @@ -77,7 +81,7 @@ func workerPixelLimit() string { } func RunImageChild(args []string) error { - if len(args) != 1 { + if len(args) < 1 || len(args) > 2 { return errors.New("image id required") } if faults.Take("transform_delay", args[0]) { @@ -90,7 +94,14 @@ func RunImageChild(args []string) error { if len(data) > int(services.MaxUploadBytes) { return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"}) } - variants, code, err := renderImage(context.Background(), data) + version := 1 + if len(args) == 2 { + version, err = strconv.Atoi(args[1]) + if err != nil { + return err + } + } + variants, code, err := renderVersion(context.Background(), data, version) if err != nil && code == "" { return err } diff --git a/src/backend/workers/image_render.go b/src/backend/workers/image_render.go index 7a7569d5..38330c3d 100644 --- a/src/backend/workers/image_render.go +++ b/src/backend/workers/image_render.go @@ -53,6 +53,16 @@ func opaque(source image.Image) image.Image { // renderImage runs only in the disposable image child in production. DecodeConfig // bounds pixels before allocation; the container bounds total memory to 512 MiB. func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) { + return renderVersion(ctx, data, 1) +} + +func renderVersion(ctx context.Context, data []byte, version int) ([]encodedVariant, string, error) { + jpegQuality, webpQuality := 85, "80" + if version == 2 { + jpegQuality, webpQuality = 75, "65" + } else if version != 1 { + return nil, "invalid_pipeline", errors.New("unsupported pipeline") + } pixels, err := strconv.Atoi(workerPixelLimit()) if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels { return nil, "", errors.New("invalid WORKER_MAX_PIXELS") @@ -73,13 +83,13 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256) thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos)) var body bytes.Buffer - if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil { + if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: jpegQuality}); err != nil { return nil, "", err } appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes()) crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos)) body = bytes.Buffer{} - if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil { + if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: jpegQuality}); err != nil { return nil, "", err } appendVariant("crop", "image/jpeg", crop, body.Bytes()) @@ -103,7 +113,7 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er if closeErr != nil { return nil, "", closeErr } - command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-") + command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", webpQuality, "-m", "4", "-metadata", "none", input, "-o", "-") webp, err := command.Output() if err != nil { return nil, "", errors.New("WebP encoding failed") diff --git a/src/backend/workers/pipeline.go b/src/backend/workers/pipeline.go index e19bda97..612d6d1a 100644 --- a/src/backend/workers/pipeline.go +++ b/src/backend/workers/pipeline.go @@ -134,13 +134,18 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec task, cancel := context.WithTimeout(ctx, timeout+15*time.Second) token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts) if workErr == nil && token != "" { + faults.Crash("before_effect", job.AttachmentID) + faults.Pause(task, "after_claim", job.AttachmentID) workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { return processImage(task, objects, a, staging) }) } cancel() + if errors.Is(workErr, repositories.ErrStaleJob) { + faults.Observe("stale_rejected", job.AttachmentID) + } completed := token != "" && workErr == nil - if workErr == nil && token == "" { + if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) { completed, workErr = store.JobTerminal(ctx, job.ID) } // Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.