Добавить версии pipeline и защиту от устаревших попыток

This commit is contained in:
TAskMAster339
2026-11-05 23:46:31 +03:00
parent 2da0b16383
commit c8ff31e5ff
16 changed files with 350 additions and 47 deletions
+15 -2
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"os"
"strconv"
"time"
"github.com/google/uuid"
@@ -14,7 +15,7 @@ import (
func runJobAdmin(args []string) error {
if len(args) < 1 {
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
@@ -43,6 +44,18 @@ func runJobAdmin(args []string) error {
}
var changed bool
switch {
case args[0] == "upgrade" && len(args) == 4:
version, e := strconv.Atoi(args[2])
if e != nil {
return errors.New("invalid pipeline version")
}
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "replay" && len(args) == 3:
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
@@ -54,7 +67,7 @@ func runJobAdmin(args []string) error {
case args[0] == "cleanup-staging" && len(args) == 2:
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
default:
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
if err != nil {
return err
+2 -5
View File
@@ -6,6 +6,7 @@ import (
"github.com/google/uuid"
"io"
"log"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"net/http"
"net/url"
@@ -120,11 +121,7 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc
found := false
for _, raw := range a.Variants {
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
extension := ".jpg"
if variants[0] == "optimized" {
extension = ".webp"
}
a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
a.ContentType = v["content_type"].(string)
a.SizeBytes = int64(v["size_bytes"].(float64))
found = true
+35
View File
@@ -1,9 +1,11 @@
package faults
import (
"context"
"log"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
)
@@ -27,3 +29,36 @@ func Crash(point, id string) {
os.Exit(86)
}
}
// Observe records a test-only branch without exposing an API or a lease token.
func Observe(point, id string) {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return
}
if _, err := uuid.Parse(id); err != nil {
return
}
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
}
// Pause is operator-only and ends on release or context expiry.
func Pause(ctx context.Context, point, id string) {
if !Take(point, id) {
return
}
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if _, err := os.Stat(release); err == nil {
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
return
}
}
}
}
+15 -14
View File
@@ -8,19 +8,20 @@ import (
)
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
type AttachmentStore struct{ db *pgxpool.Pool }
@@ -37,7 +38,7 @@ func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (
defer cancel()
a := Attachment{Variants: []any{}}
var variants []byte
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode)
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
if err == nil {
err = json.Unmarshal(variants, &a.Variants)
}
+6 -6
View File
@@ -18,7 +18,7 @@ type FailedJob struct {
}
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`)
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
if err != nil {
return nil, err
}
@@ -45,12 +45,12 @@ func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exis
}
defer tx.Rollback(context.Background())
var status, key, attachment string
var deleted bool
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted)
var deleted, obsolete bool
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
if err != nil {
return false, err
}
if status != "failed" {
if obsolete || status != "failed" {
return false, nil
}
if deleted {
@@ -84,7 +84,7 @@ func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, rete
defer tx.Rollback(context.Background())
var key, attachment, status string
var eligible bool
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
@@ -110,7 +110,7 @@ func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, rete
}
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
if err != nil {
return 0, err
}
+12 -6
View File
@@ -32,7 +32,7 @@ type Job struct {
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
// Re-publication is intentional: a durable intent is never removed by publish.
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
if err != nil {
return nil, err
}
@@ -64,7 +64,7 @@ func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.
var status string
var attempts, generation int
var available bool
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
@@ -98,6 +98,8 @@ func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.
return token, tx.Commit(ctx)
}
var ErrStaleJob = errors.New("stale job lease or version")
var ErrProcessingTimeout = errors.New("processing timeout")
// ProcessJob holds the row lock across deterministic object writes. The lock and
@@ -111,9 +113,9 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
var staging string
var attempts, generation int
var a Attachment
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
if errors.Is(err, pgx.ErrNoRows) {
return errors.New("stale job lease")
return ErrStaleJob
}
if err != nil {
return err
@@ -121,6 +123,7 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
if a.Status != "processing" {
return errors.New("job is not processing")
}
a.PipelineVersion = j.PipelineVersion
variants, code, workErr := process(a, staging)
var next *time.Time
var auditCode *string
@@ -129,7 +132,10 @@ func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, m
if e != nil {
return e
}
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,published_pipeline_version=$3,error_code=NULL,updated_at=now() WHERE id=$1 AND pipeline_version=$3`, a.ID, data, j.PipelineVersion)
if err == nil {
_, err = tx.Exec(ctx, `INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) VALUES($1,$2,$3,$4)`, a.ID, j.ID, j.PipelineVersion, data)
}
} else {
reason := "temporary_failure"
if errors.Is(workErr, ErrProcessingTimeout) {
@@ -202,6 +208,6 @@ func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) er
func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, error) {
var done bool
err := s.db.QueryRow(ctx, `SELECT a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
return done, err
}
@@ -0,0 +1,11 @@
-- Downgrade is intentionally refused if multiple versions exist: never drop jobs.
DO $$ BEGIN
IF EXISTS(SELECT 1 FROM attachment_jobs WHERE pipeline_version<>1) THEN
RAISE EXCEPTION 'cannot downgrade while versioned jobs exist';
END IF;
END $$;
DROP TABLE attachment_publications,attachment_pipeline_upgrades;
ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_version_key;
ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_attachment_id_key UNIQUE(attachment_id);
ALTER TABLE attachment_jobs DROP COLUMN superseded_at;
ALTER TABLE attachments DROP COLUMN pipeline_version,DROP COLUMN published_pipeline_version;
@@ -0,0 +1,19 @@
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version BETWEEN 1 AND 2);
ALTER TABLE attachments ADD COLUMN published_pipeline_version integer NOT NULL DEFAULT 0 CHECK(published_pipeline_version BETWEEN 0 AND 2);
UPDATE attachments SET published_pipeline_version=1 WHERE status='ready';
ALTER TABLE attachment_jobs DROP CONSTRAINT attachment_jobs_attachment_id_key;
ALTER TABLE attachment_jobs ADD CONSTRAINT attachment_jobs_version_key UNIQUE(attachment_id,pipeline_version);
ALTER TABLE attachment_jobs ADD COLUMN superseded_at timestamptz;
CREATE TABLE attachment_pipeline_upgrades (
id bigserial PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id),
from_version integer NOT NULL,to_version integer NOT NULL,reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE attachment_publications (
id bigserial PRIMARY KEY,attachment_id uuid NOT NULL REFERENCES attachments(id),
job_id uuid NOT NULL REFERENCES attachment_jobs(id),pipeline_version integer NOT NULL,
variants jsonb NOT NULL,published_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants)
SELECT a.id,j.id,1,a.variants FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id
WHERE a.status='ready';
@@ -0,0 +1,74 @@
package repositories
import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"strings"
"unicode/utf8"
)
// VariantObjectKey separates effects of different pipelines, including retries.
func VariantObjectKey(id string, version int, name string) string {
if version < 1 {
version = 1
}
extension := ".jpg"
if name == "optimized" {
extension = ".webp"
}
return fmt.Sprintf("variants/%s/v%d/%s%s", id, version, name, extension)
}
// UpgradePipeline is an explicit operator action. It never downgrades a result.
// The row lock serializes this change with publication, replay and cleanup.
func (s *AttachmentStore) UpgradePipeline(ctx context.Context, id string, version int, reason string, exists func(context.Context, string) error) (bool, error) {
reason = strings.TrimSpace(reason)
if reason == "" || utf8.RuneCountInString(reason) > 200 {
return false, errors.New("upgrade reason must contain 1-200 characters")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var current int
var status, original, source string
err = tx.QueryRow(ctx, `SELECT a.pipeline_version,a.status,a.object_key,j.staging_key FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND j.pipeline_version=a.pipeline_version FOR UPDATE OF j,a`, id).Scan(&current, &status, &original, &source)
if err != nil {
return false, err
}
if version <= current {
return false, nil
}
if version != current+1 || version > 2 {
return false, errors.New("only sequential upgrade to pipeline 2 is supported")
}
if status == "ready" {
source = original
}
if source != "staging/jobs/"+id && source != "originals/"+id {
return false, errors.New("invalid pipeline source")
}
if err = exists(ctx, source); err != nil {
return false, errors.New("pipeline source unavailable")
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET superseded_at=clock_timestamp(),lease_token=NULL,lease_until=NULL WHERE attachment_id=$1 AND pipeline_version<$2`, id, version); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='superseded' WHERE job_id IN (SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version<$2) AND finished_at IS NULL`, id, version); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,pipeline_version) VALUES($1,$2,$3,$4)`, uuid.NewString(), id, source, version); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET pipeline_version=$2,status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=$1`, id, version); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_pipeline_upgrades(attachment_id,from_version,to_version,reason) VALUES($1,$2,$3,$4)`, id, current, version, reason); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
@@ -0,0 +1,77 @@
package tests
import (
"context"
"errors"
"github.com/google/uuid"
"messenger/src/backend/repositories"
"testing"
"time"
)
func TestPipelineUpgradeFencesOldLeaseAndPublishesV2(t *testing.T) {
f := newTestFixture(t)
ctx := context.Background()
store := repositories.NewAttachmentStore(f.db)
user, _ := createTestUser(t, f.users, "versions", "Versions")
chat := createTestChat(t, f.chats, "Versions", []string{user.ID})
id, oldID := uuid.NewString(), uuid.NewString()
if err := store.Create(ctx, repositories.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "v.png", ObjectKey: "originals/" + id, ContentType: "image/png", SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
if _, err := f.db.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1`, id); err != nil {
t.Fatal(err)
}
if _, err := f.db.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, oldID, id, "staging/jobs/"+id); err != nil {
t.Fatal(err)
}
old := repositories.Job{ID: oldID, AttachmentID: id, PipelineVersion: 1}
token, err := store.StartAttempt(ctx, old, time.Minute, 5)
if err != nil || token == "" {
t.Fatal(err)
}
changed, err := store.UpgradePipeline(ctx, id, 2, "unit test upgrade", func(context.Context, string) error { return nil })
if err != nil || !changed {
t.Fatal(err)
}
effects := 0
callback := func(a repositories.Attachment, _ string) ([]repositories.Variant, string, error) {
effects++
if a.PipelineVersion != 2 {
t.Fatal("old version reached effects")
}
return []repositories.Variant{{Name: "thumbnail"}, {Name: "crop"}, {Name: "optimized"}}, "", nil
}
if err = store.ProcessJob(ctx, old, token, 5, callback); !errors.Is(err, repositories.ErrStaleJob) || effects != 0 {
t.Fatal("old lease not fenced", err)
}
if terminal, err := store.JobTerminal(ctx, oldID); err != nil || !terminal {
t.Fatal("superseded delivery not terminal", err)
}
var newID string
if err = f.db.QueryRow(ctx, `SELECT id FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&newID); err != nil {
t.Fatal(err)
}
next := repositories.Job{ID: newID, AttachmentID: id, PipelineVersion: 2}
token, err = store.StartAttempt(ctx, next, time.Minute, 5)
if err != nil || token == "" {
t.Fatal(err)
}
if err = store.ProcessJob(ctx, next, token, 5, callback); err != nil || effects != 1 {
t.Fatal(err)
}
published, err := store.GetForMember(ctx, id, user.ID)
if err != nil || published.Status != "ready" || published.PipelineVersion != 2 {
t.Fatal("wrong active publication", err)
}
if changed, err = store.UpgradePipeline(ctx, id, 1, "no downgrade", nil); err != nil || changed {
t.Fatal("version rollback allowed", err)
}
if changed, err = store.ReplayJob(ctx, oldID, "obsolete replay", nil); err != nil || changed {
t.Fatal("old job reopened", err)
}
var count int
if err = f.db.QueryRow(ctx, `SELECT count(*) FROM attachment_publications WHERE attachment_id=$1 AND pipeline_version=2`, id).Scan(&count); err != nil || count != 1 {
t.Fatal("publication count", count, err)
}
}
+7
View File
@@ -121,6 +121,13 @@ func newTestFixture(t *testing.T) *testFixture {
if _, err = db.Exec(ctx, string(ddl)); err != nil {
t.Fatal(err)
}
ddl, err = os.ReadFile("../repositories/migrations/000009_pipeline_versions.up.sql")
if err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, string(ddl)); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = redisClient.Close() })
if err := redisClient.Ping(ctx).Err(); err != nil {
t.Fatal(err)
+2 -6
View File
@@ -15,7 +15,7 @@ import (
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
return renderInSubprocess(ctx, a.ID, data)
return renderInSubprocess(ctx, a.ID, a.PipelineVersion, data)
})
}
@@ -42,11 +42,7 @@ func publishImage(ctx context.Context, objects services.AttachmentObjects, a rep
}
metadata := []repositories.Variant{}
for _, variant := range variants {
extension := ".jpg"
if variant.Metadata.Name == "optimized" {
extension = ".webp"
}
key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension
key := repositories.VariantObjectKey(a.ID, a.PipelineVersion, variant.Metadata.Name)
if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil {
return nil, "", err
}
+41
View File
@@ -158,3 +158,44 @@ func TestOptimizedBoundsAndCenteredCrop(t *testing.T) {
t.Fatal("crop did not preserve the green center", pixel)
}
}
func TestVersionedRenderingHasDistinctImmutableResults(t *testing.T) {
if _, err := exec.LookPath("cwebp"); err != nil {
t.Skip("cwebp: use Docker builder tests")
}
imageData := image.NewNRGBA(image.Rect(0, 0, 48, 32))
for y := 0; y < 32; y++ {
for x := 0; x < 48; x++ {
imageData.SetNRGBA(x, y, color.NRGBA{R: uint8(x*17 + y), G: uint8(y*31 + x), B: uint8(x * y * 3), A: 255})
}
}
var input bytes.Buffer
if err := png.Encode(&input, imageData); err != nil {
t.Fatal(err)
}
original := append([]byte{}, input.Bytes()...)
first, _, err := renderVersion(context.Background(), input.Bytes(), 1)
if err != nil {
t.Fatal(err)
}
second, _, err := renderVersion(context.Background(), input.Bytes(), 2)
if err != nil {
t.Fatal(err)
}
if len(first) != 3 || len(second) != 3 {
t.Fatal("incomplete version")
}
changed := false
for i := range first {
if first[i].Metadata.Width != second[i].Metadata.Width || first[i].Metadata.Height != second[i].Metadata.Height {
t.Fatal("version changed geometry")
}
changed = changed || first[i].Metadata.SHA256 != second[i].Metadata.SHA256
if repositories.VariantObjectKey("sample", 1, first[i].Metadata.Name) == repositories.VariantObjectKey("sample", 2, second[i].Metadata.Name) {
t.Fatal("versions share an object key")
}
}
if !changed || !bytes.Equal(original, input.Bytes()) {
t.Fatal("versions indistinguishable or source mutated")
}
}
+15 -4
View File
@@ -8,6 +8,7 @@ import (
"io"
"os"
"os/exec"
"strconv"
"time"
"messenger/src/backend/internal/faults"
@@ -24,7 +25,7 @@ func (b *boundedOutput) Write(data []byte) (int, error) {
return b.Buffer.Write(data)
}
func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) {
func renderInSubprocess(ctx context.Context, id string, version int, data []byte) ([]encodedVariant, string, error) {
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
if err != nil {
return nil, "", err
@@ -41,7 +42,10 @@ func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedV
return nil, "", err
}
defer os.RemoveAll(directory)
command := exec.CommandContext(task, executable, "image-process", id)
if version < 1 {
version = 1
}
command := exec.CommandContext(task, executable, "image-process", id, strconv.Itoa(version))
// No database, broker or S3 credentials are needed by a decoder.
command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")}
command.Env = append(command.Env, "TMPDIR="+directory)
@@ -77,7 +81,7 @@ func workerPixelLimit() string {
}
func RunImageChild(args []string) error {
if len(args) != 1 {
if len(args) < 1 || len(args) > 2 {
return errors.New("image id required")
}
if faults.Take("transform_delay", args[0]) {
@@ -90,7 +94,14 @@ func RunImageChild(args []string) error {
if len(data) > int(services.MaxUploadBytes) {
return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"})
}
variants, code, err := renderImage(context.Background(), data)
version := 1
if len(args) == 2 {
version, err = strconv.Atoi(args[1])
if err != nil {
return err
}
}
variants, code, err := renderVersion(context.Background(), data, version)
if err != nil && code == "" {
return err
}
+13 -3
View File
@@ -53,6 +53,16 @@ func opaque(source image.Image) image.Image {
// renderImage runs only in the disposable image child in production. DecodeConfig
// bounds pixels before allocation; the container bounds total memory to 512 MiB.
func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
return renderVersion(ctx, data, 1)
}
func renderVersion(ctx context.Context, data []byte, version int) ([]encodedVariant, string, error) {
jpegQuality, webpQuality := 85, "80"
if version == 2 {
jpegQuality, webpQuality = 75, "65"
} else if version != 1 {
return nil, "invalid_pipeline", errors.New("unsupported pipeline")
}
pixels, err := strconv.Atoi(workerPixelLimit())
if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels {
return nil, "", errors.New("invalid WORKER_MAX_PIXELS")
@@ -73,13 +83,13 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er
w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256)
thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos))
var body bytes.Buffer
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil {
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: jpegQuality}); err != nil {
return nil, "", err
}
appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes())
crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos))
body = bytes.Buffer{}
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil {
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: jpegQuality}); err != nil {
return nil, "", err
}
appendVariant("crop", "image/jpeg", crop, body.Bytes())
@@ -103,7 +113,7 @@ func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, er
if closeErr != nil {
return nil, "", closeErr
}
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-")
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", webpQuality, "-m", "4", "-metadata", "none", input, "-o", "-")
webp, err := command.Output()
if err != nil {
return nil, "", errors.New("WebP encoding failed")
+6 -1
View File
@@ -134,13 +134,18 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
if workErr == nil && token != "" {
faults.Crash("before_effect", job.AttachmentID)
faults.Pause(task, "after_claim", job.AttachmentID)
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return processImage(task, objects, a, staging)
})
}
cancel()
if errors.Is(workErr, repositories.ErrStaleJob) {
faults.Observe("stale_rejected", job.AttachmentID)
}
completed := token != "" && workErr == nil
if workErr == nil && token == "" {
if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) {
completed, workErr = store.JobTerminal(ctx, job.ID)
}
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.